75 lines
2.8 KiB
Go
75 lines
2.8 KiB
Go
package research
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
|
|
"git.cin.su/m/Assistant/contracts/epistemics"
|
|
)
|
|
|
|
var ErrHTTPSourceRejected = errors.New("http research source rejected")
|
|
|
|
type HTTPConnector struct {
|
|
Client *http.Client
|
|
AllowedHosts map[string]struct{}
|
|
MaxBytes int64
|
|
}
|
|
|
|
func (c HTTPConnector) Collect(ctx context.Context, request epistemics.WebResearchRequest, nowUnixMillis int64) (epistemics.ResearchCandidate, error) {
|
|
if request.Plan.Validate() != nil || request.SourceID == "" || strings.TrimSpace(request.Statement) == "" || nowUnixMillis < 1 {
|
|
return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected
|
|
}
|
|
u, err := url.Parse(request.URL)
|
|
if err != nil || u.Scheme != "https" || u.User != nil || u.Host == "" || !c.allowedHost(strings.ToLower(u.Hostname())) {
|
|
return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected
|
|
}
|
|
client := c.Client
|
|
if client == nil {
|
|
client = &http.Client{}
|
|
}
|
|
maxBytes := c.MaxBytes
|
|
if maxBytes <= 0 || maxBytes > 4<<20 {
|
|
maxBytes = 256 << 10
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
if err != nil {
|
|
return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected
|
|
}
|
|
response, err := client.Do(req)
|
|
if err != nil {
|
|
return epistemics.ResearchCandidate{}, fmt.Errorf("read web source: %w", err)
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return epistemics.ResearchCandidate{}, fmt.Errorf("web source status: %s", response.Status)
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1))
|
|
if err != nil || int64(len(body)) > maxBytes || len(body) == 0 {
|
|
return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected
|
|
}
|
|
digest := sha256.Sum256(body)
|
|
digestText := "sha256:" + hex.EncodeToString(digest[:])
|
|
key := sha256.Sum256([]byte(u.String() + "\x00" + digestText))
|
|
keyText := hex.EncodeToString(key[:])
|
|
snapshotID := epistemics.SourceSnapshotID("snapshot:" + keyText)
|
|
return epistemics.ResearchCandidate{
|
|
PropositionKey: request.PropositionKey, Statement: request.Statement,
|
|
Source: epistemics.Source{ID: request.SourceID, Kind: "web", Locator: u.String(), Title: request.Title, Publisher: u.Hostname()},
|
|
Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: request.SourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent},
|
|
Excerpt: epistemics.SourceExcerpt{ID: epistemics.SourceExcerptID("excerpt:" + keyText), SnapshotID: snapshotID, Text: string(body), Start: 0, End: int64(len(body))},
|
|
Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone,
|
|
}, nil
|
|
}
|
|
|
|
func (c HTTPConnector) allowedHost(host string) bool {
|
|
_, ok := c.AllowedHosts[host]
|
|
return ok
|
|
}
|