package research import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" "strings" "git.cin.su/m/Assistant/contracts/epistemics" ) var ErrHTTPSourceRejected = errors.New("http research source rejected") type HTTPConnector struct { Client *http.Client AllowedHosts map[string]struct{} MaxBytes int64 } func (c HTTPConnector) Collect(ctx context.Context, request epistemics.WebResearchRequest, nowUnixMillis int64) (epistemics.ResearchCandidate, error) { if request.Plan.Validate() != nil || request.SourceID == "" || strings.TrimSpace(request.Statement) == "" || nowUnixMillis < 1 { return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected } u, err := url.Parse(request.URL) if err != nil || u.Scheme != "https" || u.User != nil || u.Host == "" || !c.allowedHost(strings.ToLower(u.Hostname())) { return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected } client := c.Client if client == nil { client = &http.Client{} } maxBytes := c.MaxBytes if maxBytes <= 0 || maxBytes > 4<<20 { maxBytes = 256 << 10 } req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected } response, err := client.Do(req) if err != nil { return epistemics.ResearchCandidate{}, fmt.Errorf("read web source: %w", err) } defer response.Body.Close() if response.StatusCode < 200 || response.StatusCode >= 300 { return epistemics.ResearchCandidate{}, fmt.Errorf("web source status: %s", response.Status) } body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1)) if err != nil || int64(len(body)) > maxBytes || len(body) == 0 { return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected } digest := sha256.Sum256(body) digestText := "sha256:" + hex.EncodeToString(digest[:]) key := sha256.Sum256([]byte(u.String() + "\x00" + digestText)) keyText := hex.EncodeToString(key[:]) snapshotID := epistemics.SourceSnapshotID("snapshot:" + keyText) return epistemics.ResearchCandidate{ PropositionKey: request.PropositionKey, Statement: request.Statement, Source: epistemics.Source{ID: request.SourceID, Kind: "web", Locator: u.String(), Title: request.Title, Publisher: u.Hostname()}, Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: request.SourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent}, Excerpt: epistemics.SourceExcerpt{ID: epistemics.SourceExcerptID("excerpt:" + keyText), SnapshotID: snapshotID, Text: string(body), Start: 0, End: int64(len(body))}, Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone, }, nil } func (c HTTPConnector) allowedHost(host string) bool { _, ok := c.AllowedHosts[host] return ok }