package research import ( "context" "crypto/sha256" "encoding/hex" "errors" "html" "io" "net/http" "net/url" "regexp" "strconv" "strings" "git.cin.su/m/Assistant/contracts/epistemics" ) var ErrSearchRejected = errors.New("web search request rejected") var ( searchLinkPattern = regexp.MustCompile(`(?is)]+class=["'][^"']*result__a[^"']*["'][^>]*>(.*?)`) searchSnippetPattern = regexp.MustCompile(`(?is)]+class=["'][^"']*result__snippet[^"']*["'][^>]*>(.*?)`) bingLinkPattern = regexp.MustCompile(`(?is)
]*>(.*?)
`) tagPattern = regexp.MustCompile(`(?s)<[^>]*>`) termPattern = regexp.MustCompile(`(?i)[\p{L}\p{N}]{2,}`) ) var searchStopTerms = map[string]struct{}{ "какая": {}, "какое": {}, "какие": {}, "какой": {}, "что": {}, "как": {}, "где": {}, "когда": {}, "почему": {}, "мне": {}, "пожалуйста": {}, "what": {}, "which": {}, "where": {}, "when": {}, "why": {}, "how": {}, "please": {}, "the": {}, "this": {}, } // SearchConnector uses a configured HTTPS search endpoint. It does not know // cities, weather terms, entities or model names: the complete semantic query // comes from the interpretation layer. type SearchConnector struct { Client *http.Client EndpointTemplate string AllowedHosts map[string]struct{} MaxBytes int64 MaxResults int } func (c SearchConnector) Collect(ctx context.Context, request epistemics.WebSearchRequest, nowUnixMillis int64) ([]epistemics.ResearchCandidate, error) { if request.Plan.Validate() != nil || strings.TrimSpace(request.Query) == "" || nowUnixMillis < 1 || !strings.Contains(c.EndpointTemplate, "%s") { return nil, ErrSearchRejected } endpoint, err := url.Parse(strings.Replace(c.EndpointTemplate, "%s", url.QueryEscape(strings.TrimSpace(request.Query)), 1)) if err != nil || endpoint.Scheme != "https" || endpoint.Host == "" || !c.allowedHost(strings.ToLower(endpoint.Hostname())) { return nil, ErrSearchRejected } client := c.Client if client == nil { client = &http.Client{} } maxBytes := c.MaxBytes if maxBytes <= 0 || maxBytes > 4<<20 { maxBytes = 512 << 10 } req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) if err != nil { return nil, ErrSearchRejected } response, err := client.Do(req) if err != nil { return nil, err } defer response.Body.Close() if response.StatusCode < 200 || response.StatusCode >= 300 { return nil, ErrSearchRejected } body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1)) if err != nil || len(body) == 0 || int64(len(body)) > maxBytes { return nil, ErrSearchRejected } links := searchLinkPattern.FindAllSubmatch(body, c.limit()) snippets := searchSnippetPattern.FindAllSubmatch(body, c.limit()) if len(links) == 0 { links = bingLinkPattern.FindAllSubmatch(body, c.limit()) snippets = bingSnippetPattern.FindAllSubmatch(body, c.limit()) } if len(links) == 0 { return nil, nil } digest := sha256.Sum256(body) digestText := "sha256:" + hex.EncodeToString(digest[:]) base := hex.EncodeToString(digest[:]) sourceID := epistemics.SourceID("source:search:" + base) snapshotID := epistemics.SourceSnapshotID("snapshot:search:" + base) results := make([]epistemics.ResearchCandidate, 0, len(links)) for index, link := range links { title := cleanHTML(string(link[1])) snippet := "" if index < len(snippets) { snippet = cleanHTML(string(snippets[index][1])) } statement := strings.TrimSpace(strings.Trim(title+" — "+snippet, " —")) if statement == "" || !searchStatementRelatesToQuery(request.Query, statement) { continue } ordinal := strconv.Itoa(index) excerptID := epistemics.SourceExcerptID("excerpt:search:" + base + ":" + ordinal) results = append(results, epistemics.ResearchCandidate{ PropositionKey: request.PropositionKey + ":result:" + ordinal, Statement: statement, Source: epistemics.Source{ID: sourceID, Kind: "web-search", Locator: endpoint.String(), Title: title, Publisher: endpoint.Hostname()}, Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: sourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent}, Excerpt: epistemics.SourceExcerpt{ID: excerptID, SnapshotID: snapshotID, Text: statement, Start: 0, End: int64(len(statement))}, Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone, }) } return results, nil } // searchStatementRelatesToQuery is a bounded relevance gate, not a truth // check. It prevents generic search pages from becoming apparent evidence // when their content has no substantive relation to the requested query. func searchStatementRelatesToQuery(query, statement string) bool { terms := termPattern.FindAllString(strings.ToLower(query), -1) content := strings.ToLower(statement) usable := 0 for _, term := range terms { if _, stop := searchStopTerms[term]; stop || len([]rune(term)) < 4 { continue } usable++ runes := []rune(term) prefixLen := 5 if len(runes) < prefixLen { prefixLen = len(runes) } if strings.Contains(content, string(runes[:prefixLen])) { return true } } return usable == 0 } func (c SearchConnector) allowedHost(host string) bool { _, ok := c.AllowedHosts[host]; return ok } func (c SearchConnector) limit() int { if c.MaxResults <= 0 || c.MaxResults > 8 { return 3 } return c.MaxResults } func cleanHTML(value string) string { return strings.Join(strings.Fields(html.UnescapeString(tagPattern.ReplaceAllString(value, ""))), " ") }