Files
Assistant/adapters/research/search_connector.go
m 3c46a5f07f
CI / test (push) Successful in 51s
CI / test (pull_request) Successful in 53s
fix: reject irrelevant research excerpts
2026-08-24 06:40:31 +03:00

151 lines
5.7 KiB
Go

package research
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"html"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"git.cin.su/m/Assistant/contracts/epistemics"
)
var ErrSearchRejected = errors.New("web search request rejected")
var (
searchLinkPattern = regexp.MustCompile(`(?is)<a[^>]+class=["'][^"']*result__a[^"']*["'][^>]*>(.*?)</a>`)
searchSnippetPattern = regexp.MustCompile(`(?is)<a[^>]+class=["'][^"']*result__snippet[^"']*["'][^>]*>(.*?)</a>`)
bingLinkPattern = regexp.MustCompile(`(?is)<li[^>]+class=["'][^"']*b_algo[^"']*["'][^>]*>.*?<h2[^>]*><a[^>]*>(.*?)</a>`)
bingSnippetPattern = regexp.MustCompile(`(?is)<li[^>]+class=["'][^"']*b_algo[^"']*["'][^>]*>.*?<p[^>]*>(.*?)</p>`)
tagPattern = regexp.MustCompile(`(?s)<[^>]*>`)
termPattern = regexp.MustCompile(`(?i)[\p{L}\p{N}]{2,}`)
)
var searchStopTerms = map[string]struct{}{
"какая": {}, "какое": {}, "какие": {}, "какой": {}, "что": {}, "как": {}, "где": {}, "когда": {}, "почему": {}, "мне": {}, "пожалуйста": {},
"what": {}, "which": {}, "where": {}, "when": {}, "why": {}, "how": {}, "please": {}, "the": {}, "this": {},
}
// SearchConnector uses a configured HTTPS search endpoint. It does not know
// cities, weather terms, entities or model names: the complete semantic query
// comes from the interpretation layer.
type SearchConnector struct {
Client *http.Client
EndpointTemplate string
AllowedHosts map[string]struct{}
MaxBytes int64
MaxResults int
}
func (c SearchConnector) Collect(ctx context.Context, request epistemics.WebSearchRequest, nowUnixMillis int64) ([]epistemics.ResearchCandidate, error) {
if request.Plan.Validate() != nil || strings.TrimSpace(request.Query) == "" || nowUnixMillis < 1 || !strings.Contains(c.EndpointTemplate, "%s") {
return nil, ErrSearchRejected
}
endpoint, err := url.Parse(strings.Replace(c.EndpointTemplate, "%s", url.QueryEscape(strings.TrimSpace(request.Query)), 1))
if err != nil || endpoint.Scheme != "https" || endpoint.Host == "" || !c.allowedHost(strings.ToLower(endpoint.Hostname())) {
return nil, ErrSearchRejected
}
client := c.Client
if client == nil {
client = &http.Client{}
}
maxBytes := c.MaxBytes
if maxBytes <= 0 || maxBytes > 4<<20 {
maxBytes = 512 << 10
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
if err != nil {
return nil, ErrSearchRejected
}
response, err := client.Do(req)
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return nil, ErrSearchRejected
}
body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1))
if err != nil || len(body) == 0 || int64(len(body)) > maxBytes {
return nil, ErrSearchRejected
}
links := searchLinkPattern.FindAllSubmatch(body, c.limit())
snippets := searchSnippetPattern.FindAllSubmatch(body, c.limit())
if len(links) == 0 {
links = bingLinkPattern.FindAllSubmatch(body, c.limit())
snippets = bingSnippetPattern.FindAllSubmatch(body, c.limit())
}
if len(links) == 0 {
return nil, nil
}
digest := sha256.Sum256(body)
digestText := "sha256:" + hex.EncodeToString(digest[:])
base := hex.EncodeToString(digest[:])
sourceID := epistemics.SourceID("source:search:" + base)
snapshotID := epistemics.SourceSnapshotID("snapshot:search:" + base)
results := make([]epistemics.ResearchCandidate, 0, len(links))
for index, link := range links {
title := cleanHTML(string(link[1]))
snippet := ""
if index < len(snippets) {
snippet = cleanHTML(string(snippets[index][1]))
}
statement := strings.TrimSpace(strings.Trim(title+" — "+snippet, " —"))
if statement == "" || !searchStatementRelatesToQuery(request.Query, statement) {
continue
}
ordinal := strconv.Itoa(index)
excerptID := epistemics.SourceExcerptID("excerpt:search:" + base + ":" + ordinal)
results = append(results, epistemics.ResearchCandidate{
PropositionKey: request.PropositionKey + ":result:" + ordinal,
Statement: statement,
Source: epistemics.Source{ID: sourceID, Kind: "web-search", Locator: endpoint.String(), Title: title, Publisher: endpoint.Hostname()},
Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: sourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent},
Excerpt: epistemics.SourceExcerpt{ID: excerptID, SnapshotID: snapshotID, Text: statement, Start: 0, End: int64(len(statement))},
Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone,
})
}
return results, nil
}
// searchStatementRelatesToQuery is a bounded relevance gate, not a truth
// check. It prevents generic search pages from becoming apparent evidence
// when their content has no substantive relation to the requested query.
func searchStatementRelatesToQuery(query, statement string) bool {
terms := termPattern.FindAllString(strings.ToLower(query), -1)
content := strings.ToLower(statement)
usable := 0
for _, term := range terms {
if _, stop := searchStopTerms[term]; stop || len([]rune(term)) < 4 {
continue
}
usable++
runes := []rune(term)
prefixLen := 5
if len(runes) < prefixLen {
prefixLen = len(runes)
}
if strings.Contains(content, string(runes[:prefixLen])) {
return true
}
}
return usable == 0
}
func (c SearchConnector) allowedHost(host string) bool { _, ok := c.AllowedHosts[host]; return ok }
func (c SearchConnector) limit() int {
if c.MaxResults <= 0 || c.MaxResults > 8 {
return 3
}
return c.MaxResults
}
func cleanHTML(value string) string {
return strings.Join(strings.Fields(html.UnescapeString(tagPattern.ReplaceAllString(value, ""))), " ")
}