277 lines
9.8 KiB
Go
277 lines
9.8 KiB
Go
package research
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"errors"
|
|
"html"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.cin.su/m/Assistant/contracts/epistemics"
|
|
)
|
|
|
|
var ErrSearchRejected = errors.New("web search request rejected")
|
|
|
|
var (
|
|
searchLinkPattern = regexp.MustCompile(`(?is)(<a[^>]+class=["'][^"']*result__a[^"']*["'][^>]*>)(.*?)</a>`)
|
|
searchSnippetPattern = regexp.MustCompile(`(?is)<a[^>]+class=["'][^"']*result__snippet[^"']*["'][^>]*>(.*?)</a>`)
|
|
bingLinkPattern = regexp.MustCompile(`(?is)<li[^>]+class=["'][^"']*b_algo[^"']*["'][^>]*>.*?<h2[^>]*>(<a[^>]*>)(.*?)</a>`)
|
|
bingSnippetPattern = regexp.MustCompile(`(?is)<li[^>]+class=["'][^"']*b_algo[^"']*["'][^>]*>.*?<p[^>]*>(.*?)</p>`)
|
|
tagPattern = regexp.MustCompile(`(?s)<[^>]*>`)
|
|
termPattern = regexp.MustCompile(`(?i)[\p{L}\p{N}]{2,}`)
|
|
)
|
|
|
|
var searchStopTerms = map[string]struct{}{
|
|
"какая": {}, "какое": {}, "какие": {}, "какой": {}, "что": {}, "как": {}, "где": {}, "когда": {}, "почему": {}, "мне": {}, "пожалуйста": {},
|
|
"what": {}, "which": {}, "where": {}, "when": {}, "why": {}, "how": {}, "please": {}, "the": {}, "this": {},
|
|
}
|
|
|
|
// SearchConnector uses a configured HTTPS search endpoint. It does not know
|
|
// cities, weather terms, entities or model names: the complete semantic query
|
|
// comes from the interpretation layer.
|
|
type SearchConnector struct {
|
|
Client *http.Client
|
|
EndpointTemplate string
|
|
AllowedHosts map[string]struct{}
|
|
FallbackEndpoints []SearchEndpoint
|
|
MaxBytes int64
|
|
MaxResults int
|
|
}
|
|
|
|
// SearchEndpoint is an independently allowlisted web-search route. The
|
|
// connector tries routes in order and stops at the first relevant evidence.
|
|
// This is a source-recovery policy, not a question-type classifier.
|
|
type SearchEndpoint struct {
|
|
Template string
|
|
AllowedHosts map[string]struct{}
|
|
}
|
|
|
|
func (c SearchConnector) Collect(ctx context.Context, request epistemics.WebSearchRequest, nowUnixMillis int64) ([]epistemics.ResearchCandidate, error) {
|
|
if request.Plan.Validate() != nil || strings.TrimSpace(request.Query) == "" || nowUnixMillis < 1 || !strings.Contains(c.EndpointTemplate, "%s") {
|
|
return nil, ErrSearchRejected
|
|
}
|
|
endpoints := make([]SearchEndpoint, 0, 1+len(c.FallbackEndpoints))
|
|
endpoints = append(endpoints, SearchEndpoint{Template: c.EndpointTemplate, AllowedHosts: c.AllowedHosts})
|
|
endpoints = append(endpoints, c.FallbackEndpoints...)
|
|
var lastErr error
|
|
for _, candidateEndpoint := range endpoints {
|
|
results, err := c.collectEndpoint(ctx, candidateEndpoint, request, nowUnixMillis)
|
|
if err != nil {
|
|
lastErr = err
|
|
continue
|
|
}
|
|
if len(results) > 0 {
|
|
return results, nil
|
|
}
|
|
}
|
|
if lastErr != nil {
|
|
return nil, lastErr
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (c SearchConnector) collectEndpoint(ctx context.Context, configured SearchEndpoint, request epistemics.WebSearchRequest, nowUnixMillis int64) ([]epistemics.ResearchCandidate, error) {
|
|
if !strings.Contains(configured.Template, "%s") {
|
|
return nil, ErrSearchRejected
|
|
}
|
|
searchQuery := enrichedSearchQuery(request.Query)
|
|
endpoint, err := url.Parse(strings.Replace(configured.Template, "%s", url.QueryEscape(searchQuery), 1))
|
|
if err != nil || endpoint.Scheme != "https" || endpoint.Host == "" || !allowedHost(configured.AllowedHosts, strings.ToLower(endpoint.Hostname())) {
|
|
return nil, ErrSearchRejected
|
|
}
|
|
client := c.Client
|
|
if client == nil {
|
|
client = &http.Client{}
|
|
}
|
|
maxBytes := c.MaxBytes
|
|
if maxBytes <= 0 || maxBytes > 4<<20 {
|
|
maxBytes = 512 << 10
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
|
|
if err != nil {
|
|
return nil, ErrSearchRejected
|
|
}
|
|
// Search providers commonly return a consent/challenge shell to the
|
|
// default Go client. A neutral browser identification improves the same
|
|
// generic evidence path for every query without selecting by topic.
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; DomovoyResearch/1.0)")
|
|
response, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return nil, ErrSearchRejected
|
|
}
|
|
body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1))
|
|
if err != nil || len(body) == 0 || int64(len(body)) > maxBytes {
|
|
return nil, ErrSearchRejected
|
|
}
|
|
links := searchLinkPattern.FindAllSubmatch(body, c.limit())
|
|
snippets := searchSnippetPattern.FindAllSubmatch(body, c.limit())
|
|
if len(links) == 0 {
|
|
links = bingLinkPattern.FindAllSubmatch(body, c.limit())
|
|
snippets = bingSnippetPattern.FindAllSubmatch(body, c.limit())
|
|
}
|
|
if len(links) == 0 {
|
|
return nil, nil
|
|
}
|
|
digest := sha256.Sum256(body)
|
|
digestText := "sha256:" + hex.EncodeToString(digest[:])
|
|
base := hex.EncodeToString(digest[:])
|
|
sourceID := epistemics.SourceID("source:search:" + base)
|
|
snapshotID := epistemics.SourceSnapshotID("snapshot:search:" + base)
|
|
results := make([]epistemics.ResearchCandidate, 0, len(links))
|
|
for index, link := range links {
|
|
linkURL := resolveSearchResultLocator(extractHref(string(link[1])), endpoint)
|
|
title := cleanHTML(string(link[2]))
|
|
snippet := ""
|
|
if index < len(snippets) {
|
|
snippet = cleanHTML(string(snippets[index][1]))
|
|
}
|
|
statement := strings.TrimSpace(strings.Trim(title+" — "+snippet, " —"))
|
|
if statement == "" || !searchStatementRelatesToQuery(request.Query, statement) {
|
|
continue
|
|
}
|
|
ordinal := strconv.Itoa(index)
|
|
excerptID := epistemics.SourceExcerptID("excerpt:search:" + base + ":" + ordinal)
|
|
results = append(results, epistemics.ResearchCandidate{
|
|
PropositionKey: request.PropositionKey + ":result:" + ordinal,
|
|
Statement: statement,
|
|
Source: epistemics.Source{ID: sourceID, Kind: "web-search", Locator: linkURL, Title: title, Publisher: sourcePublisher(linkURL, endpoint.Hostname())},
|
|
Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: sourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent},
|
|
Excerpt: epistemics.SourceExcerpt{ID: excerptID, SnapshotID: snapshotID, Text: statement, Start: 0, End: int64(len(statement))},
|
|
Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone,
|
|
})
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
// searchStatementRelatesToQuery is a bounded relevance gate, not a truth
|
|
// check. It prevents generic search pages from becoming apparent evidence
|
|
// when their content has no substantive relation to the requested query.
|
|
func searchStatementRelatesToQuery(query, statement string) bool {
|
|
terms := termPattern.FindAllString(strings.ToLower(query), -1)
|
|
content := strings.ToLower(statement)
|
|
productCue := containsSearchTerm(terms, "видеокарт", "gpu", "процессор", "телефон", "ноутбук", "монитор")
|
|
if productCue && hasNumericSearchTerm(terms) {
|
|
contentTerms := termPattern.FindAllString(content, -1)
|
|
return hasNumericSearchTerm(contentTerms) && containsSearchTerm(contentTerms, "видеокарт", "gpu", "процессор", "телефон", "ноутбук", "монитор")
|
|
}
|
|
usable := 0
|
|
for _, term := range terms {
|
|
if _, stop := searchStopTerms[term]; stop || len([]rune(term)) < 4 {
|
|
continue
|
|
}
|
|
usable++
|
|
runes := []rune(term)
|
|
prefixLen := 5
|
|
if len(runes) < prefixLen {
|
|
prefixLen = len(runes)
|
|
}
|
|
if strings.Contains(content, string(runes[:prefixLen])) {
|
|
return true
|
|
}
|
|
}
|
|
return usable == 0
|
|
}
|
|
|
|
func containsSearchTerm(values []string, wanted ...string) bool {
|
|
for _, value := range values {
|
|
for _, candidate := range wanted {
|
|
if strings.Contains(value, candidate) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func hasNumericSearchTerm(values []string) bool {
|
|
for _, value := range values {
|
|
for _, runeValue := range value {
|
|
if runeValue >= '0' && runeValue <= '9' {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func enrichedSearchQuery(query string) string {
|
|
trimmed := strings.TrimSpace(query)
|
|
lower := strings.ToLower(trimmed)
|
|
if (strings.Contains(lower, "цен") || strings.Contains(lower, "стоим")) && !strings.Contains(lower, "куп") {
|
|
return trimmed + " цена купить сегодня"
|
|
}
|
|
return trimmed
|
|
}
|
|
|
|
func allowedHost(hosts map[string]struct{}, host string) bool { _, ok := hosts[host]; return ok }
|
|
func (c SearchConnector) limit() int {
|
|
if c.MaxResults <= 0 || c.MaxResults > 8 {
|
|
return 3
|
|
}
|
|
return c.MaxResults
|
|
}
|
|
func cleanHTML(value string) string {
|
|
return strings.Join(strings.Fields(html.UnescapeString(tagPattern.ReplaceAllString(value, ""))), " ")
|
|
}
|
|
|
|
func extractHref(tag string) string {
|
|
match := regexp.MustCompile(`(?is)\bhref=["']([^"']+)["']`).FindStringSubmatch(tag)
|
|
if len(match) == 2 {
|
|
return match[1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func resolveSearchResultLocator(raw string, searchEndpoint *url.URL) string {
|
|
value := strings.TrimSpace(html.UnescapeString(raw))
|
|
if value == "" {
|
|
return searchEndpoint.String()
|
|
}
|
|
parsed, err := url.Parse(value)
|
|
if err != nil {
|
|
return searchEndpoint.String()
|
|
}
|
|
if target := parsed.Query().Get("uddg"); target != "" {
|
|
value = target
|
|
parsed, err = url.Parse(target)
|
|
if err != nil {
|
|
return searchEndpoint.String()
|
|
}
|
|
}
|
|
if target := parsed.Query().Get("u"); strings.HasPrefix(target, "a1") {
|
|
if decoded, decodeErr := base64.RawURLEncoding.DecodeString(strings.TrimPrefix(target, "a1")); decodeErr == nil {
|
|
if targetURL, parseErr := url.Parse(string(decoded)); parseErr == nil {
|
|
parsed = targetURL
|
|
}
|
|
}
|
|
}
|
|
if parsed.Scheme == "" {
|
|
parsed.Scheme = "https"
|
|
}
|
|
if parsed.Host == "" && strings.HasPrefix(value, "//") {
|
|
parsed, err = url.Parse("https:" + value)
|
|
}
|
|
if err != nil || parsed.Scheme != "https" || parsed.Host == "" {
|
|
return searchEndpoint.String()
|
|
}
|
|
return parsed.String()
|
|
}
|
|
|
|
func sourcePublisher(locator, fallback string) string {
|
|
if parsed, err := url.Parse(locator); err == nil && parsed.Hostname() != "" {
|
|
return parsed.Hostname()
|
|
}
|
|
return fallback
|
|
}
|