feat: route factual questions through bounded research
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/epistemics"
|
||||
)
|
||||
|
||||
var ErrSearchRejected = errors.New("web search request rejected")
|
||||
|
||||
var (
|
||||
searchLinkPattern = regexp.MustCompile(`(?is)<a[^>]+class=["'][^"']*result__a[^"']*["'][^>]*>(.*?)</a>`)
|
||||
searchSnippetPattern = regexp.MustCompile(`(?is)<a[^>]+class=["'][^"']*result__snippet[^"']*["'][^>]*>(.*?)</a>`)
|
||||
tagPattern = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||
)
|
||||
|
||||
// SearchConnector uses a configured HTTPS search endpoint. It does not know
|
||||
// cities, weather terms, entities or model names: the complete semantic query
|
||||
// comes from the interpretation layer.
|
||||
type SearchConnector struct {
|
||||
Client *http.Client
|
||||
EndpointTemplate string
|
||||
AllowedHosts map[string]struct{}
|
||||
MaxBytes int64
|
||||
MaxResults int
|
||||
}
|
||||
|
||||
func (c SearchConnector) Collect(ctx context.Context, request epistemics.WebSearchRequest, nowUnixMillis int64) ([]epistemics.ResearchCandidate, error) {
|
||||
if request.Plan.Validate() != nil || strings.TrimSpace(request.Query) == "" || nowUnixMillis < 1 || !strings.Contains(c.EndpointTemplate, "%s") {
|
||||
return nil, ErrSearchRejected
|
||||
}
|
||||
endpoint, err := url.Parse(strings.Replace(c.EndpointTemplate, "%s", url.QueryEscape(strings.TrimSpace(request.Query)), 1))
|
||||
if err != nil || endpoint.Scheme != "https" || endpoint.Host == "" || !c.allowedHost(strings.ToLower(endpoint.Hostname())) {
|
||||
return nil, ErrSearchRejected
|
||||
}
|
||||
client := c.Client
|
||||
if client == nil {
|
||||
client = &http.Client{}
|
||||
}
|
||||
maxBytes := c.MaxBytes
|
||||
if maxBytes <= 0 || maxBytes > 4<<20 {
|
||||
maxBytes = 512 << 10
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
|
||||
if err != nil {
|
||||
return nil, ErrSearchRejected
|
||||
}
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return nil, ErrSearchRejected
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1))
|
||||
if err != nil || len(body) == 0 || int64(len(body)) > maxBytes {
|
||||
return nil, ErrSearchRejected
|
||||
}
|
||||
links := searchLinkPattern.FindAllSubmatch(body, c.limit())
|
||||
snippets := searchSnippetPattern.FindAllSubmatch(body, c.limit())
|
||||
if len(links) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
digest := sha256.Sum256(body)
|
||||
digestText := "sha256:" + hex.EncodeToString(digest[:])
|
||||
base := hex.EncodeToString(digest[:])
|
||||
sourceID := epistemics.SourceID("source:search:" + base)
|
||||
snapshotID := epistemics.SourceSnapshotID("snapshot:search:" + base)
|
||||
results := make([]epistemics.ResearchCandidate, 0, len(links))
|
||||
for index, link := range links {
|
||||
title := cleanHTML(string(link[1]))
|
||||
snippet := ""
|
||||
if index < len(snippets) {
|
||||
snippet = cleanHTML(string(snippets[index][1]))
|
||||
}
|
||||
statement := strings.TrimSpace(strings.Trim(title+" — "+snippet, " —"))
|
||||
if statement == "" {
|
||||
continue
|
||||
}
|
||||
ordinal := strconv.Itoa(index)
|
||||
excerptID := epistemics.SourceExcerptID("excerpt:search:" + base + ":" + ordinal)
|
||||
results = append(results, epistemics.ResearchCandidate{
|
||||
PropositionKey: request.PropositionKey + ":result:" + ordinal,
|
||||
Statement: statement,
|
||||
Source: epistemics.Source{ID: sourceID, Kind: "web-search", Locator: endpoint.String(), Title: title, Publisher: endpoint.Hostname()},
|
||||
Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: sourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent},
|
||||
Excerpt: epistemics.SourceExcerpt{ID: excerptID, SnapshotID: snapshotID, Text: statement, Start: 0, End: int64(len(statement))},
|
||||
Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone,
|
||||
})
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c SearchConnector) allowedHost(host string) bool { _, ok := c.AllowedHosts[host]; return ok }
|
||||
func (c SearchConnector) limit() int {
|
||||
if c.MaxResults <= 0 || c.MaxResults > 8 {
|
||||
return 3
|
||||
}
|
||||
return c.MaxResults
|
||||
}
|
||||
func cleanHTML(value string) string {
|
||||
return strings.Join(strings.Fields(html.UnescapeString(tagPattern.ReplaceAllString(value, ""))), " ")
|
||||
}
|
||||
Reference in New Issue
Block a user