diff --git a/adapters/research/http_connector.go b/adapters/research/http_connector.go new file mode 100644 index 0000000..ac6e048 --- /dev/null +++ b/adapters/research/http_connector.go @@ -0,0 +1,74 @@ +package research + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "git.cin.su/m/Assistant/contracts/epistemics" +) + +var ErrHTTPSourceRejected = errors.New("http research source rejected") + +type HTTPConnector struct { + Client *http.Client + AllowedHosts map[string]struct{} + MaxBytes int64 +} + +func (c HTTPConnector) Collect(ctx context.Context, request epistemics.WebResearchRequest, nowUnixMillis int64) (epistemics.ResearchCandidate, error) { + if request.Plan.Validate() != nil || request.SourceID == "" || strings.TrimSpace(request.Statement) == "" || nowUnixMillis < 1 { + return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected + } + u, err := url.Parse(request.URL) + if err != nil || u.Scheme != "https" || u.User != nil || u.Host == "" || !c.allowedHost(strings.ToLower(u.Hostname())) { + return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected + } + client := c.Client + if client == nil { + client = &http.Client{} + } + maxBytes := c.MaxBytes + if maxBytes <= 0 || maxBytes > 4<<20 { + maxBytes = 256 << 10 + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected + } + response, err := client.Do(req) + if err != nil { + return epistemics.ResearchCandidate{}, fmt.Errorf("read web source: %w", err) + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return epistemics.ResearchCandidate{}, fmt.Errorf("web source status: %s", response.Status) + } + body, err := io.ReadAll(io.LimitReader(response.Body, maxBytes+1)) + if err != nil || int64(len(body)) > maxBytes || len(body) == 0 { + return epistemics.ResearchCandidate{}, ErrHTTPSourceRejected + } + digest := sha256.Sum256(body) + digestText := "sha256:" + hex.EncodeToString(digest[:]) + key := sha256.Sum256([]byte(u.String() + "\x00" + digestText)) + keyText := hex.EncodeToString(key[:]) + snapshotID := epistemics.SourceSnapshotID("snapshot:" + keyText) + return epistemics.ResearchCandidate{ + PropositionKey: request.PropositionKey, Statement: request.Statement, + Source: epistemics.Source{ID: request.SourceID, Kind: "web", Locator: u.String(), Title: request.Title, Publisher: u.Hostname()}, + Snapshot: epistemics.SourceSnapshot{ID: snapshotID, SourceID: request.SourceID, ContentDigest: digestText, CapturedAtUnixMillis: nowUnixMillis, Freshness: epistemics.FreshnessCurrent}, + Excerpt: epistemics.SourceExcerpt{ID: epistemics.SourceExcerptID("excerpt:" + keyText), SnapshotID: snapshotID, Text: string(body), Start: 0, End: int64(len(body))}, + Evidence: epistemics.EvidenceObserved, Coverage: epistemics.CoverageComplete, Conflict: epistemics.ConflictNone, + }, nil +} + +func (c HTTPConnector) allowedHost(host string) bool { + _, ok := c.AllowedHosts[host] + return ok +} diff --git a/cmd/research-runtime/main.go b/cmd/research-runtime/main.go index 4ada125..ef5705c 100644 --- a/cmd/research-runtime/main.go +++ b/cmd/research-runtime/main.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "strings" "time" "git.cin.su/m/Assistant/adapters/postgres" @@ -67,6 +68,26 @@ func main() { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(result) }) + webConnector := researchadapter.HTTPConnector{AllowedHosts: allowedHosts(os.Getenv("RESEARCH_HTTP_ALLOWED_HOSTS")), Client: &http.Client{Timeout: 5 * time.Second}} + mux.HandleFunc("POST /v1/research/web/evaluate", func(w http.ResponseWriter, r *http.Request) { + var request epistemics.WebResearchRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 128<<10)).Decode(&request); err != nil { + http.Error(w, "invalid web research request", http.StatusBadRequest) + return + } + candidate, err := webConnector.Collect(r.Context(), request, time.Now().UnixMilli()) + if err != nil { + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + result, err := application.EvaluateAndPersist(r.Context(), epistemics.ResearchEvaluationRequest{Plan: request.Plan, Candidates: []epistemics.ResearchCandidate{candidate}}, repository, time.Now().UnixMilli()) + if err != nil { + http.Error(w, err.Error(), http.StatusUnprocessableEntity) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) + }) addr := os.Getenv("LISTEN_ADDR") if addr == "" { addr = ":8087" @@ -75,6 +96,16 @@ func main() { log.Fatal(http.ListenAndServe(addr, mux)) } +func allowedHosts(value string) map[string]struct{} { + result := map[string]struct{}{} + for _, host := range strings.Split(value, ",") { + if host = strings.ToLower(strings.TrimSpace(host)); host != "" { + result[host] = struct{}{} + } + } + return result +} + func resolveDatabasePassword() (string, error) { if _, ok := os.LookupEnv("EPISTEMICS_POSTGRES_PASSWORD"); ok { return "", fmt.Errorf("EPISTEMICS_POSTGRES_PASSWORD direct environment secret is forbidden") diff --git a/contracts/epistemics/research.go b/contracts/epistemics/research.go index d66da33..e961d8a 100644 --- a/contracts/epistemics/research.go +++ b/contracts/epistemics/research.go @@ -122,6 +122,15 @@ type RepositoryResearchRequest struct { Statement string } +type WebResearchRequest struct { + Plan ResearchPlan + PropositionKey string + SourceID SourceID + URL string + Title string + Statement string +} + // ResolveCitation creates evidence only when all identifiers form one exact // source chain. It deliberately does not fetch, infer or rewrite source text. func ResolveCitation(source Source, snapshot SourceSnapshot, excerpt SourceExcerpt, evidence EvidenceKind, coverage CoverageState, conflict ConflictState, resolvedAtUnixMillis int64) (Citation, error) { diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 40fbadd..2be2d27 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -140,6 +140,7 @@ services: environment: LISTEN_ADDR: :8087 RESEARCH_SOURCE_ROOT: /var/lib/research-sources + RESEARCH_HTTP_ALLOWED_HOSTS: assistant.cin.su,git.cin.su POSTGRES_HOST: postgres POSTGRES_PORT: "5432" POSTGRES_DATABASE: ${POSTGRES_DB:-assistant} diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 3c8e024..99648a9 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { "task_file": "dev/tasks/DEV-000143-research-runtime-orchestration.json", "task_id": "DEV-000143", - "task_revision": 2 + "task_revision": 3 } diff --git a/dev/tasks/DEV-000143-research-runtime-orchestration.json b/dev/tasks/DEV-000143-research-runtime-orchestration.json index 62cb160..ebdb37e 100644 --- a/dev/tasks/DEV-000143-research-runtime-orchestration.json +++ b/dev/tasks/DEV-000143-research-runtime-orchestration.json @@ -1,6 +1,6 @@ { "task_id": "DEV-000143", - "task_revision": 2, + "task_revision": 3, "architecture_revision": "ADR-0018", "base_repo_revision": "f9e2173", "risk_tier": "E", @@ -12,7 +12,7 @@ "contracts": {"consume": ["ResearchEvaluationRequest, ResearchResult, TruthBundle and epistemics repository"], "may_change": ["Private persistence orchestration endpoint"]}, "semantic_owner": "epistemics", "canonical_state": ["epistemics.truth_bundles"], - "state_transitions": ["repository file -> immutable digest/excerpt -> evaluated finding -> persisted provenance"], + "state_transitions": ["repository/web source -> immutable digest/excerpt -> evaluated finding -> persisted provenance"], "side_effects_and_privileges": ["epistemics_role writes through repository only; no source mutation or provider call"], "persistence_and_migrations": ["Consumes migration 034 already in main"], "failure_retry_restart": ["idempotent repository writes; runtime restart-safe"], diff --git a/tests/research_runtime/http_connector_test.go b/tests/research_runtime/http_connector_test.go new file mode 100644 index 0000000..78bf4d8 --- /dev/null +++ b/tests/research_runtime/http_connector_test.go @@ -0,0 +1,20 @@ +package research_runtime_test + +import ( + "context" + "testing" + + researchadapter "git.cin.su/m/Assistant/adapters/research" + "git.cin.su/m/Assistant/contracts/epistemics" +) + +func TestHTTPConnectorRejectsNonAllowlistedOrNonHTTPSSources(t *testing.T) { + connector := researchadapter.HTTPConnector{AllowedHosts: map[string]struct{}{"assistant.cin.su": {}}} + base := epistemics.WebResearchRequest{Plan: epistemics.ResearchPlan{ID: "plan", Question: "q", SourceClasses: []string{"web"}, FreshnessPolicy: "current"}, SourceID: "s", Statement: "statement"} + for _, url := range []string{"http://assistant.cin.su/healthz", "https://127.0.0.1/secret", "https://evil.example/"} { + base.URL = url + if _, err := connector.Collect(context.Background(), base, 10); err != researchadapter.ErrHTTPSourceRejected { + t.Fatalf("url=%s err=%v", url, err) + } + } +}