diff --git a/adapters/research/search_connector.go b/adapters/research/search_connector.go index a3dc1d4..422cb3b 100644 --- a/adapters/research/search_connector.go +++ b/adapters/research/search_connector.go @@ -21,6 +21,8 @@ var ErrSearchRejected = errors.New("web search request rejected") var ( searchLinkPattern = regexp.MustCompile(`(?is)]+class=["'][^"']*result__a[^"']*["'][^>]*>(.*?)`) searchSnippetPattern = regexp.MustCompile(`(?is)]+class=["'][^"']*result__snippet[^"']*["'][^>]*>(.*?)`) + bingLinkPattern = regexp.MustCompile(`(?is)]+class=["'][^"']*b_algo[^"']*["'][^>]*>.*?

]*>(.*?)`) + bingSnippetPattern = regexp.MustCompile(`(?is)]+class=["'][^"']*b_algo[^"']*["'][^>]*>.*?]*>(.*?)

`) tagPattern = regexp.MustCompile(`(?s)<[^>]*>`) ) @@ -69,6 +71,10 @@ func (c SearchConnector) Collect(ctx context.Context, request epistemics.WebSear } links := searchLinkPattern.FindAllSubmatch(body, c.limit()) snippets := searchSnippetPattern.FindAllSubmatch(body, c.limit()) + if len(links) == 0 { + links = bingLinkPattern.FindAllSubmatch(body, c.limit()) + snippets = bingSnippetPattern.FindAllSubmatch(body, c.limit()) + } if len(links) == 0 { return nil, nil } diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 84e0169..9b1bfcc 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -195,8 +195,8 @@ services: LISTEN_ADDR: :8087 RESEARCH_SOURCE_ROOT: /var/lib/research-sources RESEARCH_HTTP_ALLOWED_HOSTS: assistant.cin.su,git.cin.su - RESEARCH_SEARCH_URL_TEMPLATE: ${RESEARCH_SEARCH_URL_TEMPLATE:-https://html.duckduckgo.com/html/?q=%s} - RESEARCH_SEARCH_ALLOWED_HOSTS: ${RESEARCH_SEARCH_ALLOWED_HOSTS:-html.duckduckgo.com} + RESEARCH_SEARCH_URL_TEMPLATE: ${RESEARCH_SEARCH_URL_TEMPLATE:-https://www.bing.com/search?q=%s} + RESEARCH_SEARCH_ALLOWED_HOSTS: ${RESEARCH_SEARCH_ALLOWED_HOSTS:-www.bing.com} POSTGRES_HOST: postgres POSTGRES_PORT: "5432" POSTGRES_DATABASE: ${POSTGRES_DB:-assistant} diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 17f3ba5..ab3c5e4 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000275-semantic-web-research-routing.json", - "task_id": "DEV-000275", + "task_file": "dev/tasks/DEV-000276-research-search-provider-fallback.json", + "task_id": "DEV-000276", "task_revision": 1 } diff --git a/dev/tasks/DEV-000276-research-search-provider-fallback.json b/dev/tasks/DEV-000276-research-search-provider-fallback.json new file mode 100644 index 0000000..a446e7f --- /dev/null +++ b/dev/tasks/DEV-000276-research-search-provider-fallback.json @@ -0,0 +1,23 @@ +{ + "task_id": "DEV-000276", + "task_revision": 1, + "architecture_revision": "ADR-0018", + "base_repo_revision": "c09be4c", + "risk_tier": "E", + "intent": "Replace the blocked default search provider with an available configured HTML search endpoint while preserving bounded parsing and truth-first failure behavior.", + "requirement_ids": ["research provider fallback", "no fabricated evidence", "optional external channel"], + "allowed_change_scope": {"modules": ["research-runtime", "deployment"], "paths": ["adapters/research/search_connector.go", "deploy/docker-compose.prod.yml", "tests/research_runtime/search_connector_test.go", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000276-research-search-provider-fallback.json", "docs/PROJECT_STATUS.md", "docs/evidence/**"]}, + "forbidden_change_scope": {"paths": ["architecture/**", "security/**", "database/**", "contracts/**", "LLM-local/**"]}, + "allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 1, "new_third_party_dependencies": 0, "new_database_owner_roles": 0}, + "contracts": {"consume": ["existing WebSearchRequest and evidence chain"], "may_change": ["bounded provider response parsing"]}, + "semantic_owner": "research runtime owns source collection and evidence preservation", + "canonical_state": ["existing epistemics source snapshot and citation chain"], + "state_transitions": ["configured provider response -> bounded excerpts", "provider challenge/no result -> unknown evidence"], + "side_effects_and_privileges": ["read-only HTTPS fetch to operator allowlist"], + "persistence_and_migrations": ["none"], + "failure_retry_restart": ["provider failure never becomes a supported finding"], + "assumptions": ["operator may override search URL and host allowlist", "search results are source excerpts, not generated answers"], + "acceptance_scenarios": ["Bing-style bounded result is collected", "unallowlisted endpoint is rejected", "provider challenge yields no evidence", "no entity-specific query logic exists"], + "required_tests": ["go test ./...", "go vet ./...", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "CI", "production research-runtime smoke"], + "terminal_gap_states": ["BLOCKED_BY_MISSING_CAPABILITY"] +} diff --git a/tests/research_runtime/search_connector_test.go b/tests/research_runtime/search_connector_test.go index 08da9ad..8596fbc 100644 --- a/tests/research_runtime/search_connector_test.go +++ b/tests/research_runtime/search_connector_test.go @@ -39,3 +39,16 @@ func TestSearchConnectorRejectsUnallowlistedEndpoint(t *testing.T) { t.Fatalf("expected allowlist rejection, got %v", err) } } + +func TestSearchConnectorParsesConfiguredBingStyleResults(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`
  • Source title

    Current source excerpt

  • `)) + })) + defer server.Close() + request := epistemics.WebSearchRequest{Plan: epistemics.ResearchPlan{ID: "plan", Question: "q", SourceClasses: []string{"web-search"}, FreshnessPolicy: "current"}, PropositionKey: "p", Query: "q"} + connector := researchadapter.SearchConnector{EndpointTemplate: server.URL + "?q=%s", AllowedHosts: map[string]struct{}{"127.0.0.1": {}}, Client: server.Client(), MaxResults: 3} + candidates, err := connector.Collect(context.Background(), request, 10) + if err != nil || len(candidates) != 1 || candidates[0].Statement != "Source title — Current source excerpt" { + t.Fatalf("unexpected Bing-style evidence: candidates=%+v err=%v", candidates, err) + } +}