Maven/internal/websearch/searxng_test.go

package websearch

import (
	"context"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"
)

const sampleJSON = `{
  "query": "почему небо голубое",
  "answers": ["Rayleigh scattering makes the sky blue."],
  "results": [
    {"title": "Рэлеевское рассеяние", "url": "https://ru.wikipedia.org/x", "content": "Рассеяние\n  света  на молекулах.", "engine": "wikipedia"},
    {"title": "", "url": "https://example.org/empty", "content": "", "engine": "duckduckgo"},
    {"title": "Why is the sky blue", "url": "https://example.org/2", "content": "Short answer.", "engine": "duckduckgo"}
  ]
}`

func TestParseResponse(t *testing.T) {
	got, err := ParseResponse([]byte(sampleJSON), 5)
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if len(got.Answers) != 1 || got.Answers[0] != "Rayleigh scattering makes the sky blue." {
		t.Fatalf("answers = %#v", got.Answers)
	}
	// The textless middle hit is dropped: it is a link with nothing to read.
	if len(got.Results) != 2 {
		t.Fatalf("results = %#v", got.Results)
	}
	if got.Results[0].Content != "Рассеяние света на молекулах." {
		t.Fatalf("whitespace not collapsed: %q", got.Results[0].Content)
	}
	if got.Empty() {
		t.Fatal("Empty() on a response with hits")
	}
}

// The limit counts usable hits, not raw ones — a textless entry must not push a
// real snippet out of the reply.
func TestParseResponseLimitSkipsEmpty(t *testing.T) {
	got, err := ParseResponse([]byte(sampleJSON), 2)
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if len(got.Results) != 2 {
		t.Fatalf("results = %d, want 2", len(got.Results))
	}
	if got.Results[1].Title != "Why is the sky blue" {
		t.Fatalf("second hit = %q", got.Results[1].Title)
	}
}

// Newer SearXNG emits answers as objects; older ones as bare strings. Both are
// in the wild and both must read.
func TestParseResponseObjectAnswers(t *testing.T) {
	got, err := ParseResponse([]byte(`{"answers":[{"answer":"42","url":"x"}],"results":[]}`), 5)
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if len(got.Answers) != 1 || got.Answers[0] != "42" {
		t.Fatalf("answers = %#v", got.Answers)
	}
}

func TestResponseEmpty(t *testing.T) {
	got, err := ParseResponse([]byte(`{"answers":[],"results":[]}`), 5)
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if !got.Empty() {
		t.Fatal("Empty() = false on a reply with nothing in it")
	}
}

func TestSnippetsAnswersFirst(t *testing.T) {
	got, _ := ParseResponse([]byte(sampleJSON), 5)
	lines := got.Snippets()
	if len(lines) != 3 {
		t.Fatalf("lines = %#v", lines)
	}
	if lines[0] != "Rayleigh scattering makes the sky blue." {
		t.Fatalf("answer did not lead: %q", lines[0])
	}
	if !strings.Contains(lines[1], " — ") {
		t.Fatalf("result line = %q", lines[1])
	}
	// No URL travels into the evidence: piper reads one out character by
	// character and it is not evidence anyway.
	for _, l := range lines {
		if strings.Contains(l, "http") {
			t.Fatalf("url leaked into evidence: %q", l)
		}
	}
}

// The query goes out verbatim, and the JSON format is always asked for.
func TestSearchRequest(t *testing.T) {
	var gotQuery, gotFormat, gotLang, gotEngines string
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		gotQuery = r.URL.Query().Get("q")
		gotFormat = r.URL.Query().Get("format")
		gotLang = r.URL.Query().Get("language")
		gotEngines = r.URL.Query().Get("engines")
		w.Write([]byte(sampleJSON))
	}))
	defer srv.Close()

	c := New(srv.URL, Options{Language: "ru", Engines: "duckduckgo"})
	got, err := c.Search(context.Background(), "почему небо голубое", 3)
	if err != nil {
		t.Fatalf("search: %v", err)
	}
	if gotQuery != "почему небо голубое" {
		t.Fatalf("query was rewritten: %q", gotQuery)
	}
	if gotFormat != "json" || gotLang != "ru" || gotEngines != "duckduckgo" {
		t.Fatalf("format=%q language=%q engines=%q", gotFormat, gotLang, gotEngines)
	}
	if len(got.Results) != 2 {
		t.Fatalf("results = %#v", got.Results)
	}
}

// A stock SearXNG answers 403 to format=json. The error must say so, because
// that is the one misconfiguration this client cannot work around.
func TestSearchHTTPError(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "forbidden", http.StatusForbidden)
	}))
	defer srv.Close()

	_, err := New(srv.URL, Options{}).Search(context.Background(), "x", 3)
	if err == nil || !strings.Contains(err.Error(), "403") {
		t.Fatalf("err = %v", err)
	}
}

func TestSearchEmptyQuery(t *testing.T) {
	if _, err := New("http://example.invalid", Options{}).Search(context.Background(), "  ", 3); err == nil {
		t.Fatal("empty query accepted")
	}
}

// A host that never answers a SYN must not hold the turn open for the whole
// request budget: the ZIM behind this search is the answer, and he waits
// through every second of the delay (V-508). 192.0.2.1 is TEST-NET-1, which is
// reserved for documentation and routed nowhere.
func TestSearchGivesUpOnAnUnreachableHostFast(t *testing.T) {
	c := New("http://192.0.2.1:9563", Options{Timeout: 8 * time.Second})
	start := time.Now()
	_, err := c.Search(context.Background(), "фотосинтез", 4)
	elapsed := time.Since(start)
	if err == nil {
		t.Fatal("Search reached 192.0.2.1; the address is routed nowhere")
	}
	if elapsed > 4*time.Second {
		t.Errorf("Search took %v to give up; the dial cap is %v", elapsed, dialTimeout)
	}
}

// The dial cap must not shorten a request to an instance that did connect. A
// slow SearXNG is fanning out to real engines, which is worth waiting for.
func TestSlowButReachableInstanceKeepsTheFullBudget(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(2 * dialTimeout)
		w.Header().Set("Content-Type", "application/json")
		w.Write([]byte(`{"answers":[],"results":[{"title":"Фотосинтез","url":"http://x","content":"процесс","engine":"test"}]}`))
	}))
	defer srv.Close()
	c := New(srv.URL, Options{Timeout: 8 * time.Second})
	resp, err := c.Search(context.Background(), "фотосинтез", 4)
	if err != nil {
		t.Fatalf("Search: %v", err)
	}
	if len(resp.Results) != 1 {
		t.Errorf("results = %d, want 1", len(resp.Results))
	}
}