djk/internal/app/network_browser_test.go

package app

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	fhttp "github.com/bogdanfinn/fhttp"
)

type browserFixtureClient struct {
	do     func(*fhttp.Request) (*fhttp.Response, error)
	closed atomic.Int32
}

func (client *browserFixtureClient) Do(request *fhttp.Request) (*fhttp.Response, error) {
	return client.do(request)
}
func (client *browserFixtureClient) CloseIdleConnections() { client.closed.Add(1) }

func TestHuangguoBrowserHTTP2CookiesAndNativeIsolation(t *testing.T) {
	var requests atomic.Int32
	server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		count := requests.Add(1)
		if r.ProtoMajor != 2 || !strings.Contains(r.UserAgent(), "Chrome/150.0.0.0") || r.Header.Get("Sec-Ch-Ua-Platform") != `"macOS"` {
			t.Errorf("browser fingerprint headers or HTTP/2 missing: %s %s", r.Proto, r.UserAgent())
		}
		_, err := r.Cookie("fixture_session")
		if (count == 2) != (err == nil) {
			t.Errorf("cookie session reuse/reset failed at request %d", count)
		}
		http.SetCookie(w, &http.Cookie{Name: "fixture_session", Value: "local-test-only", Path: "/", Secure: true})
		io.WriteString(w, "fixture browser page")
	}))
	server.EnableHTTP2 = true
	server.StartTLS()
	defer server.Close()
	cfg := defaultConfig()
	cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), "direct", 1
	cfg.HuangguoVideoURL, cfg.InsecureTLS = server.URL, true
	downloader := NewDownloader(cfg)
	defer downloader.client.CloseIdleConnections()
	downloader.limiter = newRequestLimiter(2, time.Nanosecond)
	transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
	var nativeCalls atomic.Int32
	transport.base = rankingTransport(func(r *http.Request) (*http.Response, error) {
		nativeCalls.Add(1)
		return rankingHTTPResponse(r, 200, "native fixture"), nil
	})
	for index := 0; index < 3; index++ {
		if index == 2 {
			downloader.client.CloseIdleConnections()
		}
		body, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "https://huangguo.video/")
		if err != nil || body != "fixture browser page" {
			t.Fatal(body, err)
		}
	}
	if nativeCalls.Load() != 0 {
		t.Fatal("browser request first used the native transport")
	}
	request, _ := http.NewRequest(http.MethodGet, "https://hongguoduanju.com/", nil)
	response, err := downloader.client.Do(request)
	if err != nil {
		t.Fatal(err)
	}
	response.Body.Close()
	if nativeCalls.Load() != 1 {
		t.Fatal("unrelated source did not keep native transport")
	}
}

func TestHuangguoBrowserRespectsCertificatesAndCancellation(t *testing.T) {
	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "fixture") }))
	defer server.Close()
	cfg := defaultConfig()
	cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), "direct", 1
	cfg.HuangguoVideoURL = server.URL
	downloader := NewDownloader(cfg)
	defer downloader.client.CloseIdleConnections()
	if _, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", ""); err == nil {
		t.Fatal("untrusted certificate was accepted by default")
	}
	blocking := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
		w.(http.Flusher).Flush()
		<-r.Context().Done()
	}))
	defer blocking.Close()
	cfg.HuangguoVideoURL = blocking.URL
	downloader = NewDownloader(cfg)
	defer downloader.client.CloseIdleConnections()
	ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
	defer cancel()
	if _, err := downloader.fetchProviderText(ctx, "https://huangguo.video/videos", ""); err == nil {
		t.Fatal("canceled browser body read succeeded")
	}
	downloader.limiter.mu.Lock()
	active := downloader.limiter.active
	downloader.limiter.mu.Unlock()
	if active != 0 {
		t.Fatal("browser timeout leaked a limiter permit")
	}
}

func TestHuangguoBrowserProxyResetAndBackoff(t *testing.T) {
	cfg := defaultConfig()
	cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), "http://proxy-one.invalid:8080", 3
	downloader := NewDownloader(cfg)
	defer downloader.client.CloseIdleConnections()
	downloader.limiter = newRequestLimiter(2, time.Nanosecond)
	transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
	var routes []string
	var clients []*browserFixtureClient
	calls := 0
	transport.newClient = func(proxy string) (browserHTTPClient, error) {
		routes = append(routes, proxy)
		client := &browserFixtureClient{do: func(r *fhttp.Request) (*fhttp.Response, error) {
			calls++
			status, body := 403, "Cloudflare: Sorry, you have been blocked"
			if len(routes) > 1 {
				status, body = 200, "fixture success"
			}
			return &fhttp.Response{StatusCode: status, Header: fhttp.Header{"Cf-Ray": {"fixture-ray"}}, Body: io.NopCloser(strings.NewReader(body))}, nil
		}}
		clients = append(clients, client)
		return client, nil
	}
	for index := 0; index < 2; index++ {
		_, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
		if err == nil || !strings.Contains(err.Error(), "Cloudflare 拒绝了当前请求") {
			t.Fatal("missing accurate block cause", err)
		}
	}
	if calls != 1 || len(routes) != 1 {
		t.Fatal("backoff did not prevent repeated requests", calls, len(routes))
	}
	downloader.proxyRouter.configure("socks5://proxy-two.invalid:1080")
	downloader.client.CloseIdleConnections()
	if clients[0].closed.Load() == 0 {
		t.Fatal("old proxy session not closed")
	}
	_, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
	if err == nil || calls != 1 {
		t.Fatal("proxy switch bypassed the existing backoff")
	}
	downloader.limiter.mu.Lock()
	delete(downloader.limiter.backoffs, "huangguo.video")
	downloader.limiter.mu.Unlock()
	body, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
	if err != nil || body != "fixture success" || len(routes) != 2 || routes[1] != "socks5://proxy-two.invalid:1080" {
		t.Fatal("updated proxy was not used", routes, err)
	}
}

func TestHuangguoBrowserRedirectUsesNativePolicy(t *testing.T) {
	cfg := defaultConfig()
	cfg.dataDir, cfg.OutputDir, cfg.ProxyURL = t.TempDir(), t.TempDir(), "direct"
	downloader := NewDownloader(cfg)
	defer downloader.client.CloseIdleConnections()
	transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
	transport.newClient = func(string) (browserHTTPClient, error) {
		return &browserFixtureClient{do: func(r *fhttp.Request) (*fhttp.Response, error) {
			return &fhttp.Response{StatusCode: 302, Header: fhttp.Header{"Location": {"https://redirect.invalid/fixture"}, "Set-Cookie": {"fixture_session=private; Path=/"}}, Body: io.NopCloser(strings.NewReader(""))}, nil
		}}, nil
	}
	transport.base = rankingTransport(func(r *http.Request) (*http.Response, error) {
		if r.URL.Host != "redirect.invalid" || r.Header.Get("Cookie") != "" || r.Header.Get("Sec-Ch-Ua") != "" {
			t.Error("redirect leaked fingerprint session headers")
		}
		return rankingHTTPResponse(r, 200, "redirected fixture"), nil
	})
	body, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
	if err != nil || body != "redirected fixture" {
		t.Fatal(body, err)
	}
}

func TestHuangguoBrowserLivePages(t *testing.T) {
	if os.Getenv("JUKU_HUANGGUO_BROWSER_PROBE") != "1" {
		t.Skip("text-only live probe is opt-in")
	}
	cfg := defaultConfig()
	cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), firstNonEmpty(os.Getenv("JUKU_HUANGGUO_BROWSER_PROXY"), "direct"), 1
	cfg.RequestIntervalMS = 2000
	downloader := NewDownloader(cfg)
	defer downloader.client.CloseIdleConnections()
	var events []diagnosticEvent
	transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
	transport.record = func(event diagnosticEvent) { events = append(events, event) }
	ctx, cancel := context.WithTimeout(context.Background(), 70*time.Second)
	defer cancel()
	urls := []string{"https://huangguo.video/videos", "https://huangguo.video/videos?category=1"}
	counts := []int{}
	detailURL := ""
	for _, address := range urls {
		body, err := downloader.fetchProviderText(ctx, address, "https://huangguo.video/")
		if err != nil {
			t.Fatal(err)
		}
		dramas := parseHuangguoVideoCards(body, address)
		if len(dramas) == 0 {
			t.Fatal("200 page contained no business records")
		}
		counts = append(counts, len(dramas))
		if detailURL == "" {
			for _, drama := range dramas {
				_, id, ok := splitProviderDramaID(drama.ID)
				if ok && strings.HasPrefix(id, "series/") {
					detailURL = "https://huangguo.video/" + id
					break
				}
			}
		}
	}
	if detailURL == "" {
		t.Fatal("missing linked detail")
	}
	body, err := downloader.fetchProviderText(ctx, detailURL, "https://huangguo.video/")
	if err != nil {
		t.Fatal(err)
	}
	episodes := parseHuangguoVideoEpisodes(body, detailURL)
	if len(episodes) == 0 {
		t.Fatal("detail contained no episode links")
	}
	counts = append(counts, len(episodes))
	evidence := map[string]any{"time": time.Now().UTC(), "client": "tls-client v1.16.0 / Chrome 150", "counts": counts, "requests": events, "route": map[bool]string{true: "direct", false: "configured-proxy"}[cfg.ProxyURL == "direct"]}
	if output := os.Getenv("JUKU_HUANGGUO_BROWSER_EVIDENCE"); output != "" {
		data, _ := json.MarshalIndent(evidence, "", "  ")
		if err := os.WriteFile(filepath.Clean(output), data, 0600); err != nil {
			t.Fatal(err)
		}
	}
	for _, event := range events {
		if event.HTTPStatus != 200 || event.Protocol != "HTTP/2.0" {
			t.Fatal("unexpected live response", event.HTTPStatus, event.Protocol)
		}
	}
	t.Logf("text-only list/category/detail counts: %v; responses=%d", counts, len(events))
}