初始提交：短剧库源码（含转码优化 ultrafast）
This commit is contained in:
yvan
@@ -0,0 +1,249 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
fhttp "github.com/bogdanfinn/fhttp"
|
||||
)
|
||||
|
||||
type browserFixtureClient struct {
|
||||
do func(*fhttp.Request) (*fhttp.Response, error)
|
||||
closed atomic.Int32
|
||||
}
|
||||
|
||||
func (client *browserFixtureClient) Do(request *fhttp.Request) (*fhttp.Response, error) {
|
||||
return client.do(request)
|
||||
}
|
||||
func (client *browserFixtureClient) CloseIdleConnections() { client.closed.Add(1) }
|
||||
|
||||
func TestHuangguoBrowserHTTP2CookiesAndNativeIsolation(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count := requests.Add(1)
|
||||
if r.ProtoMajor != 2 || !strings.Contains(r.UserAgent(), "Chrome/150.0.0.0") || r.Header.Get("Sec-Ch-Ua-Platform") != `"macOS"` {
|
||||
t.Errorf("browser fingerprint headers or HTTP/2 missing: %s %s", r.Proto, r.UserAgent())
|
||||
}
|
||||
_, err := r.Cookie("fixture_session")
|
||||
if (count == 2) != (err == nil) {
|
||||
t.Errorf("cookie session reuse/reset failed at request %d", count)
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{Name: "fixture_session", Value: "local-test-only", Path: "/", Secure: true})
|
||||
io.WriteString(w, "fixture browser page")
|
||||
}))
|
||||
server.EnableHTTP2 = true
|
||||
server.StartTLS()
|
||||
defer server.Close()
|
||||
cfg := defaultConfig()
|
||||
cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), "direct", 1
|
||||
cfg.HuangguoVideoURL, cfg.InsecureTLS = server.URL, true
|
||||
downloader := NewDownloader(cfg)
|
||||
defer downloader.client.CloseIdleConnections()
|
||||
downloader.limiter = newRequestLimiter(2, time.Nanosecond)
|
||||
transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
|
||||
var nativeCalls atomic.Int32
|
||||
transport.base = rankingTransport(func(r *http.Request) (*http.Response, error) {
|
||||
nativeCalls.Add(1)
|
||||
return rankingHTTPResponse(r, 200, "native fixture"), nil
|
||||
})
|
||||
for index := 0; index < 3; index++ {
|
||||
if index == 2 {
|
||||
downloader.client.CloseIdleConnections()
|
||||
}
|
||||
body, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "https://huangguo.video/")
|
||||
if err != nil || body != "fixture browser page" {
|
||||
t.Fatal(body, err)
|
||||
}
|
||||
}
|
||||
if nativeCalls.Load() != 0 {
|
||||
t.Fatal("browser request first used the native transport")
|
||||
}
|
||||
request, _ := http.NewRequest(http.MethodGet, "https://hongguoduanju.com/", nil)
|
||||
response, err := downloader.client.Do(request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response.Body.Close()
|
||||
if nativeCalls.Load() != 1 {
|
||||
t.Fatal("unrelated source did not keep native transport")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHuangguoBrowserRespectsCertificatesAndCancellation(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "fixture") }))
|
||||
defer server.Close()
|
||||
cfg := defaultConfig()
|
||||
cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), "direct", 1
|
||||
cfg.HuangguoVideoURL = server.URL
|
||||
downloader := NewDownloader(cfg)
|
||||
defer downloader.client.CloseIdleConnections()
|
||||
if _, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", ""); err == nil {
|
||||
t.Fatal("untrusted certificate was accepted by default")
|
||||
}
|
||||
blocking := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
w.(http.Flusher).Flush()
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer blocking.Close()
|
||||
cfg.HuangguoVideoURL = blocking.URL
|
||||
downloader = NewDownloader(cfg)
|
||||
defer downloader.client.CloseIdleConnections()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
|
||||
defer cancel()
|
||||
if _, err := downloader.fetchProviderText(ctx, "https://huangguo.video/videos", ""); err == nil {
|
||||
t.Fatal("canceled browser body read succeeded")
|
||||
}
|
||||
downloader.limiter.mu.Lock()
|
||||
active := downloader.limiter.active
|
||||
downloader.limiter.mu.Unlock()
|
||||
if active != 0 {
|
||||
t.Fatal("browser timeout leaked a limiter permit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHuangguoBrowserProxyResetAndBackoff(t *testing.T) {
|
||||
cfg := defaultConfig()
|
||||
cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), "http://proxy-one.invalid:8080", 3
|
||||
downloader := NewDownloader(cfg)
|
||||
defer downloader.client.CloseIdleConnections()
|
||||
downloader.limiter = newRequestLimiter(2, time.Nanosecond)
|
||||
transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
|
||||
var routes []string
|
||||
var clients []*browserFixtureClient
|
||||
calls := 0
|
||||
transport.newClient = func(proxy string) (browserHTTPClient, error) {
|
||||
routes = append(routes, proxy)
|
||||
client := &browserFixtureClient{do: func(r *fhttp.Request) (*fhttp.Response, error) {
|
||||
calls++
|
||||
status, body := 403, "Cloudflare: Sorry, you have been blocked"
|
||||
if len(routes) > 1 {
|
||||
status, body = 200, "fixture success"
|
||||
}
|
||||
return &fhttp.Response{StatusCode: status, Header: fhttp.Header{"Cf-Ray": {"fixture-ray"}}, Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||
}}
|
||||
clients = append(clients, client)
|
||||
return client, nil
|
||||
}
|
||||
for index := 0; index < 2; index++ {
|
||||
_, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "Cloudflare 拒绝了当前请求") {
|
||||
t.Fatal("missing accurate block cause", err)
|
||||
}
|
||||
}
|
||||
if calls != 1 || len(routes) != 1 {
|
||||
t.Fatal("backoff did not prevent repeated requests", calls, len(routes))
|
||||
}
|
||||
downloader.proxyRouter.configure("socks5://proxy-two.invalid:1080")
|
||||
downloader.client.CloseIdleConnections()
|
||||
if clients[0].closed.Load() == 0 {
|
||||
t.Fatal("old proxy session not closed")
|
||||
}
|
||||
_, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
|
||||
if err == nil || calls != 1 {
|
||||
t.Fatal("proxy switch bypassed the existing backoff")
|
||||
}
|
||||
downloader.limiter.mu.Lock()
|
||||
delete(downloader.limiter.backoffs, "huangguo.video")
|
||||
downloader.limiter.mu.Unlock()
|
||||
body, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
|
||||
if err != nil || body != "fixture success" || len(routes) != 2 || routes[1] != "socks5://proxy-two.invalid:1080" {
|
||||
t.Fatal("updated proxy was not used", routes, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHuangguoBrowserRedirectUsesNativePolicy(t *testing.T) {
|
||||
cfg := defaultConfig()
|
||||
cfg.dataDir, cfg.OutputDir, cfg.ProxyURL = t.TempDir(), t.TempDir(), "direct"
|
||||
downloader := NewDownloader(cfg)
|
||||
defer downloader.client.CloseIdleConnections()
|
||||
transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
|
||||
transport.newClient = func(string) (browserHTTPClient, error) {
|
||||
return &browserFixtureClient{do: func(r *fhttp.Request) (*fhttp.Response, error) {
|
||||
return &fhttp.Response{StatusCode: 302, Header: fhttp.Header{"Location": {"https://redirect.invalid/fixture"}, "Set-Cookie": {"fixture_session=private; Path=/"}}, Body: io.NopCloser(strings.NewReader(""))}, nil
|
||||
}}, nil
|
||||
}
|
||||
transport.base = rankingTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.URL.Host != "redirect.invalid" || r.Header.Get("Cookie") != "" || r.Header.Get("Sec-Ch-Ua") != "" {
|
||||
t.Error("redirect leaked fingerprint session headers")
|
||||
}
|
||||
return rankingHTTPResponse(r, 200, "redirected fixture"), nil
|
||||
})
|
||||
body, err := downloader.fetchProviderText(context.Background(), "https://huangguo.video/videos", "")
|
||||
if err != nil || body != "redirected fixture" {
|
||||
t.Fatal(body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHuangguoBrowserLivePages(t *testing.T) {
|
||||
if os.Getenv("JUKU_HUANGGUO_BROWSER_PROBE") != "1" {
|
||||
t.Skip("text-only live probe is opt-in")
|
||||
}
|
||||
cfg := defaultConfig()
|
||||
cfg.dataDir, cfg.OutputDir, cfg.ProxyURL, cfg.Retries = t.TempDir(), t.TempDir(), firstNonEmpty(os.Getenv("JUKU_HUANGGUO_BROWSER_PROXY"), "direct"), 1
|
||||
cfg.RequestIntervalMS = 2000
|
||||
downloader := NewDownloader(cfg)
|
||||
defer downloader.client.CloseIdleConnections()
|
||||
var events []diagnosticEvent
|
||||
transport := downloader.client.Transport.(*imageTransport).base.(*huangguoBrowserTransport)
|
||||
transport.record = func(event diagnosticEvent) { events = append(events, event) }
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 70*time.Second)
|
||||
defer cancel()
|
||||
urls := []string{"https://huangguo.video/videos", "https://huangguo.video/videos?category=1"}
|
||||
counts := []int{}
|
||||
detailURL := ""
|
||||
for _, address := range urls {
|
||||
body, err := downloader.fetchProviderText(ctx, address, "https://huangguo.video/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dramas := parseHuangguoVideoCards(body, address)
|
||||
if len(dramas) == 0 {
|
||||
t.Fatal("200 page contained no business records")
|
||||
}
|
||||
counts = append(counts, len(dramas))
|
||||
if detailURL == "" {
|
||||
for _, drama := range dramas {
|
||||
_, id, ok := splitProviderDramaID(drama.ID)
|
||||
if ok && strings.HasPrefix(id, "series/") {
|
||||
detailURL = "https://huangguo.video/" + id
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if detailURL == "" {
|
||||
t.Fatal("missing linked detail")
|
||||
}
|
||||
body, err := downloader.fetchProviderText(ctx, detailURL, "https://huangguo.video/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
episodes := parseHuangguoVideoEpisodes(body, detailURL)
|
||||
if len(episodes) == 0 {
|
||||
t.Fatal("detail contained no episode links")
|
||||
}
|
||||
counts = append(counts, len(episodes))
|
||||
evidence := map[string]any{"time": time.Now().UTC(), "client": "tls-client v1.16.0 / Chrome 150", "counts": counts, "requests": events, "route": map[bool]string{true: "direct", false: "configured-proxy"}[cfg.ProxyURL == "direct"]}
|
||||
if output := os.Getenv("JUKU_HUANGGUO_BROWSER_EVIDENCE"); output != "" {
|
||||
data, _ := json.MarshalIndent(evidence, "", " ")
|
||||
if err := os.WriteFile(filepath.Clean(output), data, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, event := range events {
|
||||
if event.HTTPStatus != 200 || event.Protocol != "HTTP/2.0" {
|
||||
t.Fatal("unexpected live response", event.HTTPStatus, event.Protocol)
|
||||
}
|
||||
}
|
||||
t.Logf("text-only list/category/detail counts: %v; responses=%d", counts, len(events))
|
||||
}
|
||||
Reference in New Issue
Block a user