Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5abdee06ff |
@@ -3,6 +3,7 @@ package browser
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -55,10 +56,12 @@ func FetchViaChrome(url string) ([]byte, error) {
|
||||
defer timeoutCancel()
|
||||
|
||||
// Navigate and wait for Cloudflare challenge to resolve.
|
||||
// Poll the page content until we get valid JSON (not the challenge page).
|
||||
log.Printf("chrome-fetch: navigating to %s (profile: %s)", url, profileDir)
|
||||
if err := chromedp.Run(ctx, chromedp.Navigate(url)); err != nil {
|
||||
log.Printf("chrome-fetch: navigate failed: %v", err)
|
||||
return nil, fmt.Errorf("chromedp navigate: %w", err)
|
||||
}
|
||||
log.Printf("chrome-fetch: navigation complete, polling for JSON...")
|
||||
|
||||
// Poll for JSON response — Cloudflare challenge takes a few seconds to clear
|
||||
ticker := time.NewTicker(1 * time.Second)
|
||||
@@ -82,15 +85,23 @@ func FetchViaChrome(url string) ([]byte, error) {
|
||||
}
|
||||
body = strings.TrimSpace(body)
|
||||
if body == "" {
|
||||
log.Printf("chrome-fetch: page body empty, waiting...")
|
||||
continue
|
||||
}
|
||||
// Check if we got actual JSON (starts with [ or {), not a challenge page
|
||||
if body[0] == '[' || body[0] == '{' {
|
||||
log.Printf("chrome-fetch: got JSON response (%d bytes)", len(body))
|
||||
// Also extract any fresh cookies for future plain HTTP attempts
|
||||
_ = extractAndSaveCookies(ctx)
|
||||
cancel() // graceful close, flushes cookies to profile
|
||||
return []byte(body), nil
|
||||
}
|
||||
// Log a snippet of what we got (challenge page, login redirect, etc.)
|
||||
snippet := body
|
||||
if len(snippet) > 200 {
|
||||
snippet = snippet[:200]
|
||||
}
|
||||
log.Printf("chrome-fetch: non-JSON body (%d bytes): %s", len(body), snippet)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,11 +102,13 @@ func fetchWithFallback(url, sessionKey string) ([]byte, error) {
|
||||
}
|
||||
if status == 403 {
|
||||
// Likely a Cloudflare JS challenge — fall back to headless Chrome
|
||||
log.Printf("HTTP 403 for %s, falling back to headless Chrome", url)
|
||||
log.Printf("HTTP 403 for %s, falling back to Chrome", url)
|
||||
chromeBody, chromeErr := browser.FetchViaChrome(url)
|
||||
if chromeErr != nil {
|
||||
return nil, fmt.Errorf("auth_expired") // treat as auth failure if Chrome also fails
|
||||
log.Printf("Chrome fallback failed: %v", chromeErr)
|
||||
return nil, fmt.Errorf("auth_expired")
|
||||
}
|
||||
log.Printf("Chrome fallback succeeded (%d bytes)", len(chromeBody))
|
||||
return chromeBody, nil
|
||||
}
|
||||
return nil, fmt.Errorf("HTTP %d", status)
|
||||
|
||||
Reference in New Issue
Block a user