Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5abdee06ff |
@@ -3,6 +3,7 @@ package browser
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -55,10 +56,12 @@ func FetchViaChrome(url string) ([]byte, error) {
|
|||||||
defer timeoutCancel()
|
defer timeoutCancel()
|
||||||
|
|
||||||
// Navigate and wait for Cloudflare challenge to resolve.
|
// Navigate and wait for Cloudflare challenge to resolve.
|
||||||
// Poll the page content until we get valid JSON (not the challenge page).
|
log.Printf("chrome-fetch: navigating to %s (profile: %s)", url, profileDir)
|
||||||
if err := chromedp.Run(ctx, chromedp.Navigate(url)); err != nil {
|
if err := chromedp.Run(ctx, chromedp.Navigate(url)); err != nil {
|
||||||
|
log.Printf("chrome-fetch: navigate failed: %v", err)
|
||||||
return nil, fmt.Errorf("chromedp navigate: %w", err)
|
return nil, fmt.Errorf("chromedp navigate: %w", err)
|
||||||
}
|
}
|
||||||
|
log.Printf("chrome-fetch: navigation complete, polling for JSON...")
|
||||||
|
|
||||||
// Poll for JSON response — Cloudflare challenge takes a few seconds to clear
|
// Poll for JSON response — Cloudflare challenge takes a few seconds to clear
|
||||||
ticker := time.NewTicker(1 * time.Second)
|
ticker := time.NewTicker(1 * time.Second)
|
||||||
@@ -82,15 +85,23 @@ func FetchViaChrome(url string) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
body = strings.TrimSpace(body)
|
body = strings.TrimSpace(body)
|
||||||
if body == "" {
|
if body == "" {
|
||||||
|
log.Printf("chrome-fetch: page body empty, waiting...")
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Check if we got actual JSON (starts with [ or {), not a challenge page
|
// Check if we got actual JSON (starts with [ or {), not a challenge page
|
||||||
if body[0] == '[' || body[0] == '{' {
|
if body[0] == '[' || body[0] == '{' {
|
||||||
|
log.Printf("chrome-fetch: got JSON response (%d bytes)", len(body))
|
||||||
// Also extract any fresh cookies for future plain HTTP attempts
|
// Also extract any fresh cookies for future plain HTTP attempts
|
||||||
_ = extractAndSaveCookies(ctx)
|
_ = extractAndSaveCookies(ctx)
|
||||||
cancel() // graceful close, flushes cookies to profile
|
cancel() // graceful close, flushes cookies to profile
|
||||||
return []byte(body), nil
|
return []byte(body), nil
|
||||||
}
|
}
|
||||||
|
// Log a snippet of what we got (challenge page, login redirect, etc.)
|
||||||
|
snippet := body
|
||||||
|
if len(snippet) > 200 {
|
||||||
|
snippet = snippet[:200]
|
||||||
|
}
|
||||||
|
log.Printf("chrome-fetch: non-JSON body (%d bytes): %s", len(body), snippet)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,11 +102,13 @@ func fetchWithFallback(url, sessionKey string) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
if status == 403 {
|
if status == 403 {
|
||||||
// Likely a Cloudflare JS challenge — fall back to headless Chrome
|
// Likely a Cloudflare JS challenge — fall back to headless Chrome
|
||||||
log.Printf("HTTP 403 for %s, falling back to headless Chrome", url)
|
log.Printf("HTTP 403 for %s, falling back to Chrome", url)
|
||||||
chromeBody, chromeErr := browser.FetchViaChrome(url)
|
chromeBody, chromeErr := browser.FetchViaChrome(url)
|
||||||
if chromeErr != nil {
|
if chromeErr != nil {
|
||||||
return nil, fmt.Errorf("auth_expired") // treat as auth failure if Chrome also fails
|
log.Printf("Chrome fallback failed: %v", chromeErr)
|
||||||
|
return nil, fmt.Errorf("auth_expired")
|
||||||
}
|
}
|
||||||
|
log.Printf("Chrome fallback succeeded (%d bytes)", len(chromeBody))
|
||||||
return chromeBody, nil
|
return chromeBody, nil
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("HTTP %d", status)
|
return nil, fmt.Errorf("HTTP %d", status)
|
||||||
|
|||||||
Reference in New Issue
Block a user