Fix Cloudflare headless detection: use non-headless with hidden window

Cloudflare detects headless Chrome and loops the JS challenge forever.
Switch to non-headless mode with an off-screen window. Also save
Cloudflare cookies (cf_clearance, __cf_bm) after Chrome fallback so
subsequent plain HTTP requests can reuse them.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

internal/browser/fetch.go

@@ -5,17 +5,19 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
+	"github.com/chromedp/cdproto/network"
 	"github.com/chromedp/chromedp"
 
 	"git.davoryn.de/calic/claude-statusline/internal/config"
 )
 
-// FetchViaChrome navigates to a URL using headless Chrome with the persistent
-// browser profile (which has Cloudflare clearance cookies) and returns the
-// response body. This bypasses Cloudflare JS challenges because Chrome runs
-// real JavaScript. Falls back to non-headless if headless fails.
+// FetchViaChrome navigates to a URL using Chrome with the persistent browser
+// profile (which has Cloudflare clearance cookies) and returns the response
+// body. Uses non-headless mode with a minimized/hidden window to avoid
+// Cloudflare's headless detection, which causes infinite challenge loops.
 func FetchViaChrome(url string) ([]byte, error) {
 	profileDir := filepath.Join(config.ConfigDir(), "browser-profile")
 	if err := os.MkdirAll(profileDir, 0o755); err != nil {
@@ -27,8 +29,15 @@ func FetchViaChrome(url string) ([]byte, error) {
 
 	execPath := findBrowserExec()
 
+	// Use non-headless mode: Cloudflare detects headless Chrome and loops
+	// the JS challenge forever. A real (but hidden) browser window passes.
 	opts := append(chromedp.DefaultExecAllocatorOptions[:],
-		chromedp.Flag("headless", true),
+		chromedp.Flag("headless", false),
+		chromedp.Flag("window-position", "-32000,-32000"), // off-screen
+		chromedp.Flag("window-size", "1,1"),
+		chromedp.Flag("disable-gpu", true),
+		chromedp.Flag("no-first-run", true),
+		chromedp.Flag("disable-extensions", true),
 		chromedp.UserDataDir(profileDir),
 	)
 	if execPath != "" {
@@ -41,36 +50,73 @@ func FetchViaChrome(url string) ([]byte, error) {
 	ctx, cancel := chromedp.NewContext(allocCtx)
 	defer cancel()
 
-	// Set a total timeout for the operation
+	// Total timeout for the operation
 	ctx, timeoutCancel := context.WithTimeout(ctx, 30*time.Second)
 	defer timeoutCancel()
 
-	var body string
-	err := chromedp.Run(ctx,
-		chromedp.Navigate(url),
-		// Wait for the body to have content (Cloudflare challenge resolves via JS)
-		chromedp.WaitReady("body"),
-		// Chrome renders JSON API responses inside a <pre> tag
-		chromedp.Text("pre", &body, chromedp.ByQuery, chromedp.NodeVisible),
-	)
-	if err != nil {
-		// Fallback: try extracting from body directly (some responses may not use <pre>)
-		var bodyFallback string
-		errFb := chromedp.Run(ctx,
-			chromedp.Text("body", &bodyFallback, chromedp.ByQuery),
-		)
-		if errFb == nil && bodyFallback != "" {
-			return []byte(bodyFallback), nil
+	// Navigate and wait for Cloudflare challenge to resolve.
+	// Poll the page content until we get valid JSON (not the challenge page).
+	if err := chromedp.Run(ctx, chromedp.Navigate(url)); err != nil {
+		return nil, fmt.Errorf("chromedp navigate: %w", err)
+	}
+
+	// Poll for JSON response — Cloudflare challenge takes a few seconds to clear
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("chromedp fetch timed out waiting for JSON response")
+		case <-ticker.C:
+			var body string
+			// Try <pre> first (Chrome wraps JSON in <pre> tags)
+			err := chromedp.Run(ctx,
+				chromedp.Text("pre", &body, chromedp.ByQuery),
+			)
+			if err != nil || body == "" {
+				// Fallback: try body directly
+				_ = chromedp.Run(ctx,
+					chromedp.Text("body", &body, chromedp.ByQuery),
+				)
+			}
+			body = strings.TrimSpace(body)
+			if body == "" {
+				continue
+			}
+			// Check if we got actual JSON (starts with [ or {), not a challenge page
+			if body[0] == '[' || body[0] == '{' {
+				// Also extract any fresh cookies for future plain HTTP attempts
+				_ = extractAndSaveCookies(ctx)
+				cancel() // graceful close, flushes cookies to profile
+				return []byte(body), nil
+			}
 		}
-		return nil, fmt.Errorf("chromedp fetch: %w", err)
 	}
-
-	if body == "" {
-		return nil, fmt.Errorf("chromedp fetch: empty response body")
-	}
-
-	// Gracefully close to flush cookies (including refreshed cf_clearance)
-	cancel()
-
-	return []byte(body), nil
+}
+
+// extractAndSaveCookies saves cf_clearance and other Cloudflare cookies
+// alongside the session key, so plain HTTP requests can try them next time.
+func extractAndSaveCookies(ctx context.Context) error {
+	cookies, err := network.GetCookies().Do(ctx)
+	if err != nil {
+		return err
+	}
+
+	var parts []string
+	for _, c := range cookies {
+		if c.Domain == ".claude.ai" || c.Domain == "claude.ai" {
+			if c.Name == "cf_clearance" || c.Name == "__cf_bm" || c.Name == "_cfuvid" {
+				parts = append(parts, c.Name+"="+c.Value)
+			}
+		}
+	}
+
+	if len(parts) == 0 {
+		return nil
+	}
+
+	// Write Cloudflare cookies to a file the fetcher can read
+	cfPath := filepath.Join(config.ConfigDir(), "cf-cookies")
+	return os.WriteFile(cfPath, []byte(strings.Join(parts, "\n")+"\n"), 0o600)
 }