Java headless browser guide: Run websites without a UI

java -version

package org.example;

public class App {
  public static void main(String[] args) {
    System.out.println("Java headless browser setup OK");
  }
}

package org.example;

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.time.Duration;

public class App {

  public static void main(String[] args) {

    // Configure Chrome to run in headless mode.
    // "--headless=new" is the recommended flag for modern Chrome versions (2025+).
    ChromeOptions options = new ChromeOptions();
    options.addArguments("--headless=new");
    options.addArguments("--disable-gpu");
    options.addArguments("--window-size=1280,800");

    WebDriver driver = null;

    try {
      // Create the WebDriver instance.
      // Selenium Manager will automatically resolve the correct ChromeDriver.
			// You still need Chrome/Firefox installed (or a container image that includes it).
			// For example, if CI can't find a browser, Selenium Manager can't save you.
      driver = new ChromeDriver(options);

      // Set page load timeout to avoid infinite waits on broken pages.
      driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(30));

      // Disable implicit waits to avoid conflicts with explicit waits.
      driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(0));

      // Open the target page.
      String url = "https://example.com";
      driver.get(url);

      // Explicit wait for JavaScript-rendered content.
      // Always wait for a specific element instead of guessing with sleep().
      WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(15));
      WebElement title = wait.until(
        ExpectedConditions.visibilityOfElementLocated(By.cssSelector("h1"))
      );

      // Extract visible text from the page.
      String text = title.getText();
      System.out.println("Page title: " + text);

      // Extract an attribute from another element.
      String href = driver.findElement(By.cssSelector("a")).getAttribute("href");
      System.out.println("First link: " + href);

    } catch (Exception e) {
      // Basic error handling.
      // In real projects, replace this with proper logging.
      System.err.println("Browser automation failed: " + e.getMessage());
      e.printStackTrace();

    } finally {
      // Always shut down the browser.
      // This prevents orphaned Chrome processes.
      if (driver != null) {
        driver.quit();
      }
    }
  }
}

package org.example;

import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.time.Duration;
import java.util.Set;

public class App {

  public static void main(String[] args) {

    // Headless Chrome config that stays stable on dev machines and CI runners.
    ChromeOptions options = new ChromeOptions();
    options.addArguments("--headless=new");
    options.addArguments("--disable-gpu");
    options.addArguments("--window-size=1280,800");
    options.addArguments("--no-sandbox"); // Useful on some CI containers.
    options.addArguments("--disable-dev-shm-usage"); // Helps avoid shared memory issues in Docker.

    WebDriver driver = null;

    try {
      driver = new ChromeDriver(options);
      driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(45));
      driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(0));

      WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(20));

      // 1) Go to the login page.
      driver.get("https://example.com/login");

      // 2) Wait for the login form to exist before interacting.
      WebElement emailInput = wait.until(
        ExpectedConditions.elementToBeClickable(By.cssSelector("input[type='email'], input[name='email']"))
      );
      WebElement passwordInput = wait.until(
        ExpectedConditions.elementToBeClickable(By.cssSelector("input[type='password'], input[name='password']"))
      );

      emailInput.clear();
      emailInput.sendKeys("YOUR_EMAIL");
      passwordInput.clear();
      passwordInput.sendKeys("YOUR_PASSWORD");

      // 3) Submit the form. Some sites want click, some accept submit().
      WebElement submit = driver.findElement(By.cssSelector("button[type='submit'], button[name='login']"));
      submit.click();

      // 4) Wait for a post-login signal. Pick a selector that only appears when logged in.
      wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("[data-test='dashboard'], .dashboard, nav")));

      // 5) Now you are logged in. Cookies are stored in the browser session automatically.
      Set<Cookie> cookies = driver.manage().getCookies();
      System.out.println("Logged in. Cookie count: " + cookies.size());

      // 6) Navigate to an authenticated page using the same driver session.
      driver.get("https://example.com/account");

      // Wait for something that proves the authenticated page loaded.
      wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("h1")));

      System.out.println("Account page loaded.");

    } catch (Exception e) {
      System.err.println("Login automation failed: " + e.getMessage());
      e.printStackTrace();
    } finally {
      if (driver != null) {
        driver.quit();
      }
    }
  }
}

// Wait for a SPA route to finish rendering by waiting for a route-specific element.
wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("[data-page='search-results']")));

// Infinite scroll pattern: scroll, then wait for more cards to load.
int previousCount = driver.findElements(By.cssSelector(".result-card")).size();
((org.openqa.selenium.JavascriptExecutor) driver).executeScript("window.scrollTo(0, document.body.scrollHeight);");
wait.until(d -> d.findElements(By.cssSelector(".result-card")).size() > previousCount);

// Loader pattern: wait for a spinner to disappear before scraping.
wait.until(ExpectedConditions.invisibilityOfElementLocated(By.cssSelector(".spinner, .loading-indicator")));

package org.example;

import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.PageLoadStrategy;

import java.nio.file.Path;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;

/**
 * Centralized Chrome setup for headless automation.
 * Keeps browser configuration out of App.java and makes it reusable.
 */
public final class BrowserFactory {

  private BrowserFactory() {}

  public static WebDriver createChrome(Path downloadDir) {

    // Configure Chrome to download files automatically to a known directory.
    // Without this, headless downloads are unreliable or blocked.
    Map<String, Object> prefs = new HashMap<>();
    prefs.put("download.default_directory", downloadDir.toAbsolutePath().toString());
    prefs.put("download.prompt_for_download", false);
    prefs.put("download.directory_upgrade", true);
    prefs.put("safebrowsing.enabled", true);

    ChromeOptions options = new ChromeOptions();
    options.setExperimentalOption("prefs", prefs);

    // Do not wait for every tracking/analytics script to finish.
    // We rely on explicit waits for elements instead.
    options.setPageLoadStrategy(PageLoadStrategy.EAGER);

    // Modern headless mode (Chrome 109+).
    options.addArguments("--headless=new");

    // Consistent viewport across machines and CI runners.
    options.addArguments("--window-size=1280,800");
    options.addArguments("--disable-gpu");

    // Slightly reduce obvious automation fingerprints.
    options.addArguments("--disable-blink-features=AutomationControlled");

    // Useful when running in Docker or locked-down environments.
    // options.addArguments("--no-sandbox");
    // options.addArguments("--disable-dev-shm-usage");

    WebDriver driver = new ChromeDriver(options);

    driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(0));
    driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(30));
    driver.manage().timeouts().scriptTimeout(Duration.ofSeconds(30));

    return driver;
  }
}

package org.example;

import java.nio.file.Files;
import java.nio.file.Path;

/**
 * Centralized configuration for the automation script.
 *
 * Keeps credentials, base URLs, and filesystem paths in one place
 * so they are not scattered across the codebase.
 */
public final class Config {

  // PragProg base URL is stable and safe to hardcode for this example.
  public static final String BASE_URL = "https://pragprog.com/";

  private final String email;
  private final String password;
  private final Path downloadDir;

  private Config(String email, String password, Path downloadDir) {
    this.email = email;
    this.password = password;
    this.downloadDir = downloadDir;
  }

  /**
   * Load configuration from the environment and prepare local directories.
   *
   * - Credentials come from environment variables (no secrets in code).
   * - Download directory is created once and reused.
   */
  public static Config load() {

    // Read credentials from environment variables.
    // This keeps secrets out of source control.
    String email = readRequiredEnv("PRAGPROG_EMAIL");
    String password = readRequiredEnv("PRAGPROG_PASSWORD");

    // Create a cross-platform download directory inside the user's home.
    Path homeDir = resolveUserHome();
    Path downloadDir = homeDir.resolve("pragprog-downloads");

    // Ensure the download directory exists before the browser starts.
    try {
      Files.createDirectories(downloadDir);
    } catch (Exception e) {
      throw new IllegalStateException(
          "Failed to create download directory: " + downloadDir.toAbsolutePath(), e
      );
    }

    return new Config(email, password, downloadDir);
  }

  public String email() {
    return email;
  }

  public String password() {
    return password;
  }

  public Path downloadDir() {
    return downloadDir;
  }

  /**
   * Read a required environment variable or fail fast with a clear error.
   */
  private static String readRequiredEnv(String name) {
    String value = System.getenv(name);
    if (value == null || value.isBlank()) {
      throw new IllegalStateException(
          "Missing required environment variable: " + name
      );
    }
    return value.trim();
  }

  /**
   * Resolve the current user's home directory in a platform-independent way.
   */
  private static Path resolveUserHome() {
    String home = System.getProperty("user.home");
    if (home == null || home.isBlank()) {
      throw new IllegalStateException("user.home system property is not set");
    }
    return Path.of(home);
  }
}

package org.example;

import org.openqa.selenium.By;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.time.Duration;
import java.util.List;

/**
 * Handles all account-related browser actions:
 * - finding the "My Orders" entry point
 * - logging in only when necessary
 *
 * Keeping this logic isolated keeps App.java readable and boring (in a good way).
 */
public final class AccountClient {

  // Main wait used for normal navigation and form interactions.
  private static final Duration MAIN_TIMEOUT = Duration.ofSeconds(20);

  // Short wait used only for presence checks (e.g. "is login form here?").
  private static final Duration SHORT_TIMEOUT = Duration.ofSeconds(5);

  private final WebDriver driver;
  private final WebDriverWait wait;

  public AccountClient(WebDriver driver) {
    this.driver = driver;
    this.wait = new WebDriverWait(driver, MAIN_TIMEOUT);
  }

  /**
   * Resolve the "My Orders" URL from PragProg's homepage.
   *
   * We do not hardcode this URL because PragProg redirects it to SendOwl,
   * and that redirect target can change over time.
   */
  public String resolveOrdersUrl() {

    driver.get(Config.BASE_URL);

    // The exact top-bar structure can change, so wait for a generic header/nav area.
    wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("header, nav")));

    // First attempt: look for a direct "My Orders" link via XPath.
    List<WebElement> links = driver.findElements(
        By.xpath("//a[normalize-space()='My Orders' or contains(normalize-space(), 'My Orders')]")
    );

    for (WebElement link : links) {
      String href = safeAttr(link, "href");
      if (!href.isBlank()) {
        return href.trim();
      }
    }

    // Fallback: scan all anchors and match by visible text.
    // This survives markup changes where text is wrapped in spans or icons.
    List<WebElement> allLinks = driver.findElements(By.cssSelector("a"));
    for (WebElement link : allLinks) {
      String text = safeText(link);
      if (text.equalsIgnoreCase("My Orders") || text.toLowerCase().contains("my orders")) {
        String href = safeAttr(link, "href");
        if (!href.isBlank()) {
          return href.trim();
        }
      }
    }

    throw new IllegalStateException("Could not find the My Orders link on PragProg home.");
  }

  /**
   * Navigate to the orders page and log in if required.
   *
   * SendOwl shows the same URL for both logged-in and logged-out users,
   * so we detect login state by checking for the login form.
   */
  public void goToOrdersAndLoginIfNeeded(String ordersUrl, String email, String password) {

    driver.get(ordersUrl);

    // Only log in if the login form is present.
    if (isLoginFormPresent()) {
      login(email, password);
    }

    // Final safety check: if the login form is still there, something went wrong.
    if (isLoginFormPresent()) {
      throw new IllegalStateException(
          "Login did not succeed (login form still present). Check credentials or captcha."
      );
    }
  }

  /**
   * Perform the SendOwl login flow.
   *
   * Uses form submission instead of clicking a specific button
   * to avoid brittle selectors.
   */
  private void login(String email, String password) {

    WebElement emailInput = wait.until(
        ExpectedConditions.elementToBeClickable(
            By.cssSelector("input[name='customer_session[email]']")
        )
    );
    WebElement passwordInput = wait.until(
        ExpectedConditions.elementToBeClickable(
            By.cssSelector("input[name='customer_session[password]']")
        )
    );

    emailInput.clear();
    emailInput.sendKeys(email);

    passwordInput.clear();
    passwordInput.sendKeys(password);

    // Submit the surrounding form to mimic a real user action.
    WebElement form = emailInput.findElement(By.xpath("./ancestor::form[1]"));
    form.submit();

    // Wait until the login form disappears, signaling a successful login.
    wait.until(ExpectedConditions.invisibilityOfElementLocated(
        By.cssSelector("input[name='customer_session[password]']")
    ));
  }

  /**
   * Check whether the login form is currently visible.
   *
   * Uses a short timeout because this is a presence probe, not a full wait.
   */
  private boolean isLoginFormPresent() {
    WebDriverWait shortWait = new WebDriverWait(driver, SHORT_TIMEOUT);
    try {
      shortWait.until(ExpectedConditions.presenceOfElementLocated(
          By.cssSelector("input[name='customer_session[email]']")
      ));
      return true;
    } catch (TimeoutException ignored) {
      return false;
    }
  }

  // ---------- Small safety helpers ----------

  private static String safeText(WebElement el) {
    try {
      String t = el.getText();
      return t == null ? "" : t.trim();
    } catch (Exception ignored) {
      return "";
    }
  }

  private static String safeAttr(WebElement el, String name) {
    try {
      String v = el.getAttribute(name);
      return v == null ? "" : v.trim();
    } catch (Exception ignored) {
      return "";
    }
  }

  private static String safeUrl(WebDriver d) {
    try {
      String u = d.getCurrentUrl();
      return u == null ? "" : u.trim();
    } catch (Exception ignored) {
      return "";
    }
  }
}

package org.example;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.Comparator;
import java.util.Optional;
import java.util.stream.Stream;

/**
 * Wait helpers for Chrome downloads.
 *
 * Clicking a download link returns immediately, but the file is written asynchronously.
 * Chrome also writes temporary ".crdownload" files while a download is in progress.
 *
 * This helper waits until:
 * - there are no temp download files
 * - a new file appears (newer than the baseline)
 * - the file size is stable across multiple checks
 */
public final class DownloadHelper {

  private DownloadHelper() {}

  // Chrome temporary download suffix (most common).
  private static final String CHROME_TEMP_SUFFIX = ".crdownload";

  // Poll interval: fast enough to feel responsive, slow enough to avoid busy looping.
  private static final long POLL_MS = 300;

  /**
   * Wait until a new download finishes and return the downloaded file path.
   *
   * The logic is based on "newest modified file after baseline + stable size".
   * This avoids fragile assumptions like "download finishes instantly" or "file count increments by 1".
   */
  public static Path waitForNewDownload(Path downloadDir, Duration timeout) {
    Instant deadline = Instant.now().plus(timeout);

    // Baseline: newest file timestamp at the moment we start waiting.
    // We only accept files that appear or change after this point.
    long baselineNewestMtime = newestRegularFile(downloadDir)
        .map(DownloadHelper::lastModifiedMillisSafe)
        .orElse(0L);

    while (Instant.now().isBefore(deadline)) {

      // If any ".crdownload" files exist, Chrome is still writing.
      // Don't trust directory state yet.
      if (hasTempDownloads(downloadDir)) {
        sleep(POLL_MS);
        continue;
      }

      // Candidate = newest regular file in the directory.
      Optional<Path> newest = newestRegularFile(downloadDir);
      if (newest.isPresent()) {
        Path candidate = newest.get();
        long mtime = lastModifiedMillisSafe(candidate);

        // Only accept files that are newer than our baseline snapshot.
        if (mtime > baselineNewestMtime) {

          // Extra safety: size must be stable across multiple reads.
          // This reduces the chance of returning a partially written file.
          if (isFileStable(candidate, Duration.ofMillis(900), 3)) {
            return candidate;
          }
        }
      }

      sleep(POLL_MS);
    }

    throw new IllegalStateException(
        "Timed out waiting for download to finish in: " + downloadDir.toAbsolutePath()
    );
  }

  private static boolean hasTempDownloads(Path dir) {
    try (Stream<Path> s = Files.list(dir)) {
      return s.anyMatch(p -> p.getFileName().toString().endsWith(CHROME_TEMP_SUFFIX));
    } catch (IOException e) {
      throw new IllegalStateException("Failed to scan download directory: " + dir.toAbsolutePath(), e);
    }
  }

  private static Optional<Path> newestRegularFile(Path dir) {
    try (Stream<Path> s = Files.list(dir)) {
      return s.filter(Files::isRegularFile)
          .max(Comparator.comparingLong(DownloadHelper::lastModifiedMillisSafe));
    } catch (IOException e) {
      throw new IllegalStateException("Failed to scan download directory: " + dir.toAbsolutePath(), e);
    }
  }

  private static long lastModifiedMillisSafe(Path p) {
    try {
      return Files.getLastModifiedTime(p).toMillis();
    } catch (IOException e) {
      return 0L;
    }
  }

  /**
   * Stability check:
   * - file exists
   * - size is > 0
   * - size stays unchanged across N reads, spaced by "interval"
   */
  private static boolean isFileStable(Path file, Duration interval, int checks) {
    if (!Files.isRegularFile(file)) return false;

    long lastSize = sizeSafe(file);
    if (lastSize <= 0) return false;

    for (int i = 0; i < checks; i++) {
      sleep(interval.toMillis());

      long size = sizeSafe(file);
      if (size <= 0) return false;
      if (size != lastSize) return false;

      lastSize = size;
    }

    return true;
  }

  private static long sizeSafe(Path file) {
    try {
      return Files.size(file);
    } catch (IOException e) {
      return -1L;
    }
  }

  private static void sleep(long ms) {
    try {
      Thread.sleep(ms);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
      throw new IllegalStateException("Interrupted while waiting for download.", e);
    }
  }
}

package org.example;

import org.openqa.selenium.*;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.nio.file.Path;
import java.time.Duration;
import java.util.*;

public class App {

  /**
   * For the demo we only grab the newest few items.
   * In a real project you might download everything, or filter by date/title.
   */
  private static final int BOOKS_TO_DOWNLOAD = 3;

  // Timeouts that match "real internet" behavior (not localhost fantasy).
  private static final Duration ORDERS_TIMEOUT = Duration.ofSeconds(45);
  private static final Duration LANDING_PAGE_TIMEOUT = Duration.ofSeconds(25);
  private static final Duration DOWNLOAD_TIMEOUT = Duration.ofSeconds(90);

  // Orders page (SendOwl) selectors
  private static final By ORDERS_TBODY = By.cssSelector("tbody#ordersdisplay");
  private static final By ORDER_DOWNLOAD_LINKS = By.cssSelector("tbody#ordersdisplay a[href*='download']");

  // Download landing page (format picker) selectors
  private static final By DOWNLOAD_LIST = By.cssSelector(".download-list");
  private static final By DOWNLOAD_ITEMS = By.cssSelector(".download-list .item");
  private static final By FILE_LINKS = By.cssSelector("a[href*='/file?product_id=']");

  public static void main(String[] args) {

    WebDriver driver = null;

    try {
      Config config = Config.load();
      driver = BrowserFactory.createChrome(config.downloadDir());

      // 1) Login (PragProg -> My Orders -> SendOwl account area)
      AccountClient account = new AccountClient(driver);
      String ordersUrl = account.resolveOrdersUrl();
      account.goToOrdersAndLoginIfNeeded(ordersUrl, config.email(), config.password());

      // 2) Orders table is dynamic: wait until the links exist, not just the HTML container.
      waitForOrdersTable(driver);

      // 3) Extract the "download landing pages" as plain strings.
      // This avoids stale element pain: we don't keep WebElements around longer than needed.
      List<String> orderDownloadPages = extractOrderDownloadPages(driver);

      if (orderDownloadPages.isEmpty()) {
        System.out.println("No download links found on orders page.");
        dumpDebugArtifacts(driver, config.downloadDir());
        return;
      }

      int limit = Math.min(BOOKS_TO_DOWNLOAD, orderDownloadPages.size());
      System.out.println("Found " + orderDownloadPages.size() + " orders. Downloading " + limit + ".");

      for (int i = 0; i < limit; i++) {
        String orderDownloadPageUrl = orderDownloadPages.get(i);
        System.out.println("Opening order download page: " + orderDownloadPageUrl);

        Path downloaded = downloadPdfFromOrderDownloadPage(driver, orderDownloadPageUrl, config.downloadDir());
        System.out.println("Saved file: " + downloaded.getFileName());
      }

      System.out.println("Done.");

    } catch (Exception e) {
      System.err.println("Automation failed: " + e.getMessage());
      e.printStackTrace();
    } finally {
      if (driver != null) driver.quit();
    }
  }

  /**
   * SendOwl orders page renders asynchronously.
   * The most reliable readiness signal is: "at least one download link exists".
   */
  private static void waitForOrdersTable(WebDriver driver) {
    WebDriverWait wait = new WebDriverWait(driver, ORDERS_TIMEOUT);

    System.out.println("Waiting for orders table...");
    wait.until(ExpectedConditions.presenceOfElementLocated(ORDERS_TBODY));

    // Wait until at least one download link is present (real content has loaded).
    wait.until(d -> d.findElements(ORDER_DOWNLOAD_LINKS).size() > 0);

    int links = driver.findElements(ORDER_DOWNLOAD_LINKS).size();
    System.out.println("Orders ready. downloadLinks=" + links);
  }

  /**
   * Extract the per-order "download landing page" URLs.
   * Each of those pages contains multiple formats (pdf/epub/mobi), which we pick from later.
   */
  private static List<String> extractOrderDownloadPages(WebDriver driver) {

    // Keep the original ordering and avoid duplicates.
    Set<String> urls = new LinkedHashSet<>();

    for (WebElement a : driver.findElements(ORDER_DOWNLOAD_LINKS)) {
      String href = safeAttr(a, "href");
      if (!href.isBlank()) urls.add(href);
    }

    return new ArrayList<>(urls);
  }

  /**
   * Each order link opens a landing page with multiple formats.
   * We always try to download the PDF, and fall back to "first available" if PDF is missing.
   */
  private static Path downloadPdfFromOrderDownloadPage(WebDriver driver, String orderDownloadPageUrl, Path downloadDir) {

    // 1) Open the landing page safely (some pages never fully "finish loading" due to scripts).
    safeNavigate(driver, orderDownloadPageUrl);

    // 2) Wait for the formats list to appear.
    WebDriverWait wait = new WebDriverWait(driver, LANDING_PAGE_TIMEOUT);
    wait.until(ExpectedConditions.presenceOfElementLocated(DOWNLOAD_LIST));

    // 3) Resolve the real file download URL.
    // On SendOwl the actual download happens at ".../file?product_id=..."
    String fileUrl = resolvePdfFileUrl(driver);

    if (fileUrl.isBlank()) {
      throw new IllegalStateException("Could not find a file download link on: " + driver.getCurrentUrl());
    }

    System.out.println("Resolved file URL: " + fileUrl);

    // 4) Trigger the actual download by navigating directly (more reliable than clicking in headless mode).
    safeNavigate(driver, fileUrl);

    // 5) Wait until Chrome finishes writing the file to disk.
    return DownloadHelper.waitForNewDownload(downloadDir, DOWNLOAD_TIMEOUT);
  }

  /**
   * Prefer PDF when multiple formats exist.
   * We inspect each ".download-list .item" block because that's where the "pdf:" label lives.
   */
  private static String resolvePdfFileUrl(WebDriver driver) {

    List<WebElement> items = driver.findElements(DOWNLOAD_ITEMS);

    // 1) Prefer the item that mentions PDF.
    for (WebElement item : items) {
      String label = safeText(item).toLowerCase();
      if (label.contains("pdf:")) {
        String href = firstHref(item, FILE_LINKS);
        if (!href.isBlank()) return href;
      }
    }

    // 2) Fallback: first available file link (whatever format it is).
    for (WebElement item : items) {
      String href = firstHref(item, FILE_LINKS);
      if (!href.isBlank()) return href;
    }

    return "";
  }

  private static String firstHref(WebElement root, By selector) {
    try {
      WebElement a = root.findElement(selector);
      String href = a.getAttribute("href");
      return href == null ? "" : href.trim();
    } catch (Exception ignored) {
      return "";
    }
  }

  /**
   * Navigate without getting stuck on pages that never "finish loading" due to analytics/scripts.
   *
   * In modern automation it's more reliable to:
   * - navigate
   * - then wait for a specific element you care about
   * than to trust "the page is fully loaded".
   */
  private static void safeNavigate(WebDriver driver, String url) {
    try {
      driver.navigate().to(url);

      // Light sanity wait: at least the DOM should exist.
      // (We still wait for specific elements elsewhere.)
      if (driver instanceof JavascriptExecutor js) {
        try {
          js.executeScript("return document.readyState");
        } catch (Exception ignored) {}
      }

    } catch (TimeoutException e) {
      // Stop loading and continue. We'll rely on explicit element waits.
      try {
        ((JavascriptExecutor) driver).executeScript("window.stop();");
      } catch (Exception ignored) {}
    }
  }

  private static String safeText(WebElement el) {
    try {
      String t = el.getText();
      return t == null ? "" : t.trim();
    } catch (Exception ignored) {
      return "";
    }
  }

  private static String safeAttr(WebElement el, String name) {
    try {
      if (el == null) return "";
      String v = el.getAttribute(name);
      return v == null ? "" : v.trim();
    } catch (Exception ignored) {
      return "";
    }
  }

  /**
   * Debug helper: writes a screenshot + HTML to the download directory.
   * This is insanely useful when a site changes markup and your selectors stop matching.
   */
  private static void dumpDebugArtifacts(WebDriver driver, Path dir) {
    try {
      byte[] png = ((TakesScreenshot) driver).getScreenshotAs(OutputType.BYTES);
      java.nio.file.Files.write(dir.resolve("debug.png"), png);

      String html = driver.getPageSource();
      java.nio.file.Files.writeString(dir.resolve("debug.html"), html);

      System.out.println("Wrote debug.png and debug.html to: " + dir.toAbsolutePath());
      System.out.println("Debug URL: " + driver.getCurrentUrl());
      System.out.println("Debug title: " + driver.getTitle());
    } catch (Exception e) {
      System.out.println("Failed to write debug artifacts: " + e.getMessage());
    }
  }
}