How to Scrape Yandex Search Results: Python & Node.js

export SCRAPINGBEE_API_KEY="your-api-key"

python -m pip install requests

import os
import sys
from urllib.parse import urlencode

import requests

API_URL = "https://app.scrapingbee.com/api/v1/"
API_KEY = os.getenv("SCRAPINGBEE_API_KEY")

if not API_KEY:
    sys.exit(
        "Missing SCRAPINGBEE_API_KEY. "
        "Set it as an environment variable before running the script."
    )

query_string = urlencode({"text": "python web scraping"})
yandex_url = f"https://yandex.com/search/?{query_string}"

params = {
    "api_key": API_KEY,
    "url": yandex_url,
    "premium_proxy": "true",
    "render_js": "false",
}

try:
    response = requests.get(API_URL, params=params, timeout=120)
except requests.Timeout:
    sys.exit("The request timed out after 120 seconds.")
except requests.ConnectionError as exc:
    sys.exit(f"Could not connect to ScrapingBee: {exc}")
except requests.RequestException as exc:
    sys.exit(f"The request failed: {exc}")

if response.status_code != 200:
    error_preview = response.text[:500].strip()

    sys.exit(
        f"ScrapingBee returned HTTP {response.status_code}.\n"
        f"Response: {error_preview or 'No response body'}"
    )

print(f"Downloaded {len(response.content):,} bytes")
print(
    "Request cost:",
    response.headers.get("Spb-cost", "not reported"),
    "credits",
)

with open("yandex-results.html", "wb") as file:
    file.write(response.content)

print("Saved the response to yandex-results.html")

params["render_js"] = "true"

# Don't forget to remove params["premium_proxy"] = "true"
params["stealth_proxy"] = "true"
params["render_js"] = "true"

params["country_code"] = "de"

python -m pip install requests beautifulsoup4

export SCRAPINGBEE_API_KEY="your-api-key"

from urllib.parse import urlencode

query_params = {
    "text": "python web scraping",
    "lr": 84,
    "p": 0,
}

yandex_url = "https://yandex.com/search/?" + urlencode(query_params)

import os
import sys
from pprint import pprint
from urllib.parse import urlencode

import requests
from bs4 import BeautifulSoup


SCRAPINGBEE_URL = "https://app.scrapingbee.com/api/v1/"


class YandexChallengeError(RuntimeError):
    """Raised when Yandex returns a captcha page instead of results."""


class YandexPageError(RuntimeError):
    """Raised when the response does not match the expected Yandex markup."""


def normalized_text(element) -> str:
    """Join nested text nodes and remove repeated whitespace."""
    if element is None:
        return ""

    return " ".join(element.stripped_strings)


def validate_yandex_body(soup: BeautifulSoup):
    """
    Return the main results container or raise an error for a challenge page.

    Yandex may return HTTP 200 for both a real results page and SmartCaptcha,
    so checking the status code alone is not enough.
    """
    result_list = soup.select_one("#search-result")

    if result_list is not None:
        return result_list

    captcha_selectors = (
        ".CheckboxCaptcha",
        ".AdvancedCaptcha",
        "[class*='SmartCaptcha']",
        "form[action*='captcha']",
        "iframe[src*='captcha']",
        "#checkbox-captcha-form",
    )

    if any(soup.select_one(selector) for selector in captcha_selectors):
        raise YandexChallengeError(
            "Yandex returned a captcha page with HTTP 200. "
            "Retry later or switch to stealth_proxy=true."
        )

    page_title = normalized_text(soup.title).lower()
    body_preview = normalized_text(soup.body)[:2000].lower()

    if "captcha" in page_title or "smartcaptcha" in body_preview:
        raise YandexChallengeError(
            "Yandex returned a captcha page with HTTP 200. "
            "Retry later or switch to stealth_proxy=true."
        )

    raise YandexPageError(
        "The response contains neither #search-result nor a known captcha "
        "marker. Yandex may have changed its HTML."
    )


def parse_yandex_results(html: str) -> list[dict[str, str]]:
    """Extract organic result titles, URLs, and snippets from Yandex HTML."""
    soup = BeautifulSoup(html, "html.parser")
    result_list = validate_yandex_body(soup)
    results = []

    for item in result_list.select("li.serp-item"):
        link = item.select_one(
            ".OrganicTitle a.OrganicTitle-Link[href], "
            ".OrganicTitle a.Link[href]"
        )
        title_node = item.select_one(".OrganicTitle-LinkText")
        snippet_node = item.select_one(
            ".Organic-ContentWrapper .OrganicTextContentSpan"
        )

        if link is None or title_node is None:
            continue

        title = normalized_text(title_node)
        url = link.get("href", "").strip()
        snippet = normalized_text(snippet_node)

        if not title or not url:
            continue

        results.append(
            {
                "title": title,
                "url": url,
                "snippet": snippet,
            }
        )

    return results


def fetch_yandex_results(
    query: str,
    region: int = 84,
    page: int = 0,
) -> list[dict[str, str]]:
    """Fetch one Yandex results page through ScrapingBee and parse it."""
    api_key = os.getenv("SCRAPINGBEE_API_KEY")

    if not api_key:
        raise RuntimeError(
            "Missing SCRAPINGBEE_API_KEY. "
            "Set it as an environment variable before running the script."
        )

    yandex_url = "https://yandex.com/search/?" + urlencode(
        {
            "text": query,
            "lr": region,
            "p": page,
        }
    )

    params = {
        "api_key": api_key,
        "url": yandex_url,
        "premium_proxy": "true",
        "render_js": "false",
    }

    try:
        response = requests.get(
            SCRAPINGBEE_URL,
            params=params,
            timeout=(10, 120),
        )
    except requests.Timeout as exc:
        raise RuntimeError("The ScrapingBee request timed out.") from exc
    except requests.ConnectionError as exc:
        raise RuntimeError("Could not connect to ScrapingBee.") from exc
    except requests.RequestException as exc:
        raise RuntimeError(f"The request failed: {exc}") from exc

    if response.status_code != 200:
        error_preview = response.text[:500].strip() or "No response body"

        raise RuntimeError(
            f"ScrapingBee returned HTTP {response.status_code}: "
            f"{error_preview}"
        )

    return parse_yandex_results(response.text)


def main() -> None:
    try:
        results = fetch_yandex_results(
            query="python web scraping",
            region=84,
            page=0,
        )
    except (RuntimeError, YandexChallengeError, YandexPageError) as exc:
        sys.exit(f"Error: {exc}")

    if not results:
        print("Yandex returned no organic results for this query.")
        return

    pprint(results[:5], sort_dicts=False)


if __name__ == "__main__":
    main()

python yandex_scraper.py

[
    {
        "title": "Python Web Scraping Tutorial",
        "url": "https://example.com/python-scraping",
        "snippet": "A practical introduction to collecting web data with Python.",
    },
    {
        "title": "Web Scraping with Beautiful Soup",
        "url": "https://example.org/beautiful-soup-guide",
        "snippet": "Learn how to parse HTML and extract structured information.",
    },
]

if link is None or title_node is None:
    continue

{
    "title": "Example result",
    "url": "https://example.com/",
    "snippet": "",
}

import csv
import random
import sys
import time


def save_results_to_csv(
    results: list[dict[str, str]],
    filename: str,
) -> None:
    fieldnames = ["title", "url", "snippet"]

    with open(filename, "w", encoding="utf-8", newline="") as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(results)


def main() -> None:
    query = "python web scraping"
    all_results: list[dict[str, str]] = []
    seen_urls: set[str] = set()

    for page in range(3):
        print(f"Scraping page {page + 1}...")

        try:
            page_results = fetch_yandex_results(
                query=query,
                region=84,
                page=page,
            )
        except (RuntimeError, YandexChallengeError, YandexPageError) as exc:
            sys.exit(f"Error on page {page + 1}: {exc}")

        for result in page_results:
            url = result["url"]

            if url in seen_urls:
                continue

            seen_urls.add(url)
            all_results.append(result)

        print(
            f"Found {len(page_results)} results, "
            f"{len(all_results)} unique results collected."
        )

        if page < 2:
            delay = random.uniform(2, 4)
            print(f"Waiting {delay:.1f} seconds...")
            time.sleep(delay)

    if not all_results:
        sys.exit("No results were collected.")

    output_file = "yandex_results.csv"
    save_results_to_csv(all_results, output_file)

    print(
        f"Saved {len(all_results)} unique results "
        f"to {output_file}."
    )


if __name__ == "__main__":
    main()

{
  "result_1": {
    "type": "item",
    "output": {
      "title": "...",
      "url": "...",
      "snippet": "..."
    }
  },
  "result_2": {
    "type": "item",
    "output": {
      "title": "...",
      "url": "...",
      "snippet": "..."
    }
  }
}

import json
import os
import sys
from pprint import pprint
from urllib.parse import urlencode, urlparse

import requests


SCRAPINGBEE_URL = "https://app.scrapingbee.com/api/v1/"
TIMEOUT = (10, 180)
RESULT_LIMIT = 5


def build_result_rule(position: int) -> dict:
    """Build an AI extraction rule for one organic result."""
    return {
        "description": (
            f"The organic Yandex web search result displayed at position "
            f"{position}. Ignore sponsored results, ads, navigation links, "
            f"related searches, image blocks, and video blocks."
        ),
        "type": "item",
        "output": {
            "title": (
                "The exact visible title of this search result."
            ),
            "url": (
                "The absolute HTTP or HTTPS destination URL linked "
                "from this search result."
            ),
            "snippet": (
                "The exact visible descriptive text shown below this "
                "result's title. Return an empty string only when the "
                "result card contains no descriptive text."
            ),
        },
    }


AI_EXTRACT_RULES = {
    f"result_{position}": build_result_rule(position)
    for position in range(1, RESULT_LIMIT + 1)
}


def build_yandex_url(
    query: str,
    region: int = 84,
    page: int = 0,
) -> str:
    """Build a Yandex search URL from a query, region, and page index."""
    if not isinstance(query, str) or not query.strip():
        raise ValueError(
            "The search query must be a non-empty string."
        )

    if not isinstance(region, int) or region < 0:
        raise ValueError(
            "The region must be a non-negative integer."
        )

    if not isinstance(page, int) or page < 0:
        raise ValueError(
            "The page index must be a non-negative integer."
        )

    return "https://yandex.com/search/?" + urlencode(
        {
            "text": query.strip(),
            "lr": region,
            "p": page,
        }
    )


def normalize_results(
    payload: object,
) -> list[dict[str, str]]:
    """Validate the AI response and return a consistent result list."""
    if not isinstance(payload, dict):
        raise RuntimeError(
            "AI extraction returned JSON, but not a JSON object."
        )

    results: list[dict[str, str]] = []

    for position in range(1, RESULT_LIMIT + 1):
        raw = payload.get(f"result_{position}")

        if not isinstance(raw, dict):
            continue

        title = str(raw.get("title") or "").strip()
        url = str(raw.get("url") or "").strip()
        snippet = str(raw.get("snippet") or "").strip()

        parsed_url = urlparse(url)

        if (
            not title
            or parsed_url.scheme not in {"http", "https"}
            or not parsed_url.netloc
        ):
            continue

        results.append(
            {
                "title": title,
                "url": url,
                "snippet": snippet,
            }
        )

    return results


def fetch_yandex_results_with_ai(
    query: str,
    region: int = 84,
    page: int = 0,
) -> list[dict[str, str]]:
    """Fetch and extract one Yandex results page with AI extraction."""
    api_key = os.getenv("SCRAPINGBEE_API_KEY")

    if not api_key:
        raise RuntimeError(
            "Missing SCRAPINGBEE_API_KEY. Set it as an "
            "environment variable before running the script."
        )

    params = {
        "api_key": api_key,
        "url": build_yandex_url(query, region, page),
        "premium_proxy": "true",
        "render_js": "false",
        "ai_selector": "#search-result",
        "ai_extract_rules": json.dumps(
            AI_EXTRACT_RULES,
            ensure_ascii=False,
        ),
    }

    try:
        response = requests.get(
            SCRAPINGBEE_URL,
            params=params,
            timeout=TIMEOUT,
        )
    except requests.Timeout as exc:
        raise RuntimeError(
            "The ScrapingBee request timed out."
        ) from exc
    except requests.ConnectionError as exc:
        raise RuntimeError(
            "Could not connect to ScrapingBee."
        ) from exc
    except requests.RequestException as exc:
        raise RuntimeError(
            f"The request failed: {exc}"
        ) from exc

    if response.status_code != 200:
        preview = response.text[:500].strip() or "No response body"

        raise RuntimeError(
            f"ScrapingBee returned HTTP {response.status_code}: "
            f"{preview}"
        )

    try:
        payload = response.json()
    except requests.exceptions.JSONDecodeError as exc:
        preview = response.text[:500].strip() or "No response body"

        raise RuntimeError(
            "ScrapingBee returned HTTP 200, but the AI extraction "
            f"response was not valid JSON: {preview}"
        ) from exc

    return normalize_results(payload)


def main() -> None:
    try:
        results = fetch_yandex_results_with_ai(
            query="python web scraping",
            region=84,
            page=0,
        )
    except (RuntimeError, ValueError) as exc:
        sys.exit(f"Error: {exc}")

    if not results:
        sys.exit("No valid organic results were extracted.")

    pprint(results, sort_dicts=False)


if __name__ == "__main__":
    main()

RESULT_LIMIT = 10

"ai_selector": "#search-result"

{
  "result_1": {
    "title": "Python Web Scraping Tutorial",
    "url": "https://example.com/python-scraping",
    "snippet": "Learn how to collect and parse web data with Python."
  },
  "result_2": {
    "title": "Web Scraping with Beautiful Soup",
    "url": "https://example.org/beautiful-soup",
    "snippet": "A guide to extracting structured data from HTML."
  }
}

[
    {
        "title": "Python Web Scraping Tutorial",
        "url": "https://example.com/python-scraping",
        "snippet": "Learn how to collect and parse web data with Python.",
    },
    {
        "title": "Web Scraping with Beautiful Soup",
        "url": "https://example.org/beautiful-soup",
        "snippet": "A guide to extracting structured data from HTML.",
    },
]

import json

EXTRACT_RULES = {
    "results": {
        "selector": "#search-result li.serp-item",
        "type": "list",
        "output": {
            "title": ".OrganicTitle-LinkText",
            "url": {
                "selector": (
                    ".OrganicTitle a.OrganicTitle-Link, "
                    ".OrganicTitle a.Link"
                ),
                "output": "@href",
            },
            "snippet": (
                ".Organic-ContentWrapper "
                ".OrganicTextContentSpan"
            ),
        },
    }
}

params = {
    "api_key": API_KEY,
    "url": yandex_url,
    "premium_proxy": "true",
    "render_js": "false",
    "extract_rules": json.dumps(EXTRACT_RULES),
}

response = requests.get(
    "https://app.scrapingbee.com/api/v1/",
    params=params,
    timeout=(10, 120),
)
response.raise_for_status()

results = response.json().get("results", [])

<div
  id="ImagesApp-..."
  data-state="{...}"
></div>

import json
import os
import sys
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup


API_URL = "https://app.scrapingbee.com/api/v1/"
YANDEX_URL = "https://yandex.com/images/search?text=nature"

API_KEY = os.getenv("SCRAPINGBEE_API_KEY")

if not API_KEY:
    sys.exit("Missing SCRAPINGBEE_API_KEY.")


def normalize_text(value: str | None) -> str:
    if not value:
        return ""

    return " ".join(str(value).split())


def normalize_url(value: str | None) -> str:
    if not value:
        return ""

    value = str(value).strip()

    if not value or value.startswith("data:"):
        return ""

    return urljoin(YANDEX_URL, value)


params = {
    "api_key": API_KEY,
    "url": YANDEX_URL,
    "premium_proxy": "true",
    "render_js": "false",

    # Enable when using "render_js": "true"
    # During my own tests, everything worked fine with JS rendering
    # "wait_browser": "networkidle2",
}

try:
    response = requests.get(
        API_URL,
        params=params,
        timeout=(10, 180),
    )
    response.raise_for_status()
except requests.Timeout:
    sys.exit("The ScrapingBee request timed out.")
except requests.ConnectionError as exc:
    sys.exit(f"Could not connect to ScrapingBee: {exc}")
except requests.RequestException as exc:
    sys.exit(f"Request failed: {exc}")

soup = BeautifulSoup(response.text, "html.parser")

captcha = soup.select_one(
    ".CheckboxCaptcha, "
    ".AdvancedCaptcha, "
    "[class*='SmartCaptcha'], "
    "form[action*='captcha'], "
    "iframe[src*='captcha'], "
    "#checkbox-captcha-form"
)

if captcha:
    sys.exit("Yandex returned a captcha page.")

images_app = soup.select_one(
    '[id^="ImagesApp-"][data-state]'
)

if images_app is None:
    sys.exit(
        "Could not find the ImagesApp data-state attribute."
    )

raw_state = images_app.get("data-state")

if not raw_state:
    sys.exit(
        "The ImagesApp data-state attribute is empty."
    )

try:
    state = json.loads(raw_state)
except json.JSONDecodeError as exc:
    start = max(0, exc.pos - 150)
    end = min(len(raw_state), exc.pos + 150)

    print(
        f"JSON error at position {exc.pos}: {exc.msg}"
    )
    print(repr(raw_state[start:end]))

    sys.exit("Could not decode ImagesApp JSON.")

try:
    items = state["initialState"]["serpList"]["items"]
    keys = items["keys"]
    entities = items["entities"]
except (KeyError, TypeError) as exc:
    sys.exit(
        f"Unexpected Yandex Images structure: {exc}"
    )

results = []

for image_id in keys:
    item = entities.get(image_id)

    if not isinstance(item, dict):
        continue

    viewer_data = item.get("viewerData")

    if not isinstance(viewer_data, dict):
        viewer_data = {}

    thumbnail_url = normalize_url(
        item.get("image")
    )

    original_url = normalize_url(
        item.get("origUrl")
        or viewer_data.get("img_href")
    )

    alt = normalize_text(item.get("alt"))

    if not thumbnail_url and not original_url:
        continue

    results.append(
        {
            "alt": alt,
            "thumbnail_url": thumbnail_url,
            "original_url": original_url,
        }
    )

if not results:
    sys.exit(
        "The JSON was parsed, but no image results were found."
    )

for index, result in enumerate(results[:5], start=1):
    print(
        f"{index}. {result['alt'] or '(no alt text)'}"
    )
    print(
        f"   Thumbnail: {result['thumbnail_url']}"
    )
    print(
        f"   Original:  {result['original_url']}"
    )

npm install scrapingbee cheerio

export SCRAPINGBEE_API_KEY="your-api-key"

// This example uses ES modules and requires Node.js 24 or later.

import scrapingbee from "scrapingbee";
import * as cheerio from "cheerio";

const { ScrapingBeeClient } = scrapingbee;

const API_KEY = process.env.SCRAPINGBEE_API_KEY;

// Keep Yandex selectors in one place because search-result markup can change.
const RESULT_LIST_SELECTOR = "#search-result";
const RESULT_ITEM_SELECTOR = "li.serp-item";

const TITLE_LINK_SELECTOR =
  ".OrganicTitle a.OrganicTitle-Link[href], " +
  ".OrganicTitle a.Link[href]";

const TITLE_TEXT_SELECTOR = ".OrganicTitle-LinkText";

const SNIPPET_SELECTOR =
  ".Organic-ContentWrapper .OrganicTextContentSpan";

// Yandex can return SmartCaptcha with HTTP 200, so the response body
// must be checked before treating it as a real search-results page.
const CAPTCHA_SELECTORS = [
  ".CheckboxCaptcha",
  ".AdvancedCaptcha",
  '[class*="SmartCaptcha"]',
  'form[action*="captcha"]',
  'iframe[src*="captcha"]',
  "#checkbox-captcha-form",
];

class YandexChallengeError extends Error {
  name = "YandexChallengeError";
}

class YandexPageError extends Error {
  name = "YandexPageError";
}

function normalizeText(value = "") {
  // Cheerio's .text() may include line breaks and repeated whitespace.
  return value.replace(/\s+/g, " ").trim();
}

function decodeBody(data) {
  // The ScrapingBee SDK may return HTML as a string or Uint8Array,
  // depending on the response and runtime.
  if (typeof data === "string") {
    return data;
  }

  if (data instanceof Uint8Array) {
    return new TextDecoder().decode(data);
  }

  if (data === undefined || data === null) {
    return "";
  }

  // This fallback also makes unexpected response shapes easier to debug.
  return JSON.stringify(data);
}

function buildYandexUrl(query, region = 84, page = 0) {
  if (typeof query !== "string" || !query.trim()) {
    throw new TypeError(
      "The search query must be a non-empty string.",
    );
  }

  if (!Number.isInteger(region) || region < 0) {
    throw new TypeError(
      "The region must be a non-negative integer.",
    );
  }

  if (!Number.isInteger(page) || page < 0) {
    throw new TypeError(
      "The page index must be a non-negative integer.",
    );
  }

  const url = new URL("https://yandex.com/search/");

  // text is the query, lr is the region ID, and p is zero-indexed.
  url.searchParams.set("text", query.trim());
  url.searchParams.set("lr", String(region));
  url.searchParams.set("p", String(page));

  return url.toString();
}

function validateYandexBody($) {
  const resultList = $(RESULT_LIST_SELECTOR).first();

  // A known result container is a stronger success signal than HTTP 200.
  if (resultList.length > 0) {
    return resultList;
  }

  const hasCaptchaElement = CAPTCHA_SELECTORS.some(
    (selector) => $(selector).length > 0,
  );

  // Text checks provide a fallback when Yandex changes captcha classes.
  const pageTitle = normalizeText(
    $("title").text(),
  ).toLowerCase();

  const bodyPreview = normalizeText($("body").text())
    .slice(0, 2_000)
    .toLowerCase();

  if (
    hasCaptchaElement ||
    pageTitle.includes("captcha") ||
    bodyPreview.includes("smartcaptcha")
  ) {
    throw new YandexChallengeError(
      "Yandex returned a captcha page with HTTP 200. " +
        "Retry later or switch to stealth_proxy=true.",
    );
  }

  // The page is neither a recognized results page nor a known captcha.
  // This usually means Yandex returned another error page or changed its HTML.
  throw new YandexPageError(
    `The response contains neither ${RESULT_LIST_SELECTOR} ` +
      "nor a known captcha marker. " +
      "Yandex may have changed its HTML.",
  );
}

function parseYandexResults(html) {
  const $ = cheerio.load(html);
  const resultList = validateYandexBody($);
  const results = [];

  resultList
    .find(RESULT_ITEM_SELECTOR)
    .each((_, element) => {
      const item = $(element);

      // Use a fallback link selector to support more than one
      // Yandex result-card variant.
      const link = item
        .find(TITLE_LINK_SELECTOR)
        .first();

      const titleNode = item
        .find(TITLE_TEXT_SELECTOR)
        .first();

      const snippetNode = item
        .find(SNIPPET_SELECTOR)
        .first();

      // .text() combines text from nested spans and bold elements.
      const title = normalizeText(titleNode.text());
      const url = normalizeText(link.attr("href"));
      const snippet = normalizeText(snippetNode.text());

      // Skip widgets and incomplete cards that are not organic results.
      if (!title || !url) {
        return;
      }

      results.push({
        title,
        url,
        snippet,
      });
    });

  return results;
}

async function fetchYandexResults(
  query,
  region = 84,
  page = 0,
) {
  if (!API_KEY) {
    throw new Error(
      "Missing SCRAPINGBEE_API_KEY. " +
        "Set it before running the script.",
    );
  }

  const client = new ScrapingBeeClient(API_KEY);

  try {
    const response = await client.htmlApi({
      url: buildYandexUrl(query, region, page),
      params: {
        // Premium proxies are cheaper than Stealth and are a useful
        // starting point. Switch proxy modes if SmartCaptcha persists.
        premium_proxy: true,

        // Yandex web results are present in the initial HTML in this example,
        // so browser rendering is not required.
        render_js: false,
      },

      // Allow enough time for the proxy request without waiting forever.
      timeout: 120_000,

      // Retry transient API and network failures.
      retries: 2,
    });

    if (response.status !== 200) {
      throw new Error(
        `ScrapingBee returned HTTP ${response.status}.`,
      );
    }

    const html = decodeBody(response.data);

    return parseYandexResults(html);
  } catch (error) {
    // Preserve parser-specific errors so the caller can distinguish
    // a captcha from an unexpected Yandex page.
    if (
      error instanceof YandexChallengeError ||
      error instanceof YandexPageError
    ) {
      throw error;
    }

    // SDK request errors may include the HTTP response on error.response.
    if (error?.response) {
      const status =
        error.response.status ?? "unknown";

      const preview = decodeBody(
        error.response.data,
      )
        .slice(0, 500)
        .trim();

      throw new Error(
        `ScrapingBee returned HTTP ${status}: ` +
          (preview || "No response body"),
        { cause: error },
      );
    }

    throw new Error(
      `Request failed: ${error.message}`,
      { cause: error },
    );
  }
}

async function main() {
  const results = await fetchYandexResults(
    "python web scraping",
    84,
    0,
  );

  if (results.length === 0) {
    console.log(
      "Yandex returned no organic results for this query.",
    );
    return;
  }

  // Print only the first five organic results.
  console.dir(results.slice(0, 5), {
    depth: null,
  });
}

try {
  await main();
} catch (error) {
  console.error(`Error: ${error.message}`);
  process.exitCode = 1;
}

node yandex-scraper.mjs