diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000..8f8128dad91 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,134 @@ +# AGENTS.md + +Guidance for AI coding agents working in the SeleniumBase repository. + +## Project overview + +SeleniumBase is a Python framework for browser automation, end-to-end testing, +web scraping, and bot-detection avoidance. It wraps Selenium/WebDriver and adds: + +- **`BaseCase`**: a `unittest.TestCase` subclass used with `pytest` / `pynose` (`self.click(...)`, `self.type(...)`, etc.) +- **`SB()`** context manager and **`Driver()`** manager: for plain `python` scripts +- **UC Mode** (undetected-chromedriver) and **CDP Mode** (`sb_cdp`, `sb.activate_cdp_mode()`): stealth automation on Chromium browsers, including Stealthy Playwright integration +- CLI tools (`seleniumbase` / `sbase`), Recorder, Dashboard, Commander GUI, MasterQA, Presenter, ChartMaker, CasePlans, and `behave` (Gherkin) support + +Language: Python (~99%). License: MIT. Docs: https://seleniumbase.io + +## Repository layout + +| Path | What it is | +| --- | --- | +| `seleniumbase/` | The main package. Core library code lives here (fixtures/`BaseCase`, plugins/pytest plugin, config/settings, console scripts, CDP/UC code, drivers, etc.) | +| `sbase/` | Thin alias package so `sbase` works as a CLI/import name. Rarely needs edits | +| `examples/` | 150+ runnable examples and tests; also serves as the project's main regression suite | +| `examples/cdp_mode/` | CDP Mode, Pure CDP (`sb_cdp`), and Stealthy Playwright examples | +| `help_docs/` | Markdown docs (method summary, CDP methods, syntax formats, options, etc.) | +| `mkdocs_build/`, `mkdocs.yml` | Documentation site build for seleniumbase.io | +| `integrations/` | CI/CD and cloud setup examples (GitHub Actions, Jenkins, Azure, GCP) | +| `.github/` | Workflows and issue/PR templates | +| `pytest.ini`, `setup.cfg`, `pyproject.toml`, `setup.py`, `requirements.txt` | Packaging and test configuration | + +Confirm exact module locations by browsing the tree before editing. Do not assume file names from memory. + +## Setup + +```bash +git clone https://github.com/seleniumbase/SeleniumBase.git +cd SeleniumBase/ +python -m venv .venv && source .venv/bin/activate # a virtualenv is recommended +pip install -e . +seleniumbase --help # or: sbase --help (verifies the install) +``` + +- Webdrivers (e.g. `chromedriver`) are downloaded automatically on first use, so network access is needed the first time. Manual fetch: `sbase get chromedriver`. +- Google Chrome is the default browser. Unbranded Chromium and Chrome-for-Testing are auto-installed if needed. +- After pulling upstream changes, re-run `pip install -e .`. + +## Running tests + +Tests are run from inside `examples/` (which has its own `pytest.ini`): + +```bash +cd examples/ +pytest my_first_test.py # run one file +pytest test_demo_site.py::DemoSiteTests::test_demo_site # run one test (FILE::CLASS::METHOD) +pytest --co -q # dry run: list what would be collected +pytest my_first_test.py --headless # no visible browser (good for CI/sandboxes) +pytest my_first_test.py --demo # slow, highlighted actions for debugging +pytest test_suite.py --rs --headless # reuse one browser session +``` + +Useful options: `-x` (stop on first failure), `-n=NUM` (parallel), `--browser=firefox|edge|...`, +`--uc` (UC Mode), `--html=report.html`, `--dashboard`, `--reruns=N`, `--pdb` / `--trace` (never in CI). + +Things to know: + +- **Most tests need a real browser and live websites.** They can fail from network problems, site changes, or bot-detection rather than from your changes. Run the smallest relevant test(s), not the whole `examples/` folder, and use `--headless` when no display is available. +- `test_fail.py` and parts of `test_suite.py` **fail on purpose** to demonstrate logging. Do not "fix" them. +- Files named `raw_*.py` are run with plain `python`, not `pytest`. +- Tests using the `sb` pytest fixture only work with `pytest`. +- Failure logs and screenshots go to `latest_logs/` (older ones to `archived_logs/`). Do not commit these. +- Test discovery: files matching `test_*.py` or `*_test.py`; methods starting with `test_`. + +## Code style and conventions + +- Match the surrounding code. This is a large, long-lived codebase; avoid drive-by reformatting or unrelated refactors. +- Follow the repo's flake8/PEP 8 configuration (see `setup.cfg` / CI config). Python lines follow the default flake8 settings, e.g. the 79-character limit. +- Keep Python version compatibility with what `setup.py` / `pyproject.toml` declare as supported. Don't use syntax newer than the minimum supported version. +- Public API naming: prefer the current names (`goto`, not `open`; `sb.goto(...)` in CDP Mode). Older aliases are kept for backwards compatibility. Don't remove or rename public methods without a strong reason. +- Selectors: CSS selectors by default; XPath is auto-detected. `:contains("text")` is supported in SeleniumBase selectors. +- Prefer built-in SeleniumBase methods (automatic waits, clean errors) over raw `self.driver` calls. Use raw WebDriver only when no SeleniumBase method exists. +- Avoid `time.sleep()` in library code; use the framework's wait methods and timeouts. `self.sleep()` is acceptable in demo/example scripts. +- Do not add heavy new dependencies. Any dependency change must be reflected in `requirements.txt`, `setup.py`, and `pyproject.toml` as applicable. + +## Making changes + +**Adding or changing a public method** + +1. Implement it in the relevant module under `seleniumbase/`. Where the method exists in more than one API surface (e.g. `BaseCase`, `SB`/`Driver`, CDP Mode `sb_cdp`), keep them consistent. +2. Add or update an example/test in `examples/` (CDP examples in `examples/cdp_mode/`). +3. Update the relevant doc in `help_docs/` (e.g. `method_summary.md`, `cdp_mode_methods.md`), and `README.md` only if it's a headline feature. +4. If the change is user-visible, note it in `CHANGELOG.md` only if maintainers' convention for the release requires it. Check recent entries first. + +**Adding a command-line option** + +Options are defined in the pytest plugin (`seleniumbase/plugins/pytest_plugin.py`) and mirrored elsewhere (pynose plugin, `behave` support, `sb_manager` / `SB()` args, docs in `help_docs/customizing_test_runs.md`, and the `options` console script). Update every place the option surfaces. + +**Docs** + +Docs are Markdown in `help_docs/` and `examples/**/ReadMe.md`. Keep links relative to GitHub `master` consistent with existing ones. + +## Stealth / bot-detection code: extra care + +UC Mode and CDP Mode are among the project's most actively developed and most fragile areas. + +- Small changes to launch flags, driver patching, timing, or CDP calls can silently break stealth on real sites. Test against the bot-detection examples (e.g. `examples/cdp_mode/raw_cdp_browserscan.py`, `raw_gitlab.py`) in **headed** mode where possible. +- Don't add code whose purpose is to help users abuse, attack, or harm third-party sites. Legitimate automation, testing, and scraping use cases are the scope. +- Be conservative with anything touching CAPTCHA handling (`solve_captcha`), proxies, and user-agent handling. + +## Safety and hygiene + +- **Never commit secrets**: proxy credentials, API keys, 2FA keys, DB or S3 credentials (these can live in `settings.py` / custom settings files). Use placeholders. +- Don't commit generated artifacts: `latest_logs/`, `archived_logs/`, `downloaded_files/`, `dashboard.html`, `report.html`, `__pycache__/`, downloaded drivers in `seleniumbase/drivers/`. +- Don't hit third-party sites in a loop or at load. Keep example scripts polite and short. +- Avoid destructive shell commands outside the repo directory. Browser profiles and driver folders can be large; clean up test-created folders you generate. + +## Pull request checklist + +- [ ] Change is focused and minimal; no unrelated formatting churn +- [ ] Relevant examples run locally (`pytest --headless` or plain `python` for `raw_*.py`) +- [ ] flake8-clean for touched files +- [ ] New/changed public behavior has an example and doc update +- [ ] No secrets, logs, or generated files included +- [ ] See `CONTRIBUTING.md` and `CODE_OF_CONDUCT.md` for project policy + +## Useful references + +- Docs site: https://seleniumbase.io +- Method summary: `help_docs/method_summary.md` +- CDP Mode methods: `help_docs/cdp_mode_methods.md` +- CDP Mode guide: `examples/cdp_mode/ReadMe.md` +- Syntax formats (BaseCase / SB / Driver / sb fixture / sb_cdp): `help_docs/syntax_formats.md` +- CLI options: `help_docs/customizing_test_runs.md` +- UC Mode: `help_docs/uc_mode.md` +- Console scripts: `seleniumbase/console_scripts/ReadMe.md` diff --git a/SKILLS.md b/SKILLS.md new file mode 100644 index 00000000000..478eb4830a2 --- /dev/null +++ b/SKILLS.md @@ -0,0 +1,275 @@ +# SKILLS.md + +Task-oriented playbooks for AI agents using or working on **SeleniumBase**, the Python framework for browser automation, E2E testing, scraping, and stealth. Each skill says when to use it, what to do, and what to avoid. + +> Companion to `AGENTS.md` (repo conventions, layout, contribution rules). This file is about *doing things with* SeleniumBase. Method names below are taken from the README. For the full API see `help_docs/method_summary.md` (BaseCase/SB) and `help_docs/cdp_mode_methods.md` (CDP Mode), and verify a method exists there before using it. + +## Skill index + +1. [Choose the right syntax format](#1-choose-the-right-syntax-format) +2. [Write a pytest E2E test (BaseCase)](#2-write-a-pytest-e2e-test-basecase) +3. [Write a standalone automation script (SB / Driver)](#3-write-a-standalone-automation-script-sb--driver) +4. [Scrape or automate with stealth (Pure CDP Mode)](#4-scrape-or-automate-with-stealth-pure-cdp-mode) +5. [Handle bot-detection and CAPTCHAs (UC + CDP Mode)](#5-handle-bot-detection-and-captchas-uc--cdp-mode) +6. [Use Playwright through a stealthy browser](#6-use-playwright-through-a-stealthy-browser) +7. [Handle iframes, tabs, alerts, and JavaScript](#7-handle-iframes-tabs-alerts-and-javascript) +8. [Debug a failing or flaky test](#8-debug-a-failing-or-flaky-test) +9. [Run in CI, headless, or in parallel](#9-run-in-ci-headless-or-in-parallel) +10. [Scaffold a new test project](#10-scaffold-a-new-test-project) +11. [Generate tests with the Recorder](#11-generate-tests-with-the-recorder) +12. [Produce reports and dashboards](#12-produce-reports-and-dashboards) +13. [Migrate raw Selenium code](#13-migrate-raw-selenium-code) +14. [Configure proxies, user agents, and browsers](#14-configure-proxies-user-agents-and-browsers) + +--- + +## 1. Choose the right syntax format + +**Use when:** starting any new script or test. + +| Need | Use | Run with | +| --- | --- | --- | +| Structured tests, pytest features, reports, CLI options | `BaseCase` class (`self.click(...)`) | `pytest` / `pynose` | +| Tests written as pytest functions | `sb` pytest fixture | `pytest` only | +| One-off automation or scraping script | `with SB(...) as sb:` | `python` | +| Drop-in improved Selenium driver | `Driver()` | `python` | +| Maximum stealth, no WebDriver | `sb_cdp.Chrome()` (Pure CDP Mode) | `python` | +| Gherkin/BDD | `behave` features | `behave` | + +**Rule of thumb:** tests belong in `BaseCase` under pytest; scripts belong in `SB()` or `sb_cdp`; anything that must evade bot-detection should start with CDP Mode. See `help_docs/syntax_formats.md`. + +## 2. Write a pytest E2E test (BaseCase) + +**Use when:** verifying behavior of a web app. + +```python +from seleniumbase import BaseCase +BaseCase.main(__name__, __file__) # lets `python file.py` invoke pytest + +class LoginTests(BaseCase): + def test_login(self): + self.goto("https://www.saucedemo.com") + self.type("#user-name", "standard_user") + self.type("#password", "secret_sauce\n") # "\n" presses Enter + self.assert_element("div.inventory_list") + self.assert_exact_text("Products", "span.title") +``` + +**Steps** +1. Name the file `test_*.py` or `*_test.py`, and methods `test_*`. The class name can be anything. +2. Navigate with `self.goto(url)`. Interact with `self.click`, `self.type`, `self.select_option_by_text`, `self.hover_and_click`, `self.drag_and_drop`. +3. Verify with `assert_element`, `assert_text`, `assert_exact_text`, `assert_title`, `assert_downloaded_file`, `assert_no_404_errors`, `assert_no_js_errors`. +4. Run: `pytest test_login.py` (add `--headless` on servers). + +**Guidelines** +- Selectors are CSS by default (XPath auto-detected). Use `:contains("text")` and `[attr*="partial"]` when handy. +- SeleniumBase methods wait automatically (default timeouts). Pass `timeout=N` to override. Do **not** add `time.sleep()` to fix timing. +- Use `self.type(sel, text)`, not `add_text`/`send_keys`, unless you deliberately don't want the field cleared. +- Batch several checks on one page with **deferred asserts** (`deferred_assert_element`, `deferred_assert_text`, ..., then `self.process_deferred_asserts()`). Call it before navigating to a new page. +- For conditionals use `is_element_visible`, `is_element_present`, `is_text_visible`, `is_link_text_visible`. + +## 3. Write a standalone automation script (SB / Driver) + +**Use when:** you need a script, not a test suite. + +```python +from seleniumbase import SB + +with SB(test=True) as sb: # add uc=True for UC Mode, headless=True for headless + sb.goto("seleniumbase.io/simple/login") + sb.type("#username", "demo_user") + sb.type("#password", "secret_pass") + sb.click('a:contains("Sign in")') + sb.assert_exact_text("Welcome!", "h1") +``` + +`Driver()` gives an improved Selenium driver (same helper methods). Always close it: + +```python +from seleniumbase import Driver +driver = Driver() +try: + driver.goto("https://example.com") +finally: + driver.quit() +``` + +Raw Selenium is available via `sb.driver` / `self.driver`. Run with plain `python script.py`. Name such files `raw_*.py` in `examples/` so pytest doesn't collect them. + +## 4. Scrape or automate with stealth (Pure CDP Mode) + +**Use when:** scraping or automating sites that fingerprint WebDriver. Requires a Chromium-based browser. + +```python +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome() # options seen in docs: incognito=True, guest=True, +sb.goto("https://news.ycombinator.com/submitted?id=seleniumbase") # locale="en", ad_block=True, +for el in sb.find_elements("span.titleline > a"): # use_chromium=True, cft=True + print("* " + el.text) +sb.quit() +``` + +**Steps** +1. Create the browser with `sb_cdp.Chrome(...)`; navigate with `sb.goto(url)`. +2. Locate with `find_elements`, act with `click`/`type`, and check with `assert_element`/`assert_text`. Confirm names in `help_docs/cdp_mode_methods.md`. +3. Use `sb.highlight(...)`/`sb.flash(...)` only for demos; skip them in production scrapers. +4. Always call `sb.quit()` (use `try/finally`). + +**Choosing a browser:** Google Chrome is the default. Alternatives via method args (`cft=True`, `use_chromium=True`, `browser="edge"`, `browser="brave"`) or CLI flags (`--cft`, `--chromium`, `--edge`, `--brave`). Only unbranded Chromium and Chrome-for-Testing auto-install. + +**Avoid:** mixing this with Firefox/Safari (CDP Mode is Chromium-only). + +## 5. Handle bot-detection and CAPTCHAs (UC + CDP Mode) + +**Use when:** a page shows a Cloudflare-style challenge or Turnstile, or blocks a normal WebDriver session. + +```python +from seleniumbase import SB + +with SB(uc=True, test=True, locale="en") as sb: + sb.activate_cdp_mode("https://gitlab.com/users/sign_in") + sb.sleep(2) + sb.solve_captcha() # does nothing if no CAPTCHA is present + sb.assert_element('label[for="user_login"]') +``` + +Or in Pure CDP Mode: `sb = sb_cdp.Chrome(incognito=True); sb.goto(url); sb.sleep(2); sb.solve_captcha()`. + +**Guidance** +- Prefer **CDP Mode** (via `activate_cdp_mode` or `sb_cdp`) for maximum stealth. UC Mode alone is the older path. +- Use `sb.click_if_visible(selector)` for optional consent banners. +- Verify stealth against test pages such as browserscan.net/bot-detection or bot.sannysoft.com. Run **headed** when investigating detection problems; headless can change your fingerprint. +- Keep this to legitimate testing, monitoring, and scraping. Respect site terms and robots policies, and don't build tooling meant to attack or abuse sites. + +## 6. Use Playwright through a stealthy browser + +**Use when:** you already have Playwright code and want SeleniumBase's stealth. + +```python +from playwright.sync_api import sync_playwright +from seleniumbase import sb_cdp + +sb = sb_cdp.Chrome(guest=True) +endpoint_url = sb.get_endpoint_url() + +with sync_playwright() as p: + browser = p.chromium.connect_over_cdp(endpoint_url) + page = browser.contexts[0].pages[0] + page.goto("https://bot.sannysoft.com/") +``` + +Install both packages: `pip install seleniumbase playwright`. Reuse the existing context and page (`contexts[0].pages[0]`) rather than creating new ones. Examples: `examples/cdp_mode/playwright/`. + +## 7. Handle iframes, tabs, alerts, and JavaScript + +- **iframes:** `self.switch_to_frame("iframe")` → act → `self.switch_to_parent_frame()`; or `with self.frame_switch("iframe"):` (nestable). Exit all with `self.switch_to_default_content()`. +- **Tabs/windows:** SeleniumBase auto-switches to new tabs that don't open `about:blank`. Otherwise `self.switch_to_window(1)`; back with `self.switch_to_default_window()`. Use `open_new_window()` to create one. +- **Alerts:** `self.accept_alert()` / `self.dismiss_alert()`. If `self.click()` dismisses the pop-up itself (it waits for `readyState`), use `self.find_element(SEL).click()` and then `accept_alert()`. +- **JavaScript:** `self.execute_script(...)`; call `self.activate_jquery()` first if you need jQuery on a page without it. On pages with a strict CSP, add `--disable-csp`. +- **Raw WebDriver escape hatch:** `self.driver.`. + +## 8. Debug a failing or flaky test + +Work through in order: + +1. **Reproduce narrowly:** `pytest file.py::Class::test_name` (add `-x -v`). +2. **Watch it:** `--demo` (slows and highlights actions) or run headed. +3. **Read the evidence:** on failure, screenshots and logs are saved in `./latest_logs/`. +4. **Pause interactively:** `--pdb` (post-mortem, browser stays open) or `--trace` (debug from test start), or drop in `breakpoint()`. Never use these in CI. +5. **Check the selector:** inspect the page, prefer stable attributes over long chains; use `wait_for_element` / `assert_element` with a larger `timeout=`. +6. **Global timing slack:** `--timeout-multiplier=2`, or `--pls=eager` for slow pages. +7. **Retry as a last resort:** `--reruns=1 --reruns-delay=1`, or `@retry_on_exception()`. Retries hide bugs, so fix the root cause first. +8. **Stealth-related failures:** switch to CDP Mode (skill 5) and test headed. + +`test_fail.py` is meant to fail. It's a logging demo, not a bug. + +## 9. Run in CI, headless, or in parallel + +```bash +pytest tests/ --headless --rs --html=report.html --junit-xml=report.xml +pytest tests/ -n=4 --headless # parallel across 4 workers +pytest tests/ --xvfb # Linux virtual display when a headed browser is needed +``` + +- Linux runs headless by default; use `--headed` to force a GUI. `--headless2` supports extensions. +- `--rs` reuses one browser session for all tests (faster, but state leaks between tests; use `--crumbs` to clear cookies between them). +- Set `--driver-version=VER` to pin the driver. +- No `--pdb`, `--trace`, or `--show-report` in CI. +- Cache/allow downloads of browsers and drivers on first run. Offline runs work only if drivers were previously downloaded. +- Ready-made CI examples live in `integrations/` (GitHub Actions, Jenkins, Azure, Google Cloud). `sbase mkdir DIR --gha` adds a GitHub Actions workflow. +- Scale out with Selenium Grid: `--server=HOST --port=PORT` (see `seleniumbase/utilities/selenium_grid/`). + +## 10. Scaffold a new test project + +```bash +sbase mkdir ui_tests # config files + sample tests + boilerplates +sbase mkdir ui_tests --basic # only pytest.ini, setup.cfg, requirements.txt, __init__.py +sbase mkdir ui_tests --gha # also adds a GitHub Actions workflow +``` + +- `pytest.ini` is the most important file (defaults for pytest); `setup.cfg` is for `pynose`. +- Each test folder needs an (empty) `__init__.py` so tests can import siblings. +- `sbase mkfile FILE.py` creates a single test file. Boilerplates for page objects and the `sb` fixture are included by the full scaffold. + +## 11. Generate tests with the Recorder + +```bash +sbase recorder # desktop app +sbase mkrec test_new.py # (alias: codegen) start recording to a file +pytest test_new.py --rec # ...or use the pytest options below +``` + +Recorder pytest flags include `--recorder`, `--rec-sb-mgr` (emit `SB()` code), `--rec-sb-cdp` (emit `sb_cdp` code), `--rec-behave`, and `--rec-print`. Treat recorded output as a draft: replace brittle selectors, remove needless `sleep` calls, and add real assertions. See `help_docs/recorder_mode.md`. + +## 12. Produce reports and dashboards + +| Goal | Command | +| --- | --- | +| Live dashboard (`dashboard.html`) | `pytest --dashboard --rs --headless` | +| pytest HTML report | `pytest --html=report.html` | +| Dashboard folded into the HTML report | `pytest --dashboard --html=report.html` | +| JUnit XML for CI | `pytest --junit-xml=report.xml` | +| pynose report | `pynose test_suite.py --report` (`--show-report` only locally) | +| behave | `behave features/ -D dashboard -D headless` | +| Allure | `pip install allure-pytest` (not bundled), then `pytest --alluredir=allure_results` | + +Serve the dashboard locally: `python -m http.server 1948`, then open `http://localhost:1948/dashboard.html`. + +## 13. Migrate raw Selenium code + +| Raw Selenium | SeleniumBase | +| --- | --- | +| `WebDriverWait(...).until(EC.element_to_be_clickable(...)).click()` | `self.click(sel, timeout=10)` | +| `driver.find_element(By.CSS_SELECTOR, s).clear(); .send_keys(t)` | `self.type(s, t)` | +| Manual waits + `assert el.is_displayed()` | `self.assert_element(s)` | +| `driver.get(url)` | `self.goto(url)` | +| Hand-rolled argparse for browser choice | `--browser=...`, `--headless`, etc. | + +Migration examples: `examples/migration/raw_selenium/`. CLI helper: `sbase convert WEBDRIVER_UNITTEST_FILE.py`. Keep raw `self.driver` calls only where no SeleniumBase equivalent exists. + +## 14. Configure proxies, user agents, and browsers + +```bash +pytest t.py --proxy=IP:PORT +pytest t.py --proxy=USER:PASS@IP:PORT # authenticated (Chromium only) +pytest t.py --proxy="socks5://IP:PORT" # socks4/socks5 supported +pytest t.py --proxy=proxy1 # key from seleniumbase/config/proxy_list.py +pytest t.py --agent="USER AGENT STRING" # Chromium and Firefox +pytest t.py --locale=en --mobile # locale, mobile emulation +pytest t.py --chrome | --edge | --firefox | --safari | --brave | --chromium | --cft +``` + +- Per-run overrides of defaults (timeouts, credentials) go in a custom settings file: `--settings-file=custom_settings.py` (see `examples/custom_settings.py`). +- Pass test data with `--data`, `--var1..3`, `--variables`, `--env`, `--account`; read them via `self.data`, `self.var1`, `self.env`, etc. +- **Never hard-code real credentials.** Use environment variables or an untracked settings file. + +--- + +## Cross-cutting rules for agents + +- **Prefer the smallest change and the smallest test run.** Browser tests are slow and touch live sites. +- **Look up before you write.** Confirm a method or flag in `help_docs/` or the source (`seleniumbase/plugins/pytest_plugin.py` defines pytest options) instead of guessing. +- **Match runner to format.** `sb` fixture → `pytest` only; `raw_*.py` → `python`; BDD → `behave`. +- **Clean up.** Close browsers (`quit()`, context managers, `finally`), and don't commit `latest_logs/`, `archived_logs/`, `downloaded_files/`, or report files. +- **Be a good web citizen.** Rate-limit, respect site terms, and use stealth features only for legitimate automation. diff --git a/mcp_servers/README.md b/mcp_servers/README.md index da55a5a9009..00123b644e4 100644 --- a/mcp_servers/README.md +++ b/mcp_servers/README.md @@ -4,11 +4,11 @@ ### The [SeleniumBase](https://github.com/seleniumbase/SeleniumBase) MCP server provides stealthy browser automation over the [Model Context Protocol](https://modelcontextprotocol.io) for MCP clients. -This server, (located in `server.py`), uses SeleniumBase's [Pure CDP Mode](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/cdp_mode_methods.md) (`seleniumbase.sb_cdp.Chrome`), where the browser is driven entirely over the Chrome DevTools Protocol, and there is no WebDriver in the loop at all, which makes it SeleniumBase's stealthiest mode. CAPTCHA-solving is available through `solve_captcha()`. +This server (located in `server.py`) uses SeleniumBase's [Pure CDP Mode](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/cdp_mode_methods.md) (`seleniumbase.sb_cdp.Chrome`), where the browser is driven entirely over the Chrome DevTools Protocol, and there is no WebDriver in the loop at all, which makes it SeleniumBase's stealthiest mode. CAPTCHA-solving is available through `solve_captcha()`. -Other SeleniumBase automation styles, (such as `Driver()` and `SB()`), have their own MCP servers in [seleniumbase/seleniumbase-mcp](https://github.com/seleniumbase/seleniumbase-mcp). +Other SeleniumBase automation styles (such as `Driver()` and `SB()`) have their own MCP servers in [seleniumbase/seleniumbase-mcp](https://github.com/seleniumbase/seleniumbase-mcp). -`headless` defaults to `None` in `start_browser`, which resolves to headless on Linux (typical for server/container environments) and headed on Windows/macOS. Pass `headless=True` or `headless=False` explicitly to override this for any OS; headless mode may be less stealthy. +`headless` defaults to `None` in `start_browser`, which resolves to headless mode on Linux (typical for server/container environments) and headed mode on Windows/macOS. Pass `headless=True` or `headless=False` explicitly to override this for any OS; headless mode may be less stealthy. ## 1. Install @@ -20,7 +20,7 @@ There are two ways to get the `seleniumbase-mcp` command: pip install "seleniumbase[mcp]" ``` -This installs `seleniumbase` from PyPI along with the `mcp[cli]` extra, and registers a `seleniumbase-mcp` console-script command. Your MCP client config can be as simple as `{"command": "seleniumbase-mcp"}` (see step 3's Option A). +This installs `seleniumbase` from PyPI along with the `mcp[cli]` extra, and registers a `seleniumbase-mcp` console-script command. Your MCP client config can be as simple as `{"command": "seleniumbase-mcp"}` (see Step 3's Option A). **If you're working from a `git clone` of this repo (instead of a PyPI install):** @@ -37,7 +37,7 @@ uv sync Pure CDP Mode doesn't use WebDriver, so no `chromedriver` download is needed... just a working Chrome/Chromium install. -(If you don't want to use `uv`, you can use a standard Python virtual environment instead: `python3 -m venv venv && pip install -r requirements.txt` works too. `requirements.txt` installs the local SeleniumBase checkout via `-e .` the same way. Substitute `python server.py` for `uv run seleniumbase-mcp` everywhere below, and use absolute `venv/bin/python` + script path in your MCP client config instead of the path-free options.) +If you don't want to use `uv`, you can use a standard Python virtual environment instead: `python3 -m venv venv && pip install -r requirements.txt`, which installs the local SeleniumBase checkout via `-e .`. Then substitute `python server.py` for `uv run seleniumbase-mcp` everywhere below, and use absolute `venv/bin/python` + script path in your MCP client config instead of the path-free options. Without `uv`, you can directly run the `seleniumbase-mcp` command to start the server after `pip`-installing `seleniumbase`. @@ -53,11 +53,11 @@ Or if using `uv`: uv run mcp dev server.py ``` -That opens the MCP Inspector, where you can test commands ("Tools"). Use Ctrl+C to exit from the terminal. Next step is wiring it into a client harness. +That opens the MCP Inspector, where you can test commands ("Tools"). Use Ctrl+C to exit the MCP Inspector from the terminal. The next step is wiring it into a client harness. ## 3. Connect it to Claude Desktop -Claude Desktop doesn't run from a "project" directory the way Claude Code does, so a bare `uv run seleniumbase-mcp` isn't guaranteed to find this folder. Two ways to get a stable config: +Claude Desktop doesn't run from a "project" directory the way Claude Code does, so a bare `uv run seleniumbase-mcp` isn't guaranteed to find this folder. There are two ways to create a stable configuration: **Option A — global install (recommended, zero paths anywhere):** @@ -75,7 +75,7 @@ This puts `seleniumbase-mcp` on your `PATH` permanently (run `uv tool ensurepath } ``` -Note this bakes in the location of the SeleniumBase checkout at install time (since `seleniumbase` resolves to `../` via the editable path source). If you move or delete this clone, re-run `uv tool install .` from its new location. +Note that this bakes in the location of the SeleniumBase checkout at install time (since `seleniumbase` resolves to `../` via the editable path source). If you move or delete this clone, re-run `uv tool install .` from its new location. **Option B — point `uv` at this folder directly (one absolute path, but no venv/interpreter path to track down, and no separate install step):** @@ -95,7 +95,7 @@ The location of `claude_desktop_config.json` depends on your system: - macOS: `~/Library/Application Support/Claude/claude_desktop_config.json` - Windows: `%APPDATA%\Claude\claude_desktop_config.json` -Restart Claude Desktop. You should see a 🔨 tools icon indicating the server connected, with the following MCP tools available through the tools interface: +Restart Claude Desktop. You should see a 🔨 tools icon indicating the server connected, with the following MCP tools available through the interface: * `start_browser` * `close_browser` @@ -164,7 +164,7 @@ Most tools accept a `selector` argument. Behavior varies slightly by tool, so ch ## Tools exposed -Tools here are grouped around a shared `selector` convention. Several near-identical one-off tools (e.g. separate click/hover/drag/wait/cookie/storage variants) have been consolidated into a single tool with a `mode`/`action`/`state`/`check` parameter, so there are fewer near-neighbor tools to disambiguate between while every underlying capability stays available. Tool names also follow a verb+object convention (`click_element`, `focus_element`, `scroll_page`, `save_page`, `open_url`) rather than bare verbs, so a tool's name signals what it acts on without needing to read its description. +Tools here are grouped around a shared `selector` convention. Several near-identical one-off tools (e.g. separate click/hover/drag/wait/cookie/storage variants) have been consolidated into a single tool with a `mode`/`action`/`state`/`check` parameter, so there are fewer near-neighbor tools to disambiguate between while keeping every underlying capability available. Tool names also follow a verb+object convention (`click_element`, `focus_element`, `scroll_page`, `save_page`, `open_url`) rather than bare verbs, so a tool's name signals what it acts on without needing to read its description. | Group | Tool(s) | | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -177,7 +177,7 @@ Tools here are grouped around a shared `selector` convention. Several near-ident | Cookies & storage | `manage_cookies(action: get_all/clear/save/load)`, `manage_storage(storage: local/session, action: get/set)` | | Scrolling | `scroll_page(direction: up/down/top/bottom, amount)` | | Windows & tabs | `manage_window(action: get_rect/set_rect/maximize/minimize)`, `manage_tabs(action: list_tabs/open_new_tab/switch_to_tab/switch_to_newest_tab/close_active_tab)` | -| Captcha | `solve_captcha` | +| CAPTCHA | `solve_captcha` | | Output & misc | `save_page(format: screenshot/html/pdf)`, `run_javascript` | ## Design notes / things to adapt for your use case @@ -188,7 +188,7 @@ Tools here are grouped around a shared `selector` convention. Several near-ident - **`start_browser` retries once before failing.** If the first launch attempt raises, it's retried once automatically before returning an error. This was added after seeing occasional first-attempt failures when testing against Glama's MCP Inspector; it costs nothing on the common case where the first launch already succeeds. -- **Two error-handling paths, by design.** Most failures (a selector isn't found, an assertion fails, an invalid `action`/`mode`/`check` value is passed) are caught by the `handle_sb_errors` decorator and returned as a descriptive string, e.g. `Error in click_element: NoSuchElementException - ...`, so the calling agent can read the failure and self-correct. There's one deliberate exception: calling any tool other than `start_browser`/`close_browser` when no browser session is running raises `ToolError` (via the shared `_get_sb()` helper) instead of returning a string. `handle_sb_errors` explicitly re-raises `ToolError` rather than catching it, so this surfaces to the MCP client as a real tool-call error (`is_error=True`), not as ordinary text the agent has to pattern-match on. `start_browser` and `close_browser` handle their own lifecycle errors directly (e.g. "already running", a failed `quit()`) and also return strings rather than raising. +- **Two error-handling paths, by design.** Most failures (an element matching a selector isn't found, an assertion fails, an invalid `action`/`mode`/`check` value is passed) are caught by the `handle_sb_errors` decorator and returned as a descriptive string, e.g. `Error in click_element: NoSuchElementException - ...`, so the calling agent can read the failure and self-correct. There's one deliberate exception: calling any tool other than `start_browser`/`close_browser` when no browser session is running raises `ToolError` (via the shared `_get_sb()` helper) instead of returning a string. `handle_sb_errors` explicitly re-raises `ToolError` rather than catching it, so this surfaces to the MCP client as a real tool-call error (`is_error=True`), not as ordinary text the agent has to pattern-match on. `start_browser` and `close_browser` handle their own lifecycle errors directly (e.g. "already running", a failed `quit()`) and also return strings rather than raising. - **`find_elements` catches its own lookup failures.** Its default `timeout` is 0.5 seconds (not 5, unlike most other tools here). A failed or empty lookup never raises: no matches returns `{"count": 0, "matches": []}`, and an actual lookup error (e.g. an unsupported selector) returns `{"count": 0, "matches": [], "error": "
"}` — the error lives inside the returned dict rather than surfacing as a top-level string from `handle_sb_errors`. Pass a longer `timeout` explicitly if the elements you're looking for may still be loading. @@ -204,4 +204,4 @@ Tools here are grouped around a shared `selector` convention. Several near-ident ## Extending -Adding a tool is just adding a `@mcp.tool()`-decorated function (wrapped in `handle_sb_errors`) that calls the matching `sb_cdp.Chrome` method — SeleniumBase has methods for file uploads, network conditions, and more that aren't wrapped above yet. +Adding a tool is just adding a `@mcp.tool()`-decorated function (wrapped in `handle_sb_errors`) that calls the matching `sb_cdp.Chrome` method. SeleniumBase has methods for file uploads, network conditions, and more that aren't wrapped above yet. diff --git a/mcp_servers/pyproject.toml b/mcp_servers/pyproject.toml index 9827f9ea7e9..f80f112c63d 100644 --- a/mcp_servers/pyproject.toml +++ b/mcp_servers/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "seleniumbase-mcp" -version = "1.3.11dev0" +version = "1.3.12dev0" description = "MCP server exposing SeleniumBase CDP Mode as tools for MCP clients." readme = "README.md" requires-python = ">=3.10" diff --git a/requirements.txt b/requirements.txt index eec5116138e..8b94abcc462 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ certifi>=2026.7.22 exceptiongroup>=1.3.1 websockets~=16.1.1;python_version=="3.10" websockets>=16.1.1;python_version>="3.11" -filelock>=4.0.3 +filelock>=4.0.7 fasteners>=0.20 mycdp>=1.4.0 pynose>=1.5.5 @@ -64,7 +64,7 @@ rich>=15.0.0,<16 # --- Testing Requirements --- # # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) -coverage>=7.16.1 +coverage>=7.16.2 pytest-cov>=7.1.0 flake8==7.4.1 mccabe==0.7.0 diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 5844a9a2d26..2b67f23fa12 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.54.12" +__version__ = "4.54.13" diff --git a/server.json b/server.json index 9f07543506f..5e198b1918b 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.seleniumbase/seleniumbase", "title": "SeleniumBase MCP", "description": "Stealthy browser automation, testing, and web-scraping via CDP Mode.", - "version": "4.54.12", + "version": "4.54.13", "repository": { "url": "https://github.com/seleniumbase/SeleniumBase", "source": "github" @@ -13,7 +13,7 @@ { "registryType": "pypi", "identifier": "seleniumbase", - "version": "4.54.12", + "version": "4.54.13", "transport": { "type": "stdio" }, diff --git a/setup.py b/setup.py index de7bcb4cf5d..d808da65d80 100755 --- a/setup.py +++ b/setup.py @@ -173,7 +173,7 @@ 'exceptiongroup>=1.3.1', 'websockets~=16.1.1;python_version=="3.10"', 'websockets>=16.1.1;python_version>="3.11"', - 'filelock>=4.0.3', + 'filelock>=4.0.7', 'fasteners>=0.20', 'mycdp>=1.4.0', 'pynose>=1.5.5', @@ -239,7 +239,7 @@ # pip install -e .[coverage] # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ - 'coverage>=7.16.1', + 'coverage>=7.16.2', 'pytest-cov>=7.1.0', ], # pip install -e .[flake8] @@ -252,9 +252,10 @@ ], # pip install -e .[mcp] # (Adds the "seleniumbase-mcp" console script: An MCP server that - # exposes SeleniumBase's Pure CDP Mode as tools for MCP clients) + # exposes SeleniumBase's Pure CDP Mode as tools for MCP clients. + # Example usage: `mcp dev server.py`) "mcp": [ - "mcp[cli]>=2.1.1,<3.0.0", + "mcp[cli]>=2.2.0,<3.0.0", ], # pip install -e .[mss] # (An optional library for tile_windows() in CDP Mode.) @@ -299,10 +300,9 @@ 'PyAutoGUI>=0.9.54;platform_system!="Linux"', ], # pip install -e .[uv] - # Required for local MCP server debugging with: - # mcp dev server.py + # (For the MCP integration and more.) "uv": [ - "uv>=0.12.18" + "uv>=0.12.21" ], }, packages=[