From 61f0ac03db44542abcc5f2f362778101dcf0305c Mon Sep 17 00:00:00 2001
From: Ethan Werner <mail@ethanwerner.com>
Date: Wed, 12 Aug 2026 14:34:58 +0400
Subject: initial commit

---
 scraper/__main__.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 scraper/__main__.py

(limited to 'scraper/__main__.py')

diff --git a/scraper/__main__.py b/scraper/__main__.py
new file mode 100644
index 0000000..329b102
--- /dev/null
+++ b/scraper/__main__.py
@@ -0,0 +1,53 @@
+import asyncio
+import argparse
+import shutil
+from scraper.spiders.page import PageScraper
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Scrape pages")
+    parser.add_argument("url", help="URL to scrape")
+    parser.add_argument("-o", "--output", default="data", help="Output directory")
+    parser.add_argument(
+        "-d", "--depth", type=int, required=True, help="Link traversal depth"
+    )
+    parser.add_argument(
+        "-c",
+        "--clean",
+        action="store_true",
+        help="Remove output directory before scraping",
+    )
+    parser.add_argument("-n", "--max-pages", type=int, help="Max pages to scrape")
+    parser.add_argument("-a", "--max-assets", type=int, help="Max assets to download")
+    parser.add_argument("-p", "--parallel", type=int, help="Number of parallel workers")
+    parser.add_argument(
+        "-t", "--timeout", type=int, help="Playwright default timeout in ms"
+    )
+    parser.add_argument("--page-timeout", type=int, help="Per-page timeout in seconds")
+    args = parser.parse_args()
+
+    if args.clean:
+        shutil.rmtree(args.output, ignore_errors=True)
+
+    scraper_kwargs = {
+        "url": args.url,
+        "output_dir": args.output,
+        "max_depth": args.depth,
+    }
+    if args.max_pages is not None:
+        scraper_kwargs["max_pages"] = args.max_pages
+    if args.max_assets is not None:
+        scraper_kwargs["max_assets"] = args.max_assets
+    if args.parallel is not None:
+        scraper_kwargs["max_parallel"] = args.parallel
+    if args.timeout is not None:
+        scraper_kwargs["timeout"] = args.timeout
+    if args.page_timeout is not None:
+        scraper_kwargs["page_timeout"] = args.page_timeout
+
+    scraper = PageScraper(**scraper_kwargs)
+    asyncio.run(scraper.run())
+
+
+if __name__ == "__main__":
+    main()
-- 
cgit v1.3.1