1 files changed, 53 insertions, 0 deletions
diff --git a/scraper/__main__.py b/scraper/__main__.py
new file mode 100644
index 0000000..329b102
--- /dev/null
+++ b/scraper/__main__.py
@@ -0,0 +1,53 @@
+import asyncio
+import argparse
+import shutil
+from scraper.spiders.page import PageScraper
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Scrape pages")
+    parser.add_argument("url", help="URL to scrape")
+    parser.add_argument("-o", "--output", default="data", help="Output directory")
+    parser.add_argument(
+        "-d", "--depth", type=int, required=True, help="Link traversal depth"
+    )
+    parser.add_argument(
+        "-c",
+        "--clean",
+        action="store_true",
+        help="Remove output directory before scraping",
+    )
+    parser.add_argument("-n", "--max-pages", type=int, help="Max pages to scrape")
+    parser.add_argument("-a", "--max-assets", type=int, help="Max assets to download")
+    parser.add_argument("-p", "--parallel", type=int, help="Number of parallel workers")
+    parser.add_argument(
+        "-t", "--timeout", type=int, help="Playwright default timeout in ms"
+    )
+    parser.add_argument("--page-timeout", type=int, help="Per-page timeout in seconds")
+    args = parser.parse_args()
+
+    if args.clean:
+        shutil.rmtree(args.output, ignore_errors=True)
+
+    scraper_kwargs = {
+        "url": args.url,
+        "output_dir": args.output,
+        "max_depth": args.depth,
+    }
+    if args.max_pages is not None:
+        scraper_kwargs["max_pages"] = args.max_pages
+    if args.max_assets is not None:
+        scraper_kwargs["max_assets"] = args.max_assets
+    if args.parallel is not None:
+        scraper_kwargs["max_parallel"] = args.parallel
+    if args.timeout is not None:
+        scraper_kwargs["timeout"] = args.timeout
+    if args.page_timeout is not None:
+        scraper_kwargs["page_timeout"] = args.page_timeout
+
+    scraper = PageScraper(**scraper_kwargs)
+    asyncio.run(scraper.run())
+
+
+if __name__ == "__main__":
+    main()