1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
import asyncio
import argparse
import shutil
from scraper.spiders.page import PageScraper
def main():
parser = argparse.ArgumentParser(description="Scrape pages")
parser.add_argument("url", help="URL to scrape")
parser.add_argument("-o", "--output", default="data", help="Output directory")
parser.add_argument(
"-d", "--depth", type=int, required=True, help="Link traversal depth"
)
parser.add_argument(
"-c",
"--clean",
action="store_true",
help="Remove output directory before scraping",
)
parser.add_argument("-n", "--max-pages", type=int, help="Max pages to scrape")
parser.add_argument("-a", "--max-assets", type=int, help="Max assets to download")
parser.add_argument("-p", "--parallel", type=int, help="Number of parallel workers")
parser.add_argument(
"-t", "--timeout", type=int, help="Playwright default timeout in ms"
)
parser.add_argument("--page-timeout", type=int, help="Per-page timeout in seconds")
args = parser.parse_args()
if args.clean:
shutil.rmtree(args.output, ignore_errors=True)
scraper_kwargs = {
"url": args.url,
"output_dir": args.output,
"max_depth": args.depth,
}
if args.max_pages is not None:
scraper_kwargs["max_pages"] = args.max_pages
if args.max_assets is not None:
scraper_kwargs["max_assets"] = args.max_assets
if args.parallel is not None:
scraper_kwargs["max_parallel"] = args.parallel
if args.timeout is not None:
scraper_kwargs["timeout"] = args.timeout
if args.page_timeout is not None:
scraper_kwargs["page_timeout"] = args.page_timeout
scraper = PageScraper(**scraper_kwargs)
asyncio.run(scraper.run())
if __name__ == "__main__":
main()
|