initial commitHEADmaster

10 files changed, 1653 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..94fe76f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+.venv/
+data/
+__pycache__/
+*.pyc
+**/.content_hash
+.ruff_cache/
+*.code-workspace
diff --git a/embedding/embedder.py b/embedding/embedder.py
new file mode 100644
index 0000000..db353ab
--- /dev/null
+++ b/embedding/embedder.py
@@ -0,0 +1,124 @@
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from PIL import Image
+from transformers import (
+    AutoModel,
+    BertTokenizer,
+    SiglipModel,
+    SiglipProcessor,
+    WhisperModel,
+    WhisperFeatureExtractor,
+)
+
+
+class CompactEmbedder(nn.Module):
+    def __init__(self, local_dir="./multi-modal-embed-small"):
+        local_dir = os.path.abspath(local_dir)
+        super().__init__()
+        text_path = os.path.normpath(f"{local_dir}/../base-models/all-MiniLM-L6-v2")
+        image_path = os.path.normpath(
+            f"{local_dir}/../base-models/siglip-base-patch16-512"
+        )
+        audio_path = os.path.normpath(f"{local_dir}/../base-models/whisper-tiny")
+
+        self.text_tokenizer = BertTokenizer.from_pretrained(
+            text_path, local_files_only=True
+        )
+        self.text_encoder = AutoModel.from_pretrained(text_path, local_files_only=True)
+
+        self.image_processor = SiglipProcessor.from_pretrained(
+            image_path, local_files_only=True
+        )
+        self.image_encoder = SiglipModel.from_pretrained(
+            image_path, local_files_only=True
+        ).vision_model
+        self.audio_processor = WhisperFeatureExtractor.from_pretrained(
+            audio_path, local_files_only=True
+        )
+        self.image_proj = nn.Linear(768, 384)
+        self.audio_processor = WhisperFeatureExtractor.from_pretrained(
+            audio_path, local_files_only=True
+        )
+        self.audio_encoder = WhisperModel.from_pretrained(
+            audio_path, local_files_only=True
+        ).encoder
+
+        checkpoint_path = f"{local_dir}/model.pt"
+        state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
+
+        self.text_encoder.load_state_dict(
+            {
+                k.replace("text_encoder.encoder.", ""): v
+                for k, v in state_dict.items()
+                if k.startswith("text_encoder.encoder.")
+            }
+        )
+
+        self.image_encoder.load_state_dict(
+            {
+                k.replace("image_encoder.vision_encoder.", ""): v
+                for k, v in state_dict.items()
+                if k.startswith("image_encoder.vision_encoder.")
+            }
+        )
+
+        self.image_proj.load_state_dict(
+            {
+                k.replace("image_encoder.projection.", ""): v
+                for k, v in state_dict.items()
+                if k.startswith("image_encoder.projection.")
+            }
+        )
+
+        self.audio_encoder.load_state_dict(
+            {
+                k.replace("audio_encoder.encoder.", ""): v
+                for k, v in state_dict.items()
+                if k.startswith("audio_encoder.encoder.")
+            }
+        )
+
+        self.eval()
+
+    @torch.no_grad()
+    def encode_text(self, texts):
+        if isinstance(texts, str):
+            texts = [texts]
+        inputs = self.text_tokenizer(
+            texts, padding=True, truncation=True, return_tensors="pt"
+        )
+        inputs = {k: v.to(next(self.parameters()).device) for k, v in inputs.items()}
+        outputs = self.text_encoder(**inputs)
+        embeddings = outputs.last_hidden_state.mean(dim=1)
+        return F.normalize(embeddings, p=2, dim=-1)
+
+    @torch.no_grad()
+    def encode_image(self, images):
+        if isinstance(images, Image.Image):
+            images = [images]
+        inputs = self.image_processor(images=images, return_tensors="pt")
+        inputs = {k: v.to(next(self.parameters()).device) for k, v in inputs.items()}
+        outputs = self.image_encoder(**inputs)
+        embeddings = self.image_proj(outputs.pooler_output)
+        return F.normalize(embeddings, p=2, dim=-1)
+
+    @torch.no_grad()
+    def encode_audio(self, waveform):
+        if isinstance(waveform, torch.Tensor):
+            waveform = waveform.squeeze().numpy()
+        inputs = self.audio_processor(
+            waveform, sampling_rate=16000, return_tensors="pt"
+        )
+        inputs = {k: v.to(next(self.parameters()).device) for k, v in inputs.items()}
+        outputs = self.audio_encoder(**inputs)
+        embeddings = outputs.last_hidden_state.mean(dim=1)
+        return F.normalize(embeddings, p=2, dim=-1)
+
+
+if __name__ == "__main__":
+    model = CompactEmbedder(local_dir="./models/multi-modal-embed-small")
+
+    text_vec = model.encode_text("example query")
+    print(f"Text shape: {text_vec.shape}")
diff --git a/embedding/server.py b/embedding/server.py
new file mode 100644
index 0000000..b58f041
--- /dev/null
+++ b/embedding/server.py
@@ -0,0 +1,111 @@
+import base64
+import io
+import os
+import sys
+from typing import Any
+
+import numpy as np
+import torch
+import uvicorn
+from fastapi import FastAPI
+from pydantic import BaseModel
+from PIL import Image
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from embedding.embedder import CompactEmbedder
+
+app = FastAPI()
+model = None
+
+
+@app.on_event("startup")
+def load_model():
+    global model
+    model = CompactEmbedder(local_dir="./models/multi-modal-embed-small")
+
+
+class InlineData(BaseModel):
+    mime_type: str
+    data: str
+
+
+class Part(BaseModel):
+    text: str | None = None
+    inline_data: InlineData | None = None
+
+
+class Content(BaseModel):
+    parts: list[Part]
+
+
+class EmbedContentRequest(BaseModel):
+    model: str = "models/multi-modal-embed-small"
+    content: Content
+    output_dimensionality: int | None = None
+    task_type: str | None = None
+    title: str | None = None
+
+
+class BatchEmbedContentRequest(BaseModel):
+    requests: list[EmbedContentRequest]
+
+
+def _encode_content(content: Content) -> list[float]:
+    vecs: list[Any] = []
+    for part in content.parts:
+        if part.text:
+            vecs.append(model.encode_text(part.text))
+        if part.inline_data:
+            raw = base64.b64decode(part.inline_data.data)
+            mime = part.inline_data.mime_type
+            if mime.startswith("image/"):
+                img = Image.open(io.BytesIO(raw)).convert("RGB")
+                vecs.append(model.encode_image(img))
+            elif mime.startswith("audio/"):
+                audio = np.frombuffer(raw, dtype=np.float32)
+                vecs.append(model.encode_audio(audio))
+    if not vecs:
+        vecs.append(model.encode_text(""))
+    vec = vecs[0] if len(vecs) == 1 else torch.mean(torch.stack(vecs), dim=0)
+    return vec.squeeze(0).tolist()
+
+
+class EmbeddingResponse(BaseModel):
+    embedding: dict[str, list[float]]
+
+
+class EmbedContentResponse(BaseModel):
+    embedding: dict[str, list[float]]
+
+
+class BatchEmbedContentResponse(BaseModel):
+    embeddings: list[dict[str, list[float]]]
+
+
+@app.post("/v1beta/models/{model_id}:embedContent")
+async def embed_content(
+    model_id: str,
+    req: EmbedContentRequest,
+):
+    emb = _encode_content(req.content)
+    if req.output_dimensionality:
+        emb = emb[: req.output_dimensionality]
+    return {"embedding": {"values": emb}}
+
+
+@app.post("/v1beta/models/{model_id}:batchEmbedContents")
+async def batch_embed_contents(
+    model_id: str,
+    req: BatchEmbedContentRequest,
+):
+    embeddings = []
+    for sub_req in req.requests:
+        emb = _encode_content(sub_req.content)
+        if sub_req.output_dimensionality:
+            emb = emb[: sub_req.output_dimensionality]
+        embeddings.append({"values": emb})
+    return {"embeddings": embeddings}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)
diff --git a/news.code-workspace b/news.code-workspace
new file mode 100644
index 0000000..876a149
--- /dev/null
+++ b/news.code-workspace
@@ -0,0 +1,8 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {}
+}
\ No newline at end of file
diff --git a/news_sites.json b/news_sites.json
new file mode 100644
index 0000000..8c657a4
--- /dev/null
+++ b/news_sites.json
@@ -0,0 +1,898 @@
+[
+  {
+    "name": "Yahoo News",
+    "url": {
+      "main": "https://news.yahoo.com",
+      "rss": "https://rss.news.yahoo.com/rss/topstories"
+    }
+  },
+  {
+    "name": "BBC News",
+    "url": {
+      "main": "https://www.bbc.com/news",
+      "rss": "http://feeds.bbci.co.uk/news/rss.xml"
+    }
+  },
+  {
+    "name": "CNN",
+    "url": {
+      "main": "https://www.cnn.com",
+      "rss": "https://rss.cnn.com/rss/edition.rss"
+    }
+  },
+  {
+    "name": "Al Jazeera English",
+    "url": {
+      "main": "https://www.aljazeera.com",
+      "rss": "https://www.aljazeera.com/xml/rss/all.xml"
+    }
+  },
+  {
+    "name": "The Guardian",
+    "url": {
+      "main": "https://www.theguardian.com",
+      "rss": "https://feeds.theguardian.com/theguardian/world/rss"
+    }
+  },
+  {
+    "name": "Fox News",
+    "url": {
+      "main": "https://www.foxnews.com",
+      "rss": "https://feed.feedburner.com/foxnews"
+    }
+  },
+  {
+    "name": "Reuters",
+    "url": {
+      "main": "https://www.reuters.com",
+      "rss": "https://www.reutersagency.com/feed/?best-topics=all&posts_type=best"
+    }
+  },
+  {
+    "name": "France 24",
+    "url": {
+      "main": "https://www.france24.com",
+      "rss": "https://www.france24.com/en/feed/rss/news"
+    }
+  },
+  {
+    "name": "The New York Times",
+    "url": {
+      "main": "https://www.nytimes.com",
+      "rss": "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml"
+    }
+  },
+  {
+    "name": "The Independent",
+    "url": {
+      "main": "https://www.independent.co.uk",
+      "rss": "https://www.independent.co.uk/rss"
+    }
+  },
+  {
+    "name": "DW News",
+    "url": {
+      "main": "https://www.dw.com",
+      "rss": "https://rss.dw.com/xml/rss-en-all"
+    }
+  },
+  {
+    "name": "The Washington Post",
+    "url": {
+      "main": "https://www.washingtonpost.com",
+      "rss": "https://feeds.washingtonpost.com/rss/rss"
+    }
+  },
+  {
+    "name": "NBC News",
+    "url": {
+      "main": "https://www.nbcnews.com",
+      "rss": "https://feeds.nbcnews.com/nbcnews/public/tnbc/topstories"
+    }
+  },
+  {
+    "name": "CBS News",
+    "url": {
+      "main": "https://www.cbsnews.com",
+      "rss": "https://www.cbsnews.com/latest/"
+    }
+  },
+  {
+    "name": "ABC News",
+    "url": {
+      "main": "https://abcnews.go.com",
+      "rss": "https://abcnews.go.com/abcnews/keynotes"
+    }
+  },
+  {
+    "name": "MSNBC",
+    "url": {
+      "main": "https://www.msnbc.com",
+      "rss": "https://www.msnbc.com/id/msnbc/rss"
+    }
+  },
+  {
+    "name": "Forbes",
+    "url": {
+      "main": "https://www.forbes.com",
+      "rss": "https://www.forbes.com/forbes/feed/rss.xml"
+    }
+  },
+  {
+    "name": "Business Insider",
+    "url": {
+      "main": "https://www.businessinsider.com",
+      "rss": "https://www.businessinsider.com/rss"
+    }
+  },
+  {
+    "name": "The Times of India",
+    "url": {
+      "main": "https://timesofindia.indiatimes.com",
+      "rss": "http://timesofindia.indiatimes.com/rssfeedstopstories.cms"
+    }
+  },
+  {
+    "name": "Euronews",
+    "url": {
+      "main": "https://www.euronews.com",
+      "rss": "https://www.euronews.com/rss"
+    }
+  },
+  {
+    "name": "Al Arabiya",
+    "url": {
+      "main": "https://www.alarabiya.net",
+      "rss": "https://www.alarabiya.net/feed"
+    }
+  },
+  {
+    "name": "Sky News",
+    "url": {
+      "main": "https://news.sky.com",
+      "rss": "https://news.sky.com/rss"
+    }
+  },
+  {
+    "name": "NDTV",
+    "url": {
+      "main": "https://www.ndtv.com",
+      "rss": "https://feeds.feedburner.com/ndtvnews-topstories"
+    }
+  },
+  {
+    "name": "India Today",
+    "url": {
+      "main": "https://www.indiatoday.in",
+      "rss": "https://www.indiatoday.in/rss"
+    }
+  },
+  {
+    "name": "HuffPost",
+    "url": {
+      "main": "https://www.huffpost.com",
+      "rss": "https://www.huffpost.com/feeds/index"
+    }
+  },
+  {
+    "name": "The Telegraph",
+    "url": {
+      "main": "https://www.telegraph.co.uk",
+      "rss": "https://www.telegraph.co.uk/rss.xml"
+    }
+  },
+  {
+    "name": "El Pa\u00eds",
+    "url": {
+      "main": "https://elpais.com",
+      "rss": "https://feeds.elpais.com/elpais/compact/portada.xml"
+    }
+  },
+  {
+    "name": "Politico",
+    "url": {
+      "main": "https://www.politico.com",
+      "rss": "https://www.politico.com/rss/political-calculate.xml"
+    }
+  },
+  {
+    "name": "CNBC",
+    "url": {
+      "main": "https://www.cnbc.com",
+      "rss": "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114"
+    }
+  },
+  {
+    "name": "The Hindu",
+    "url": {
+      "main": "https://www.thehindu.com",
+      "rss": "https://www.thehindu.com/news/national/feeder/default.rss"
+    }
+  },
+  {
+    "name": "Le Monde",
+    "url": {
+      "main": "https://www.lemonde.fr",
+      "rss": "https://www.lemonde.fr/rss/une.xml"
+    }
+  },
+  {
+    "name": "Bloomberg",
+    "url": {
+      "main": "https://www.bloomberg.com",
+      "rss": "https://feeds.bloomberg.com/markets/news.rss"
+    }
+  },
+  {
+    "name": "The Economist",
+    "url": {
+      "main": "https://www.economist.com",
+      "rss": "https://www.economist.com/rss"
+    }
+  },
+  {
+    "name": "ABC News Australia",
+    "url": {
+      "main": "https://www.abc.net.au/news",
+      "rss": "https://www.abc.net.au/feed/latestnews/rss.xml"
+    }
+  },
+  {
+    "name": "Corriere della Sera",
+    "url": {
+      "main": "https://www.corriere.it",
+      "rss": "https://feedfeed.corriere.it/rss.xml"
+    }
+  },
+  {
+    "name": "Bild",
+    "url": {
+      "main": "https://www.bild.de",
+      "rss": "https://www.bild.de/rssfeeds/top/rss-feeder.top.aufkleber.ilw.xml"
+    }
+  },
+  {
+    "name": "Der Spiegel",
+    "url": {
+      "main": "https://www.spiegel.de",
+      "rss": "https://www.spiegel.de/schlagzeilen/index.rss"
+    }
+  },
+  {
+    "name": "Daily Mail",
+    "url": {
+      "main": "https://www.dailymail.co.uk",
+      "rss": "https://www.dailymail.co.uk/indices/news-rss/index.rss"
+    }
+  },
+  {
+    "name": "Le Figaro",
+    "url": {
+      "main": "https://www.lefigaro.fr",
+      "rss": "https://www.lefigaro.fr/rss/figaro.xml"
+    }
+  },
+  {
+    "name": "CBC News",
+    "url": {
+      "main": "https://www.cbc.ca/news",
+      "rss": "https://rss.cbc.ca/readerfeed/cc/rss/NewsTopStories.xml"
+    }
+  },
+  {
+    "name": "USA Today",
+    "url": {
+      "main": "https://www.usatoday.com",
+      "rss": "https://www.usatoday.com/rss"
+    }
+  },
+  {
+    "name": "Wall Street Journal",
+    "url": {
+      "main": "https://www.wsj.com",
+      "rss": "https://feeds.a.dj.com/rss/RSSWorldNews.xml"
+    }
+  },
+  {
+    "name": "NHK World Japan",
+    "url": {
+      "main": "https://www3.nhk.or.jp/nhkworld",
+      "rss": "https://www3.nhk.or.jp/news/xml/lw_news_rss.xml"
+    }
+  },
+  {
+    "name": "NPR",
+    "url": {
+      "main": "https://www.npr.org",
+      "rss": "https://feeds.npr.org/1001/rss.xml"
+    }
+  },
+  {
+    "name": "SBS News",
+    "url": {
+      "main": "https://www.sbs.com.au/news",
+      "rss": "https://www.sbs.com.au/news/feed"
+    }
+  },
+  {
+    "name": "The Australian",
+    "url": {
+      "main": "https://www.theaustralian.com.au",
+      "rss": "https://www.theaustralian.com.au/rss/feed"
+    }
+  },
+  {
+    "name": "Nikkei Asia",
+    "url": {
+      "main": "https://asia.nikkei.com",
+      "rss": "https://asia.nikkei.com/feed"
+    }
+  },
+  {
+    "name": "Financial Times",
+    "url": {
+      "main": "https://www.ft.com",
+      "rss": "https://www.ft.com/rss/home"
+    }
+  },
+  {
+    "name": "RT",
+    "url": {
+      "main": "https://www.rt.com",
+      "rss": "https://www.rt.com/rss/"
+    }
+  },
+  {
+    "name": "The Japan Times",
+    "url": {
+      "main": "https://www.japantimes.co.jp",
+      "rss": "https://www.japantimes.co.jp/feed/"
+    }
+  },
+  {
+    "name": "S\u00fcddeutsche Zeitung",
+    "url": {
+      "main": "https://www.sueddeutsche.de",
+      "rss": "https://rss.sueddeutsche.de/alles"
+    }
+  },
+  {
+    "name": "La Repubblica",
+    "url": {
+      "main": "https://www.repubblica.it",
+      "rss": "https://feed.pressreader.com/it/la-repubblica"
+    }
+  },
+  {
+    "name": "Frankfurter Allgemeine",
+    "url": {
+      "main": "https://www.faz.net",
+      "rss": "https://www.faz.net/rss/aktuell"
+    }
+  },
+  {
+    "name": "iNews",
+    "url": {
+      "main": "https://inews.co.uk",
+      "rss": "https://inews.co.uk/feed/"
+    }
+  },
+  {
+    "name": "Metro UK",
+    "url": {
+      "main": "https://metro.co.uk",
+      "rss": "https://metro.co.uk/feed/"
+    }
+  },
+  {
+    "name": "The Atlantic",
+    "url": {
+      "main": "https://www.theatlantic.com",
+      "rss": "https://www.theatlantic.com/feeds/latest"
+    }
+  },
+  {
+    "name": "Vox",
+    "url": {
+      "main": "https://www.vox.com",
+      "rss": "https://www.vox.com/rss/index.xml"
+    }
+  },
+  {
+    "name": "Time",
+    "url": {
+      "main": "https://time.com",
+      "rss": "https://time.com/time_rss_100.xml"
+    }
+  },
+  {
+    "name": "Newsweek",
+    "url": {
+      "main": "https://www.newsweek.com",
+      "rss": "https://www.newsweek.com/rss"
+    }
+  },
+  {
+    "name": "The Verge",
+    "url": {
+      "main": "https://www.theverge.com",
+      "rss": "https://www.theverge.com/rss/index.xml"
+    }
+  },
+  {
+    "name": "TechCrunch",
+    "url": {
+      "main": "https://techcrunch.com",
+      "rss": "https://techcrunch.com/feed/"
+    }
+  },
+  {
+    "name": "Ars Technica",
+    "url": {
+      "main": "https://arstechnica.com",
+      "rss": "https://feeds.arstechnica.com/arstechnica/index"
+    }
+  },
+  {
+    "name": "The Hill",
+    "url": {
+      "main": "https://thehill.com",
+      "rss": "https://thehill.com/feed/"
+    }
+  },
+  {
+    "name": "Axios",
+    "url": {
+      "main": "https://www.axios.com",
+      "rss": "https://www.axios.com/rss-feeds"
+    }
+  },
+  {
+    "name": "Wired",
+    "url": {
+      "main": "https://www.wired.com",
+      "rss": "https://www.wired.com/feed/rss"
+    }
+  },
+  {
+    "name": "The Daily Beast",
+    "url": {
+      "main": "https://www.thedailybeast.com",
+      "rss": "https://www.thedailybeast.com/rss"
+    }
+  },
+  {
+    "name": "ProPublica",
+    "url": {
+      "main": "https://www.propublica.org",
+      "rss": "https://www.propublica.org/feed"
+    }
+  },
+  {
+    "name": "South China Morning Post",
+    "url": {
+      "main": "https://www.scmp.com",
+      "rss": "https://www.scmp.com/rss"
+    }
+  },
+  {
+    "name": "The Straits Times",
+    "url": {
+      "main": "https://www.straitstimes.com",
+      "rss": "https://www.straitstimes.com/rss"
+    }
+  },
+  {
+    "name": "The Globe and Mail",
+    "url": {
+      "main": "https://www.theglobeandmail.com",
+      "rss": "https://www.theglobeandmail.com/rss/latest.headlines.xml"
+    }
+  },
+  {
+    "name": "The Irish Times",
+    "url": {
+      "main": "https://www.irishtimes.com",
+      "rss": "https://www.irishtimes.com/rss"
+    }
+  },
+  {
+    "name": "Le Soir",
+    "url": {
+      "main": "https://www.lesoir.be",
+      "rss": "https://www.lesoir.be/rss"
+    }
+  },
+  {
+    "name": "ABC Espa\u00f1a",
+    "url": {
+      "main": "https://www.abc.es",
+      "rss": "https://www.abc.es/rss"
+    }
+  },
+  {
+    "name": "El Mundo",
+    "url": {
+      "main": "https://www.elmundo.es",
+      "rss": "https://www.elmundo.es/elmundo/rss/portada.xml"
+    }
+  },
+  {
+    "name": "L'Express",
+    "url": {
+      "main": "https://www.lexpress.fr",
+      "rss": "https://www.lexpress.fr/feeds/flux-rss-une.xml"
+    }
+  },
+  {
+    "name": "Clar\u00edn",
+    "url": {
+      "main": "https://www.clarin.com",
+      "rss": "https://www.clarin.com/feed/"
+    }
+  },
+  {
+    "name": "Folha de S.Paulo",
+    "url": {
+      "main": "https://www1.folha.uol.com.br",
+      "rss": "https://www.folha.uol.com.br/podcasts/feed"
+    }
+  },
+  {
+    "name": "O Globo",
+    "url": {
+      "main": "https://oglobo.globo.com",
+      "rss": "https://oglobo.globo.com/rss"
+    }
+  },
+  {
+    "name": "The Korea Herald",
+    "url": {
+      "main": "https://www.koreaherald.com",
+      "rss": "https://www.koreaherald.com/rss"
+    }
+  },
+  {
+    "name": "The National",
+    "url": {
+      "main": "https://www.thenationalnews.com",
+      "rss": "https://www.thenationalnews.com/rss"
+    }
+  },
+  {
+    "name": "Meduza",
+    "url": {
+      "main": "https://meduza.io",
+      "rss": "https://meduza.io/feed"
+    }
+  },
+  {
+    "name": "AFP",
+    "url": {
+      "main": "https://www.afp.com",
+      "rss": "https://www.afp.com/en/feed"
+    }
+  },
+  {
+    "name": "Variety",
+    "url": {
+      "main": "https://variety.com",
+      "rss": "https://variety.com/feed/"
+    }
+  },
+  {
+    "name": "Rolling Stone",
+    "url": {
+      "main": "https://www.rollingstone.com",
+      "rss": "https://www.rollingstone.com/feed/"
+    }
+  },
+  {
+    "name": "Foreign Policy",
+    "url": {
+      "main": "https://foreignpolicy.com",
+      "rss": "https://foreignpolicy.com/feed/"
+    }
+  },
+  {
+    "name": "News.com.au",
+    "url": {
+      "main": "https://www.news.com.au",
+      "rss": "https://www.news.com.au/rss"
+    }
+  },
+  {
+    "name": "The Times",
+    "url": {
+      "main": "https://www.thetimes.co.uk",
+      "rss": "https://www.thetimes.co.uk/rss"
+    }
+  },
+  {
+    "name": "Handelsblatt",
+    "url": {
+      "main": "https://www.handelsblatt.com",
+      "rss": "https://www.handelsblatt.com/feeds/rss"
+    }
+  },
+  {
+    "name": "Die Welt",
+    "url": {
+      "main": "https://www.welt.de",
+      "rss": "https://www.welt.de/rss/local.php"
+    }
+  },
+  {
+    "name": "Il Giornale",
+    "url": {
+      "main": "https://www.ilgiornale.it",
+      "rss": "https://www.ilgiornale.it/feeds/rss"
+    }
+  },
+  {
+    "name": "The Sydney Morning Herald",
+    "url": {
+      "main": "https://www.smh.com.au",
+      "rss": "https://www.smh.com.au/rss/headlines"
+    }
+  },
+  {
+    "name": "Al Jazeera Arabic",
+    "url": {
+      "main": "https://www.aljazeera.net",
+      "rss": "https://www.aljazeera.com/xml/rss/all.xml"
+    }
+  },
+  {
+    "name": "Deccan Chronicle",
+    "url": {
+      "main": "https://www.deccanchronicle.com",
+      "rss": "https://www.deccanchronicle.com/feed"
+    }
+  },
+  {
+    "name": "Mint",
+    "url": {
+      "main": "https://www.livemint.com",
+      "rss": "https://www.livemint.com/rss/"
+    }
+  },
+  {
+    "name": "Business Standard",
+    "url": {
+      "main": "https://www.business-standard.com",
+      "rss": "https://www.business-standard.com/rss"
+    }
+  },
+  {
+    "name": "The Hindu BusinessLine",
+    "url": {
+      "main": "https://www.thehindubusinessline.com",
+      "rss": "https://www.thehindubusinessline.com/rss"
+    }
+  },
+  {
+    "name": "La Naci\u00f3n",
+    "url": {
+      "main": "https://www.lanacion.com.ar",
+      "rss": "https://www.lanacion.com.ar/rss"
+    }
+  },
+  {
+    "name": "O Estado de S. Paulo",
+    "url": {
+      "main": "https://www.estadao.com.br",
+      "rss": "https://www.estadao.com.br/rss/"
+    }
+  },
+  {
+    "name": "Le Devoir",
+    "url": {
+      "main": "https://www.ledevoir.com",
+      "rss": "https://www.ledevoir.com/rss"
+    }
+  },
+  {
+    "name": "Korea Times",
+    "url": {
+      "main": "https://www.koreatimes.co.kr",
+      "rss": "https://www.koreatimes.co.kr/rss"
+    }
+  },
+  {
+    "name": "Nikkei",
+    "url": {
+      "main": "https://www.nikkei.com",
+      "rss": "https://www.nikkei.com/feed/"
+    }
+  },
+  {
+    "name": "Daily Nation",
+    "url": {
+      "main": "https://www.nation.co.ke",
+      "rss": "https://www.nation.co.ke/rss/"
+    }
+  },
+  {
+    "name": "The Print",
+    "url": {
+      "main": "https://theprint.in",
+      "rss": "https://theprint.in/feed/"
+    }
+  },
+  {
+    "name": "The Quint",
+    "url": {
+      "main": "https://www.thequint.com",
+      "rss": "https://www.thequint.com/feed"
+    }
+  },
+  {
+    "name": "WION",
+    "url": {
+      "main": "https://www.wionews.com",
+      "rss": "https://www.wionews.com/rss"
+    }
+  },
+  {
+    "name": "The Conversation",
+    "url": {
+      "main": "https://theconversation.com",
+      "rss": "https://theconversation.com/feed"
+    }
+  },
+  {
+    "name": "The Wire",
+    "url": {
+      "main": "https://thewire.in",
+      "rss": "https://thewire.in/feed/"
+    }
+  },
+  {
+    "name": "The Guardian Australia",
+    "url": {
+      "main": "https://www.theguardian.com/au",
+      "rss": "https://feeds.theguardian.com/theguardian/australia-news/rss"
+    }
+  },
+  {
+    "name": "The Guardian Canada",
+    "url": {
+      "main": "https://www.theguardian.com/au",
+      "rss": "https://feeds.theguardian.com/theguardian/us-news/rss"
+    }
+  },
+  {
+    "name": "El Pa\u00eds Semanal",
+    "url": {
+      "main": "https://semanal.elpais.com",
+      "rss": "https://feeds.elpais.com/elpais/compact/portada.xml"
+    }
+  },
+  {
+    "name": "BBC Afrikaans",
+    "url": {
+      "main": "https://www.bbc.com/afrikaans",
+      "rss": "https://www.bbc.com/news/world/africa/rss.xml"
+    }
+  },
+  {
+    "name": "Africa News",
+    "url": {
+      "main": "https://africanews.com",
+      "rss": "https://africanews.com/feed/"
+    }
+  },
+  {
+    "name": "Pagina/12",
+    "url": {
+      "main": "https://www.pagina12.com.ar",
+      "rss": "https://www.pagina12.com.ar/rss"
+    }
+  },
+  {
+    "name": "The Malaysian Reserve",
+    "url": {
+      "main": "https://www.malaysianreserve.com",
+      "rss": "https://www.malaysianreserve.com/feed"
+    }
+  },
+  {
+    "name": "The Standard",
+    "url": {
+      "main": "https://www.thestandard.com.hk",
+      "rss": "https://www.thestandard.com.hk/feed"
+    }
+  },
+  {
+    "name": "Manila Bulletin",
+    "url": {
+      "main": "https://mb.com.ph",
+      "rss": "https://mb.com.ph/feed/"
+    }
+  },
+  {
+    "name": "The New Humanist",
+    "url": {
+      "main": "https://the-new-humanist.co.uk",
+      "rss": "https://www.the-new-humanist.co.uk/feed"
+    }
+  },
+  {
+    "name": "The Canary",
+    "url": {
+      "main": "https://thecanary.co",
+      "rss": "https://thecanary.co/feed/"
+    }
+  },
+  {
+    "name": "The Free Press",
+    "url": {
+      "main": "https://www.thefreepress.com",
+      "rss": "https://www.thefreepress.com/feed"
+    }
+  },
+  {
+    "name": "Reason",
+    "url": {
+      "main": "https://reason.com",
+      "rss": "https://reason.com/feed/"
+    }
+  },
+  {
+    "name": "The Daily Caller",
+    "url": {
+      "main": "https://dailycaller.com",
+      "rss": "https://dailycaller.com/feed/"
+    }
+  },
+  {
+    "name": "The Federalist",
+    "url": {
+      "main": "https://thefederalist.com",
+      "rss": "https://thefederalist.com/feed/"
+    }
+  },
+  {
+    "name": "The Nation",
+    "url": {
+      "main": "https://www.thenation.com",
+      "rss": "https://www.thenation.com/feed/"
+    }
+  },
+  {
+    "name": "Dissent",
+    "url": {
+      "main": "https://www.dissentmagazine.org",
+      "rss": "https://www.dissentmagazine.org/feed"
+    }
+  },
+  {
+    "name": "The New Republic",
+    "url": {
+      "main": "https://newrepublic.com",
+      "rss": "https://newrepublic.com/feed/"
+    }
+  },
+  {
+    "name": "Harper's Magazine",
+    "url": {
+      "main": "https://harpers.org",
+      "rss": "https://harpers.org/feed/"
+    }
+  },
+  {
+    "name": "The Nation (UK)",
+    "url": {
+      "main": "https://www.nation.come",
+      "rss": "https://www.thenation.co.uk/feed"
+    }
+  },
+  {
+    "name": "Al-Monitor",
+    "url": {
+      "main": "https://www.al-monitor.com",
+      "rss": "https://www.al-monitor.com/rss"
+    }
+  }
+]
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..070fc35
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+playwright
+pyquery
diff --git a/scraper/__init__.py b/scraper/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/scraper/__init__.py
diff --git a/scraper/__main__.py b/scraper/__main__.py
new file mode 100644
index 0000000..329b102
--- /dev/null
+++ b/scraper/__main__.py
@@ -0,0 +1,53 @@
+import asyncio
+import argparse
+import shutil
+from scraper.spiders.page import PageScraper
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Scrape pages")
+    parser.add_argument("url", help="URL to scrape")
+    parser.add_argument("-o", "--output", default="data", help="Output directory")
+    parser.add_argument(
+        "-d", "--depth", type=int, required=True, help="Link traversal depth"
+    )
+    parser.add_argument(
+        "-c",
+        "--clean",
+        action="store_true",
+        help="Remove output directory before scraping",
+    )
+    parser.add_argument("-n", "--max-pages", type=int, help="Max pages to scrape")
+    parser.add_argument("-a", "--max-assets", type=int, help="Max assets to download")
+    parser.add_argument("-p", "--parallel", type=int, help="Number of parallel workers")
+    parser.add_argument(
+        "-t", "--timeout", type=int, help="Playwright default timeout in ms"
+    )
+    parser.add_argument("--page-timeout", type=int, help="Per-page timeout in seconds")
+    args = parser.parse_args()
+
+    if args.clean:
+        shutil.rmtree(args.output, ignore_errors=True)
+
+    scraper_kwargs = {
+        "url": args.url,
+        "output_dir": args.output,
+        "max_depth": args.depth,
+    }
+    if args.max_pages is not None:
+        scraper_kwargs["max_pages"] = args.max_pages
+    if args.max_assets is not None:
+        scraper_kwargs["max_assets"] = args.max_assets
+    if args.parallel is not None:
+        scraper_kwargs["max_parallel"] = args.parallel
+    if args.timeout is not None:
+        scraper_kwargs["timeout"] = args.timeout
+    if args.page_timeout is not None:
+        scraper_kwargs["page_timeout"] = args.page_timeout
+
+    scraper = PageScraper(**scraper_kwargs)
+    asyncio.run(scraper.run())
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scraper/spiders/__init__.py b/scraper/spiders/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/scraper/spiders/__init__.py
diff --git a/scraper/spiders/page.py b/scraper/spiders/page.py
new file mode 100644
index 0000000..4edcfa6
--- /dev/null
+++ b/scraper/spiders/page.py
@@ -0,0 +1,450 @@
+import asyncio
+import datetime
+import hashlib
+import json
+import logging
+import os
+import re
+import urllib.parse
+from pyquery import PyQuery as pq
+from playwright.async_api import async_playwright
+
+logger = logging.getLogger(__name__)
+
+UNLIMITED = 2**31
+
+
+class PageScraper:
+    def __init__(
+        self,
+        url,
+        output_dir="data",
+        max_depth=5,
+        max_pages=None,
+        max_assets=None,
+        max_parallel=1,
+        timeout=60000,
+        page_timeout=120,
+    ):
+        self.url = url
+        self.output_dir = output_dir
+        self.max_depth = max_depth
+        self.max_pages = max_pages
+        self.max_assets = max_assets
+        self.max_parallel = max_parallel
+        self.timeout = timeout
+        self.page_timeout = page_timeout
+        self.page_count = 0
+        self.asset_count = 0
+        self.enqueued = set()
+        self._pages = {}
+        self._queue = asyncio.Queue()
+        self._busy = 0
+        self._origin_base = ".".join(
+            urllib.parse.urlparse(url).netloc.rstrip(".").split(".")[-2:]
+        )
+
+    def _url_dir(self, url):
+        parsed = urllib.parse.urlparse(url)
+        path = parsed.path.strip("/")
+        return (
+            os.path.join(self.output_dir, parsed.netloc, path)
+            if path
+            else os.path.join(self.output_dir, parsed.netloc)
+        )
+
+    def _read_meta(self, url):
+        meta_path = os.path.join(self._url_dir(url), "initial.json")
+        if not os.path.isfile(meta_path):
+            return None
+        try:
+            with open(meta_path) as f:
+                return json.load(f)
+        except (json.JSONDecodeError, FileNotFoundError):
+            return None
+
+    def _content_changed(self, url, content):
+        new_hash = hashlib.sha256(content.encode()).hexdigest()
+        previous = self._read_meta(url)
+        if previous:
+            return previous.get("content", {}).get("initial_sha256") != new_hash
+        return True
+
+    def _is_asset(self, resource_type):
+        return resource_type in (
+            "stylesheet",
+            "image",
+            "media",
+            "font",
+            "script",
+            "fetch",
+            "xhr",
+            "other",
+        )
+
+    def _asset_path(self, url, content_type):
+        parsed = urllib.parse.urlparse(url)
+        netloc = parsed.netloc or parsed.path.split("/")[0]
+        path = parsed.path.strip("/") if parsed.path else netloc
+        if path.startswith(netloc):
+            path = "/" + path[len(netloc) :]
+        ext_map = {
+            "image/png": "png",
+            "image/jpeg": "jpg",
+            "image/webp": "webp",
+            "image/gif": "gif",
+            "image/svg+xml": "svg",
+            "image/x-icon": "ico",
+            "application/javascript": "js",
+            "text/javascript": "js",
+            "text/css": "css",
+            "font/woff2": "woff2",
+            "font/woff": "woff",
+            "application/json": "json",
+            "application/pdf": "pdf",
+        }
+        _, ext = os.path.splitext(path)
+        if ext:
+            return os.path.join(self.output_dir, netloc, path)
+        mime_ext = ext_map.get(content_type.split(";")[0].strip().lower())
+        if mime_ext:
+            return os.path.join(self.output_dir, netloc, path, f"index.{mime_ext}")
+        return os.path.join(self.output_dir, netloc, path, "index.dat")
+
+    async def _on_response(self, response, current_url_ref):
+        try:
+            url = response.url
+            if not self._is_asset(response.request.resource_type):
+                return
+            if self.max_assets is not None and self.asset_count >= self.max_assets:
+                return
+            content_type = response.headers.get("content-type", "")
+            asset_path = self._asset_path(url, content_type)
+            if os.path.isfile(asset_path):
+                return
+            self.asset_count += 1
+            try:
+                body = await response.body()
+            except Exception:
+                return
+            try:
+                os.makedirs(os.path.dirname(asset_path), exist_ok=True)
+            except OSError:
+                return
+            with open(asset_path, "wb") as f:
+                f.write(body)
+            asset_hash = hashlib.sha256(body).hexdigest()
+            meta = {
+                "url": url,
+                "request": {
+                    "method": response.request.method,
+                    "headers": dict(response.request.headers),
+                },
+                "response": {
+                    "status": response.status,
+                    "headers": dict(response.headers),
+                },
+                "content": {"sha256": asset_hash},
+                "scrape": {
+                    "timestamp": datetime.datetime.now(
+                        datetime.timezone.utc
+                    ).isoformat(),
+                },
+            }
+            with open(asset_path + ".json", "w", encoding="utf-8") as f:
+                json.dump(meta, f, indent=2)
+            page_url = current_url_ref.get()
+            if page_url and page_url in self._pages:
+                self._pages[page_url]["assets"].append(
+                    {
+                        "url": url,
+                    }
+                )
+        except Exception:
+            pass
+
+    def _extract_article_links(self, doc, page_url):
+        links = set()
+        skip_patterns = [
+            re.compile(
+                r"^/(about|contact|privacy|terms|ad|subscribe|newsletter|"
+                r"login|signup|careers|feedback|site|help|support|faq|"
+                r"sitemap|robots|feed|rss|api|static|assets|img|images|"
+                r"css|js|scripts|fonts|cdn|media|video|audio)$"
+            ),
+            re.compile(
+                r"\.(png|jpg|jpeg|gif|svg|ico|css|js|woff|woff2|ttf|pdf|xml|json|html)$"
+            ),
+            re.compile(r"^#"),
+            re.compile(r"^mailto:"),
+            re.compile(r"^javascript:"),
+            re.compile(r"^/$"),
+        ]
+        for a in doc("a").items():
+            href = a.attr("href") or ""
+            resolved = urllib.parse.urljoin(page_url, href)
+            parsed = urllib.parse.urlparse(resolved)
+            parsed_base = ".".join(parsed.netloc.rstrip(".").split(".")[-2:])
+            if parsed_base != self._origin_base:
+                continue
+            path = parsed.path.strip("/")
+            if not path:
+                continue
+            if any(s.search(parsed.path) for s in skip_patterns):
+                continue
+            segments = [s for s in path.split("/") if s]
+            if len(segments) >= 2:
+                links.add(resolved)
+        links.discard(self.url)
+        for seen in self.enqueued:
+            links.discard(seen)
+        return links
+
+    async def _process_page(self, page, page_url, depth):
+        headers = {}
+        previous = self._read_meta(page_url)
+        if previous:
+            etag = previous.get("response", {}).get("etag")
+            if etag:
+                headers["if-none-match"] = etag
+            timestamp = previous.get("scrape", {}).get("timestamp")
+            if timestamp:
+                headers["if-modified-since"] = timestamp
+        try:
+            await page.set_extra_http_headers(headers)
+            resp = await page.goto(page_url, wait_until="domcontentloaded")
+            await page.set_extra_http_headers({})
+        except Exception as e:
+            logger.error(f"Failed to navigate to {page_url}: {e}")
+            return
+        page_url = resp.url
+        if resp.request.redirected_from:
+            redirect_url = resp.request.redirected_from.url
+            page_dir = self._url_dir(redirect_url)
+            meta_path = os.path.join(page_dir, "initial.json")
+            if not os.path.exists(meta_path):
+                os.makedirs(page_dir, exist_ok=True)
+                prev_resp = resp.request.redirected_from.existing_response
+                meta = {
+                    "request": {
+                        "method": resp.request.method,
+                        "url": redirect_url,
+                        "final_url": page_url,
+                        "redirect_chain": [],
+                        "headers": dict(resp.request.headers),
+                    },
+                    "response": {
+                        "status": prev_resp.status if prev_resp else None,
+                        "headers": dict(resp.headers),
+                    },
+                    "content": {"sha256": ""},
+                    "scrape": {
+                        "timestamp": datetime.datetime.now(
+                            datetime.timezone.utc
+                        ).isoformat(),
+                    },
+                }
+                with open(meta_path, "w", encoding="utf-8") as f:
+                    json.dump(meta, f, indent=2)
+        if resp.status == 304:
+            page_dir = self._url_dir(page_url)
+            initial_path = os.path.join(page_dir, "initial.html")
+            if os.path.isfile(initial_path):
+                with open(initial_path, encoding="utf-8") as f:
+                    local_html = f.read()
+                await self._enqueue_links(local_html, page_url, depth)
+            return
+        initial_html = await page.content()
+        content_hash = hashlib.sha256(initial_html.encode()).hexdigest()
+        changed = self._content_changed(page_url, initial_html)
+
+        if changed:
+            at_limit = self.max_pages is not None and self.page_count >= self.max_pages
+            if at_limit:
+                return
+            self.page_count += 1
+            logger.info(f"Scraping page {self.page_count}: {page_url}")
+            page_dir = self._url_dir(page_url)
+            os.makedirs(page_dir, exist_ok=True)
+            with open(
+                os.path.join(page_dir, "initial.html"), "w", encoding="utf-8"
+            ) as f:
+                f.write(initial_html)
+
+            await self._wait_for_hydrated(page)
+            await self._scroll_page(page)
+            final_html = await page.content()
+            final_hash = hashlib.sha256(final_html.encode()).hexdigest()
+            with open(os.path.join(page_dir, "index.html"), "w", encoding="utf-8") as f:
+                f.write(final_html)
+            page_data = {"assets": []}
+            self._pages[page_url] = page_data
+            meta = {
+                "request": {
+                    "method": resp.request.method,
+                    "url": page_url,
+                    "final_url": page_url,
+                    "redirect_chain": [],
+                    "headers": dict(resp.request.headers),
+                },
+                "response": {
+                    "status": resp.status,
+                    "headers": dict(resp.headers),
+                    "etag": resp.headers.get("etag"),
+                },
+                "content": {
+                    "sha256": final_hash,
+                    "initial_sha256": content_hash,
+                },
+                "scrape": {
+                    "timestamp": datetime.datetime.now(
+                        datetime.timezone.utc
+                    ).isoformat(),
+                },
+            }
+            with open(
+                os.path.join(page_dir, "initial.json"), "w", encoding="utf-8"
+            ) as f:
+                json.dump(meta, f, indent=2)
+            await self._enqueue_links(initial_html, page_url, depth)
+            await self._enqueue_links(final_html, page_url, depth)
+        else:
+            logger.info(f"Page unchanged, skipping: {page_url}")
+            page_dir = self._url_dir(page_url)
+            initial_path = os.path.join(page_dir, "initial.html")
+            if os.path.isfile(initial_path):
+                with open(initial_path, encoding="utf-8") as f:
+                    local_html = f.read()
+                await self._enqueue_links(local_html, page_url, depth)
+
+    async def _wait_for_hydrated(self, page):
+        prev_size = -1
+        stable = 0
+        for _ in range(10):
+            html = await page.content()
+            curr_size = len(html)
+            if abs(curr_size - prev_size) < 500:
+                stable += 1
+            else:
+                stable = 0
+            prev_size = curr_size
+            if stable >= 2:
+                break
+            await asyncio.sleep(1)
+
+    async def _scroll_page(self, page):
+        """Scroll to bottom and back to trigger lazy loading."""
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await asyncio.sleep(1)
+        await page.evaluate("window.scrollTo(0, 0)")
+
+    async def _enqueue_links(self, html, page_url, depth):
+        if self.max_pages is not None and self.page_count >= self.max_pages:
+            return
+        if depth >= self.max_depth:
+            return
+        doc = pq(html)
+        remaining = (
+            (self.max_pages - self.page_count)
+            if self.max_pages is not None
+            else UNLIMITED
+        )
+        for link_url in self._extract_article_links(doc, page_url):
+            if remaining <= 0:
+                break
+            if link_url not in self.enqueued:
+                self.enqueued.add(link_url)
+                remaining -= 1
+                await self._queue.put((link_url, depth + 1))
+
+    async def _worker(self, worker_id, pages):
+        page = pages[worker_id]
+        current_url_ref = {"url": None}
+        page.on("response", lambda resp, r=current_url_ref: self._on_response(resp, r))
+        while True:
+            if self.max_pages is not None and self.page_count >= self.max_pages:
+                break
+            item = await self._queue.get()
+            if item is None:
+                break
+            url, depth = item
+            if depth >= self.max_depth:
+                self._queue.task_done()
+                continue
+            self._busy += 1
+            try:
+                if current_url_ref["url"] and current_url_ref["url"] in self._pages:
+                    self._pages[current_url_ref["url"]]["assets"] = self._pages[
+                        current_url_ref["url"]
+                    ].get("assets", [])
+                current_url_ref["url"] = url
+                await asyncio.wait_for(
+                    self._process_page(page, url, depth),
+                    timeout=self.page_timeout,
+                )
+                await asyncio.sleep(0.3)
+            except asyncio.TimeoutError:
+                logger.warning(f"Timeout processing {url}")
+            except Exception as e:
+                logger.error(f"Error processing {url}: {e}")
+            finally:
+                self._busy -= 1
+                self._queue.task_done()
+
+    async def run(self):
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(
+                headless=True,
+                args=["--disable-dev-shm-usage"],
+                executable_path="/usr/bin/chromium-browser",
+            )
+            context = await browser.new_context(
+                user_agent=(
+                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+                    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+                ),
+                viewport={"width": 1920, "height": 1080},
+            )
+            await context.add_init_script(
+                "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
+            )
+            pages = []
+            for i in range(self.max_parallel):
+                pg = await context.new_page()
+                pg.set_default_timeout(self.timeout)
+                pages.append(pg)
+            logger.info(
+                f"Scraping {self.url} (depth={self.max_depth}, parallel={self.max_parallel})"
+            )
+            await self._queue.put((self.url, 0))
+            workers = [
+                asyncio.create_task(self._worker(i, pages))
+                for i in range(self.max_parallel)
+            ]
+            watchdog = 0
+            try:
+                while True:
+                    await asyncio.sleep(0.5)
+                    watchdog += 1
+                    if watchdog >= 300:
+                        logger.warning("Watchdog: queue not draining")
+                        watchdog = 0
+                    if self._queue.empty() and self._busy == 0:
+                        for _ in range(self.max_parallel):
+                            await self._queue.put(None)
+                        break
+                    if self.max_pages is not None and self.page_count >= self.max_pages:
+                        for _ in range(self.max_parallel):
+                            await self._queue.put(None)
+                        break
+                await asyncio.gather(*workers)
+            except Exception:
+                logger.warning("Browser crashed, stopping scrape")
+                for _ in range(self.max_parallel):
+                    await self._queue.put(None)
+                await asyncio.gather(*workers, return_exceptions=True)
+            logger.info(f"Done: {self.page_count} pages, {self.asset_count} assets")
+            try:
+                await browser.close()
+            except Exception:
+                pass