From 61f0ac03db44542abcc5f2f362778101dcf0305c Mon Sep 17 00:00:00 2001 From: Ethan Werner Date: Wed, 12 Aug 2026 14:34:58 +0400 Subject: initial commit --- .gitignore | 7 + embedding/embedder.py | 124 ++++++ embedding/server.py | 111 ++++++ news.code-workspace | 8 + news_sites.json | 898 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + scraper/__init__.py | 0 scraper/__main__.py | 53 +++ scraper/spiders/__init__.py | 0 scraper/spiders/page.py | 450 ++++++++++++++++++++++ 10 files changed, 1653 insertions(+) create mode 100644 .gitignore create mode 100644 embedding/embedder.py create mode 100644 embedding/server.py create mode 100644 news.code-workspace create mode 100644 news_sites.json create mode 100644 requirements.txt create mode 100644 scraper/__init__.py create mode 100644 scraper/__main__.py create mode 100644 scraper/spiders/__init__.py create mode 100644 scraper/spiders/page.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..94fe76f --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +data/ +__pycache__/ +*.pyc +**/.content_hash +.ruff_cache/ +*.code-workspace diff --git a/embedding/embedder.py b/embedding/embedder.py new file mode 100644 index 0000000..db353ab --- /dev/null +++ b/embedding/embedder.py @@ -0,0 +1,124 @@ +import os +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image +from transformers import ( + AutoModel, + BertTokenizer, + SiglipModel, + SiglipProcessor, + WhisperModel, + WhisperFeatureExtractor, +) + + +class CompactEmbedder(nn.Module): + def __init__(self, local_dir="./multi-modal-embed-small"): + local_dir = os.path.abspath(local_dir) + super().__init__() + text_path = os.path.normpath(f"{local_dir}/../base-models/all-MiniLM-L6-v2") + image_path = os.path.normpath( + f"{local_dir}/../base-models/siglip-base-patch16-512" + ) + audio_path = os.path.normpath(f"{local_dir}/../base-models/whisper-tiny") + + self.text_tokenizer = BertTokenizer.from_pretrained( + text_path, local_files_only=True + ) + self.text_encoder = AutoModel.from_pretrained(text_path, local_files_only=True) + + self.image_processor = SiglipProcessor.from_pretrained( + image_path, local_files_only=True + ) + self.image_encoder = SiglipModel.from_pretrained( + image_path, local_files_only=True + ).vision_model + self.audio_processor = WhisperFeatureExtractor.from_pretrained( + audio_path, local_files_only=True + ) + self.image_proj = nn.Linear(768, 384) + self.audio_processor = WhisperFeatureExtractor.from_pretrained( + audio_path, local_files_only=True + ) + self.audio_encoder = WhisperModel.from_pretrained( + audio_path, local_files_only=True + ).encoder + + checkpoint_path = f"{local_dir}/model.pt" + state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=True) + + self.text_encoder.load_state_dict( + { + k.replace("text_encoder.encoder.", ""): v + for k, v in state_dict.items() + if k.startswith("text_encoder.encoder.") + } + ) + + self.image_encoder.load_state_dict( + { + k.replace("image_encoder.vision_encoder.", ""): v + for k, v in state_dict.items() + if k.startswith("image_encoder.vision_encoder.") + } + ) + + self.image_proj.load_state_dict( + { + k.replace("image_encoder.projection.", ""): v + for k, v in state_dict.items() + if k.startswith("image_encoder.projection.") + } + ) + + self.audio_encoder.load_state_dict( + { + k.replace("audio_encoder.encoder.", ""): v + for k, v in state_dict.items() + if k.startswith("audio_encoder.encoder.") + } + ) + + self.eval() + + @torch.no_grad() + def encode_text(self, texts): + if isinstance(texts, str): + texts = [texts] + inputs = self.text_tokenizer( + texts, padding=True, truncation=True, return_tensors="pt" + ) + inputs = {k: v.to(next(self.parameters()).device) for k, v in inputs.items()} + outputs = self.text_encoder(**inputs) + embeddings = outputs.last_hidden_state.mean(dim=1) + return F.normalize(embeddings, p=2, dim=-1) + + @torch.no_grad() + def encode_image(self, images): + if isinstance(images, Image.Image): + images = [images] + inputs = self.image_processor(images=images, return_tensors="pt") + inputs = {k: v.to(next(self.parameters()).device) for k, v in inputs.items()} + outputs = self.image_encoder(**inputs) + embeddings = self.image_proj(outputs.pooler_output) + return F.normalize(embeddings, p=2, dim=-1) + + @torch.no_grad() + def encode_audio(self, waveform): + if isinstance(waveform, torch.Tensor): + waveform = waveform.squeeze().numpy() + inputs = self.audio_processor( + waveform, sampling_rate=16000, return_tensors="pt" + ) + inputs = {k: v.to(next(self.parameters()).device) for k, v in inputs.items()} + outputs = self.audio_encoder(**inputs) + embeddings = outputs.last_hidden_state.mean(dim=1) + return F.normalize(embeddings, p=2, dim=-1) + + +if __name__ == "__main__": + model = CompactEmbedder(local_dir="./models/multi-modal-embed-small") + + text_vec = model.encode_text("example query") + print(f"Text shape: {text_vec.shape}") diff --git a/embedding/server.py b/embedding/server.py new file mode 100644 index 0000000..b58f041 --- /dev/null +++ b/embedding/server.py @@ -0,0 +1,111 @@ +import base64 +import io +import os +import sys +from typing import Any + +import numpy as np +import torch +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel +from PIL import Image + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from embedding.embedder import CompactEmbedder + +app = FastAPI() +model = None + + +@app.on_event("startup") +def load_model(): + global model + model = CompactEmbedder(local_dir="./models/multi-modal-embed-small") + + +class InlineData(BaseModel): + mime_type: str + data: str + + +class Part(BaseModel): + text: str | None = None + inline_data: InlineData | None = None + + +class Content(BaseModel): + parts: list[Part] + + +class EmbedContentRequest(BaseModel): + model: str = "models/multi-modal-embed-small" + content: Content + output_dimensionality: int | None = None + task_type: str | None = None + title: str | None = None + + +class BatchEmbedContentRequest(BaseModel): + requests: list[EmbedContentRequest] + + +def _encode_content(content: Content) -> list[float]: + vecs: list[Any] = [] + for part in content.parts: + if part.text: + vecs.append(model.encode_text(part.text)) + if part.inline_data: + raw = base64.b64decode(part.inline_data.data) + mime = part.inline_data.mime_type + if mime.startswith("image/"): + img = Image.open(io.BytesIO(raw)).convert("RGB") + vecs.append(model.encode_image(img)) + elif mime.startswith("audio/"): + audio = np.frombuffer(raw, dtype=np.float32) + vecs.append(model.encode_audio(audio)) + if not vecs: + vecs.append(model.encode_text("")) + vec = vecs[0] if len(vecs) == 1 else torch.mean(torch.stack(vecs), dim=0) + return vec.squeeze(0).tolist() + + +class EmbeddingResponse(BaseModel): + embedding: dict[str, list[float]] + + +class EmbedContentResponse(BaseModel): + embedding: dict[str, list[float]] + + +class BatchEmbedContentResponse(BaseModel): + embeddings: list[dict[str, list[float]]] + + +@app.post("/v1beta/models/{model_id}:embedContent") +async def embed_content( + model_id: str, + req: EmbedContentRequest, +): + emb = _encode_content(req.content) + if req.output_dimensionality: + emb = emb[: req.output_dimensionality] + return {"embedding": {"values": emb}} + + +@app.post("/v1beta/models/{model_id}:batchEmbedContents") +async def batch_embed_contents( + model_id: str, + req: BatchEmbedContentRequest, +): + embeddings = [] + for sub_req in req.requests: + emb = _encode_content(sub_req.content) + if sub_req.output_dimensionality: + emb = emb[: sub_req.output_dimensionality] + embeddings.append({"values": emb}) + return {"embeddings": embeddings} + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/news.code-workspace b/news.code-workspace new file mode 100644 index 0000000..876a149 --- /dev/null +++ b/news.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/news_sites.json b/news_sites.json new file mode 100644 index 0000000..8c657a4 --- /dev/null +++ b/news_sites.json @@ -0,0 +1,898 @@ +[ + { + "name": "Yahoo News", + "url": { + "main": "https://news.yahoo.com", + "rss": "https://rss.news.yahoo.com/rss/topstories" + } + }, + { + "name": "BBC News", + "url": { + "main": "https://www.bbc.com/news", + "rss": "http://feeds.bbci.co.uk/news/rss.xml" + } + }, + { + "name": "CNN", + "url": { + "main": "https://www.cnn.com", + "rss": "https://rss.cnn.com/rss/edition.rss" + } + }, + { + "name": "Al Jazeera English", + "url": { + "main": "https://www.aljazeera.com", + "rss": "https://www.aljazeera.com/xml/rss/all.xml" + } + }, + { + "name": "The Guardian", + "url": { + "main": "https://www.theguardian.com", + "rss": "https://feeds.theguardian.com/theguardian/world/rss" + } + }, + { + "name": "Fox News", + "url": { + "main": "https://www.foxnews.com", + "rss": "https://feed.feedburner.com/foxnews" + } + }, + { + "name": "Reuters", + "url": { + "main": "https://www.reuters.com", + "rss": "https://www.reutersagency.com/feed/?best-topics=all&posts_type=best" + } + }, + { + "name": "France 24", + "url": { + "main": "https://www.france24.com", + "rss": "https://www.france24.com/en/feed/rss/news" + } + }, + { + "name": "The New York Times", + "url": { + "main": "https://www.nytimes.com", + "rss": "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml" + } + }, + { + "name": "The Independent", + "url": { + "main": "https://www.independent.co.uk", + "rss": "https://www.independent.co.uk/rss" + } + }, + { + "name": "DW News", + "url": { + "main": "https://www.dw.com", + "rss": "https://rss.dw.com/xml/rss-en-all" + } + }, + { + "name": "The Washington Post", + "url": { + "main": "https://www.washingtonpost.com", + "rss": "https://feeds.washingtonpost.com/rss/rss" + } + }, + { + "name": "NBC News", + "url": { + "main": "https://www.nbcnews.com", + "rss": "https://feeds.nbcnews.com/nbcnews/public/tnbc/topstories" + } + }, + { + "name": "CBS News", + "url": { + "main": "https://www.cbsnews.com", + "rss": "https://www.cbsnews.com/latest/" + } + }, + { + "name": "ABC News", + "url": { + "main": "https://abcnews.go.com", + "rss": "https://abcnews.go.com/abcnews/keynotes" + } + }, + { + "name": "MSNBC", + "url": { + "main": "https://www.msnbc.com", + "rss": "https://www.msnbc.com/id/msnbc/rss" + } + }, + { + "name": "Forbes", + "url": { + "main": "https://www.forbes.com", + "rss": "https://www.forbes.com/forbes/feed/rss.xml" + } + }, + { + "name": "Business Insider", + "url": { + "main": "https://www.businessinsider.com", + "rss": "https://www.businessinsider.com/rss" + } + }, + { + "name": "The Times of India", + "url": { + "main": "https://timesofindia.indiatimes.com", + "rss": "http://timesofindia.indiatimes.com/rssfeedstopstories.cms" + } + }, + { + "name": "Euronews", + "url": { + "main": "https://www.euronews.com", + "rss": "https://www.euronews.com/rss" + } + }, + { + "name": "Al Arabiya", + "url": { + "main": "https://www.alarabiya.net", + "rss": "https://www.alarabiya.net/feed" + } + }, + { + "name": "Sky News", + "url": { + "main": "https://news.sky.com", + "rss": "https://news.sky.com/rss" + } + }, + { + "name": "NDTV", + "url": { + "main": "https://www.ndtv.com", + "rss": "https://feeds.feedburner.com/ndtvnews-topstories" + } + }, + { + "name": "India Today", + "url": { + "main": "https://www.indiatoday.in", + "rss": "https://www.indiatoday.in/rss" + } + }, + { + "name": "HuffPost", + "url": { + "main": "https://www.huffpost.com", + "rss": "https://www.huffpost.com/feeds/index" + } + }, + { + "name": "The Telegraph", + "url": { + "main": "https://www.telegraph.co.uk", + "rss": "https://www.telegraph.co.uk/rss.xml" + } + }, + { + "name": "El Pa\u00eds", + "url": { + "main": "https://elpais.com", + "rss": "https://feeds.elpais.com/elpais/compact/portada.xml" + } + }, + { + "name": "Politico", + "url": { + "main": "https://www.politico.com", + "rss": "https://www.politico.com/rss/political-calculate.xml" + } + }, + { + "name": "CNBC", + "url": { + "main": "https://www.cnbc.com", + "rss": "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114" + } + }, + { + "name": "The Hindu", + "url": { + "main": "https://www.thehindu.com", + "rss": "https://www.thehindu.com/news/national/feeder/default.rss" + } + }, + { + "name": "Le Monde", + "url": { + "main": "https://www.lemonde.fr", + "rss": "https://www.lemonde.fr/rss/une.xml" + } + }, + { + "name": "Bloomberg", + "url": { + "main": "https://www.bloomberg.com", + "rss": "https://feeds.bloomberg.com/markets/news.rss" + } + }, + { + "name": "The Economist", + "url": { + "main": "https://www.economist.com", + "rss": "https://www.economist.com/rss" + } + }, + { + "name": "ABC News Australia", + "url": { + "main": "https://www.abc.net.au/news", + "rss": "https://www.abc.net.au/feed/latestnews/rss.xml" + } + }, + { + "name": "Corriere della Sera", + "url": { + "main": "https://www.corriere.it", + "rss": "https://feedfeed.corriere.it/rss.xml" + } + }, + { + "name": "Bild", + "url": { + "main": "https://www.bild.de", + "rss": "https://www.bild.de/rssfeeds/top/rss-feeder.top.aufkleber.ilw.xml" + } + }, + { + "name": "Der Spiegel", + "url": { + "main": "https://www.spiegel.de", + "rss": "https://www.spiegel.de/schlagzeilen/index.rss" + } + }, + { + "name": "Daily Mail", + "url": { + "main": "https://www.dailymail.co.uk", + "rss": "https://www.dailymail.co.uk/indices/news-rss/index.rss" + } + }, + { + "name": "Le Figaro", + "url": { + "main": "https://www.lefigaro.fr", + "rss": "https://www.lefigaro.fr/rss/figaro.xml" + } + }, + { + "name": "CBC News", + "url": { + "main": "https://www.cbc.ca/news", + "rss": "https://rss.cbc.ca/readerfeed/cc/rss/NewsTopStories.xml" + } + }, + { + "name": "USA Today", + "url": { + "main": "https://www.usatoday.com", + "rss": "https://www.usatoday.com/rss" + } + }, + { + "name": "Wall Street Journal", + "url": { + "main": "https://www.wsj.com", + "rss": "https://feeds.a.dj.com/rss/RSSWorldNews.xml" + } + }, + { + "name": "NHK World Japan", + "url": { + "main": "https://www3.nhk.or.jp/nhkworld", + "rss": "https://www3.nhk.or.jp/news/xml/lw_news_rss.xml" + } + }, + { + "name": "NPR", + "url": { + "main": "https://www.npr.org", + "rss": "https://feeds.npr.org/1001/rss.xml" + } + }, + { + "name": "SBS News", + "url": { + "main": "https://www.sbs.com.au/news", + "rss": "https://www.sbs.com.au/news/feed" + } + }, + { + "name": "The Australian", + "url": { + "main": "https://www.theaustralian.com.au", + "rss": "https://www.theaustralian.com.au/rss/feed" + } + }, + { + "name": "Nikkei Asia", + "url": { + "main": "https://asia.nikkei.com", + "rss": "https://asia.nikkei.com/feed" + } + }, + { + "name": "Financial Times", + "url": { + "main": "https://www.ft.com", + "rss": "https://www.ft.com/rss/home" + } + }, + { + "name": "RT", + "url": { + "main": "https://www.rt.com", + "rss": "https://www.rt.com/rss/" + } + }, + { + "name": "The Japan Times", + "url": { + "main": "https://www.japantimes.co.jp", + "rss": "https://www.japantimes.co.jp/feed/" + } + }, + { + "name": "S\u00fcddeutsche Zeitung", + "url": { + "main": "https://www.sueddeutsche.de", + "rss": "https://rss.sueddeutsche.de/alles" + } + }, + { + "name": "La Repubblica", + "url": { + "main": "https://www.repubblica.it", + "rss": "https://feed.pressreader.com/it/la-repubblica" + } + }, + { + "name": "Frankfurter Allgemeine", + "url": { + "main": "https://www.faz.net", + "rss": "https://www.faz.net/rss/aktuell" + } + }, + { + "name": "iNews", + "url": { + "main": "https://inews.co.uk", + "rss": "https://inews.co.uk/feed/" + } + }, + { + "name": "Metro UK", + "url": { + "main": "https://metro.co.uk", + "rss": "https://metro.co.uk/feed/" + } + }, + { + "name": "The Atlantic", + "url": { + "main": "https://www.theatlantic.com", + "rss": "https://www.theatlantic.com/feeds/latest" + } + }, + { + "name": "Vox", + "url": { + "main": "https://www.vox.com", + "rss": "https://www.vox.com/rss/index.xml" + } + }, + { + "name": "Time", + "url": { + "main": "https://time.com", + "rss": "https://time.com/time_rss_100.xml" + } + }, + { + "name": "Newsweek", + "url": { + "main": "https://www.newsweek.com", + "rss": "https://www.newsweek.com/rss" + } + }, + { + "name": "The Verge", + "url": { + "main": "https://www.theverge.com", + "rss": "https://www.theverge.com/rss/index.xml" + } + }, + { + "name": "TechCrunch", + "url": { + "main": "https://techcrunch.com", + "rss": "https://techcrunch.com/feed/" + } + }, + { + "name": "Ars Technica", + "url": { + "main": "https://arstechnica.com", + "rss": "https://feeds.arstechnica.com/arstechnica/index" + } + }, + { + "name": "The Hill", + "url": { + "main": "https://thehill.com", + "rss": "https://thehill.com/feed/" + } + }, + { + "name": "Axios", + "url": { + "main": "https://www.axios.com", + "rss": "https://www.axios.com/rss-feeds" + } + }, + { + "name": "Wired", + "url": { + "main": "https://www.wired.com", + "rss": "https://www.wired.com/feed/rss" + } + }, + { + "name": "The Daily Beast", + "url": { + "main": "https://www.thedailybeast.com", + "rss": "https://www.thedailybeast.com/rss" + } + }, + { + "name": "ProPublica", + "url": { + "main": "https://www.propublica.org", + "rss": "https://www.propublica.org/feed" + } + }, + { + "name": "South China Morning Post", + "url": { + "main": "https://www.scmp.com", + "rss": "https://www.scmp.com/rss" + } + }, + { + "name": "The Straits Times", + "url": { + "main": "https://www.straitstimes.com", + "rss": "https://www.straitstimes.com/rss" + } + }, + { + "name": "The Globe and Mail", + "url": { + "main": "https://www.theglobeandmail.com", + "rss": "https://www.theglobeandmail.com/rss/latest.headlines.xml" + } + }, + { + "name": "The Irish Times", + "url": { + "main": "https://www.irishtimes.com", + "rss": "https://www.irishtimes.com/rss" + } + }, + { + "name": "Le Soir", + "url": { + "main": "https://www.lesoir.be", + "rss": "https://www.lesoir.be/rss" + } + }, + { + "name": "ABC Espa\u00f1a", + "url": { + "main": "https://www.abc.es", + "rss": "https://www.abc.es/rss" + } + }, + { + "name": "El Mundo", + "url": { + "main": "https://www.elmundo.es", + "rss": "https://www.elmundo.es/elmundo/rss/portada.xml" + } + }, + { + "name": "L'Express", + "url": { + "main": "https://www.lexpress.fr", + "rss": "https://www.lexpress.fr/feeds/flux-rss-une.xml" + } + }, + { + "name": "Clar\u00edn", + "url": { + "main": "https://www.clarin.com", + "rss": "https://www.clarin.com/feed/" + } + }, + { + "name": "Folha de S.Paulo", + "url": { + "main": "https://www1.folha.uol.com.br", + "rss": "https://www.folha.uol.com.br/podcasts/feed" + } + }, + { + "name": "O Globo", + "url": { + "main": "https://oglobo.globo.com", + "rss": "https://oglobo.globo.com/rss" + } + }, + { + "name": "The Korea Herald", + "url": { + "main": "https://www.koreaherald.com", + "rss": "https://www.koreaherald.com/rss" + } + }, + { + "name": "The National", + "url": { + "main": "https://www.thenationalnews.com", + "rss": "https://www.thenationalnews.com/rss" + } + }, + { + "name": "Meduza", + "url": { + "main": "https://meduza.io", + "rss": "https://meduza.io/feed" + } + }, + { + "name": "AFP", + "url": { + "main": "https://www.afp.com", + "rss": "https://www.afp.com/en/feed" + } + }, + { + "name": "Variety", + "url": { + "main": "https://variety.com", + "rss": "https://variety.com/feed/" + } + }, + { + "name": "Rolling Stone", + "url": { + "main": "https://www.rollingstone.com", + "rss": "https://www.rollingstone.com/feed/" + } + }, + { + "name": "Foreign Policy", + "url": { + "main": "https://foreignpolicy.com", + "rss": "https://foreignpolicy.com/feed/" + } + }, + { + "name": "News.com.au", + "url": { + "main": "https://www.news.com.au", + "rss": "https://www.news.com.au/rss" + } + }, + { + "name": "The Times", + "url": { + "main": "https://www.thetimes.co.uk", + "rss": "https://www.thetimes.co.uk/rss" + } + }, + { + "name": "Handelsblatt", + "url": { + "main": "https://www.handelsblatt.com", + "rss": "https://www.handelsblatt.com/feeds/rss" + } + }, + { + "name": "Die Welt", + "url": { + "main": "https://www.welt.de", + "rss": "https://www.welt.de/rss/local.php" + } + }, + { + "name": "Il Giornale", + "url": { + "main": "https://www.ilgiornale.it", + "rss": "https://www.ilgiornale.it/feeds/rss" + } + }, + { + "name": "The Sydney Morning Herald", + "url": { + "main": "https://www.smh.com.au", + "rss": "https://www.smh.com.au/rss/headlines" + } + }, + { + "name": "Al Jazeera Arabic", + "url": { + "main": "https://www.aljazeera.net", + "rss": "https://www.aljazeera.com/xml/rss/all.xml" + } + }, + { + "name": "Deccan Chronicle", + "url": { + "main": "https://www.deccanchronicle.com", + "rss": "https://www.deccanchronicle.com/feed" + } + }, + { + "name": "Mint", + "url": { + "main": "https://www.livemint.com", + "rss": "https://www.livemint.com/rss/" + } + }, + { + "name": "Business Standard", + "url": { + "main": "https://www.business-standard.com", + "rss": "https://www.business-standard.com/rss" + } + }, + { + "name": "The Hindu BusinessLine", + "url": { + "main": "https://www.thehindubusinessline.com", + "rss": "https://www.thehindubusinessline.com/rss" + } + }, + { + "name": "La Naci\u00f3n", + "url": { + "main": "https://www.lanacion.com.ar", + "rss": "https://www.lanacion.com.ar/rss" + } + }, + { + "name": "O Estado de S. Paulo", + "url": { + "main": "https://www.estadao.com.br", + "rss": "https://www.estadao.com.br/rss/" + } + }, + { + "name": "Le Devoir", + "url": { + "main": "https://www.ledevoir.com", + "rss": "https://www.ledevoir.com/rss" + } + }, + { + "name": "Korea Times", + "url": { + "main": "https://www.koreatimes.co.kr", + "rss": "https://www.koreatimes.co.kr/rss" + } + }, + { + "name": "Nikkei", + "url": { + "main": "https://www.nikkei.com", + "rss": "https://www.nikkei.com/feed/" + } + }, + { + "name": "Daily Nation", + "url": { + "main": "https://www.nation.co.ke", + "rss": "https://www.nation.co.ke/rss/" + } + }, + { + "name": "The Print", + "url": { + "main": "https://theprint.in", + "rss": "https://theprint.in/feed/" + } + }, + { + "name": "The Quint", + "url": { + "main": "https://www.thequint.com", + "rss": "https://www.thequint.com/feed" + } + }, + { + "name": "WION", + "url": { + "main": "https://www.wionews.com", + "rss": "https://www.wionews.com/rss" + } + }, + { + "name": "The Conversation", + "url": { + "main": "https://theconversation.com", + "rss": "https://theconversation.com/feed" + } + }, + { + "name": "The Wire", + "url": { + "main": "https://thewire.in", + "rss": "https://thewire.in/feed/" + } + }, + { + "name": "The Guardian Australia", + "url": { + "main": "https://www.theguardian.com/au", + "rss": "https://feeds.theguardian.com/theguardian/australia-news/rss" + } + }, + { + "name": "The Guardian Canada", + "url": { + "main": "https://www.theguardian.com/au", + "rss": "https://feeds.theguardian.com/theguardian/us-news/rss" + } + }, + { + "name": "El Pa\u00eds Semanal", + "url": { + "main": "https://semanal.elpais.com", + "rss": "https://feeds.elpais.com/elpais/compact/portada.xml" + } + }, + { + "name": "BBC Afrikaans", + "url": { + "main": "https://www.bbc.com/afrikaans", + "rss": "https://www.bbc.com/news/world/africa/rss.xml" + } + }, + { + "name": "Africa News", + "url": { + "main": "https://africanews.com", + "rss": "https://africanews.com/feed/" + } + }, + { + "name": "Pagina/12", + "url": { + "main": "https://www.pagina12.com.ar", + "rss": "https://www.pagina12.com.ar/rss" + } + }, + { + "name": "The Malaysian Reserve", + "url": { + "main": "https://www.malaysianreserve.com", + "rss": "https://www.malaysianreserve.com/feed" + } + }, + { + "name": "The Standard", + "url": { + "main": "https://www.thestandard.com.hk", + "rss": "https://www.thestandard.com.hk/feed" + } + }, + { + "name": "Manila Bulletin", + "url": { + "main": "https://mb.com.ph", + "rss": "https://mb.com.ph/feed/" + } + }, + { + "name": "The New Humanist", + "url": { + "main": "https://the-new-humanist.co.uk", + "rss": "https://www.the-new-humanist.co.uk/feed" + } + }, + { + "name": "The Canary", + "url": { + "main": "https://thecanary.co", + "rss": "https://thecanary.co/feed/" + } + }, + { + "name": "The Free Press", + "url": { + "main": "https://www.thefreepress.com", + "rss": "https://www.thefreepress.com/feed" + } + }, + { + "name": "Reason", + "url": { + "main": "https://reason.com", + "rss": "https://reason.com/feed/" + } + }, + { + "name": "The Daily Caller", + "url": { + "main": "https://dailycaller.com", + "rss": "https://dailycaller.com/feed/" + } + }, + { + "name": "The Federalist", + "url": { + "main": "https://thefederalist.com", + "rss": "https://thefederalist.com/feed/" + } + }, + { + "name": "The Nation", + "url": { + "main": "https://www.thenation.com", + "rss": "https://www.thenation.com/feed/" + } + }, + { + "name": "Dissent", + "url": { + "main": "https://www.dissentmagazine.org", + "rss": "https://www.dissentmagazine.org/feed" + } + }, + { + "name": "The New Republic", + "url": { + "main": "https://newrepublic.com", + "rss": "https://newrepublic.com/feed/" + } + }, + { + "name": "Harper's Magazine", + "url": { + "main": "https://harpers.org", + "rss": "https://harpers.org/feed/" + } + }, + { + "name": "The Nation (UK)", + "url": { + "main": "https://www.nation.come", + "rss": "https://www.thenation.co.uk/feed" + } + }, + { + "name": "Al-Monitor", + "url": { + "main": "https://www.al-monitor.com", + "rss": "https://www.al-monitor.com/rss" + } + } +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..070fc35 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +playwright +pyquery diff --git a/scraper/__init__.py b/scraper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scraper/__main__.py b/scraper/__main__.py new file mode 100644 index 0000000..329b102 --- /dev/null +++ b/scraper/__main__.py @@ -0,0 +1,53 @@ +import asyncio +import argparse +import shutil +from scraper.spiders.page import PageScraper + + +def main(): + parser = argparse.ArgumentParser(description="Scrape pages") + parser.add_argument("url", help="URL to scrape") + parser.add_argument("-o", "--output", default="data", help="Output directory") + parser.add_argument( + "-d", "--depth", type=int, required=True, help="Link traversal depth" + ) + parser.add_argument( + "-c", + "--clean", + action="store_true", + help="Remove output directory before scraping", + ) + parser.add_argument("-n", "--max-pages", type=int, help="Max pages to scrape") + parser.add_argument("-a", "--max-assets", type=int, help="Max assets to download") + parser.add_argument("-p", "--parallel", type=int, help="Number of parallel workers") + parser.add_argument( + "-t", "--timeout", type=int, help="Playwright default timeout in ms" + ) + parser.add_argument("--page-timeout", type=int, help="Per-page timeout in seconds") + args = parser.parse_args() + + if args.clean: + shutil.rmtree(args.output, ignore_errors=True) + + scraper_kwargs = { + "url": args.url, + "output_dir": args.output, + "max_depth": args.depth, + } + if args.max_pages is not None: + scraper_kwargs["max_pages"] = args.max_pages + if args.max_assets is not None: + scraper_kwargs["max_assets"] = args.max_assets + if args.parallel is not None: + scraper_kwargs["max_parallel"] = args.parallel + if args.timeout is not None: + scraper_kwargs["timeout"] = args.timeout + if args.page_timeout is not None: + scraper_kwargs["page_timeout"] = args.page_timeout + + scraper = PageScraper(**scraper_kwargs) + asyncio.run(scraper.run()) + + +if __name__ == "__main__": + main() diff --git a/scraper/spiders/__init__.py b/scraper/spiders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scraper/spiders/page.py b/scraper/spiders/page.py new file mode 100644 index 0000000..4edcfa6 --- /dev/null +++ b/scraper/spiders/page.py @@ -0,0 +1,450 @@ +import asyncio +import datetime +import hashlib +import json +import logging +import os +import re +import urllib.parse +from pyquery import PyQuery as pq +from playwright.async_api import async_playwright + +logger = logging.getLogger(__name__) + +UNLIMITED = 2**31 + + +class PageScraper: + def __init__( + self, + url, + output_dir="data", + max_depth=5, + max_pages=None, + max_assets=None, + max_parallel=1, + timeout=60000, + page_timeout=120, + ): + self.url = url + self.output_dir = output_dir + self.max_depth = max_depth + self.max_pages = max_pages + self.max_assets = max_assets + self.max_parallel = max_parallel + self.timeout = timeout + self.page_timeout = page_timeout + self.page_count = 0 + self.asset_count = 0 + self.enqueued = set() + self._pages = {} + self._queue = asyncio.Queue() + self._busy = 0 + self._origin_base = ".".join( + urllib.parse.urlparse(url).netloc.rstrip(".").split(".")[-2:] + ) + + def _url_dir(self, url): + parsed = urllib.parse.urlparse(url) + path = parsed.path.strip("/") + return ( + os.path.join(self.output_dir, parsed.netloc, path) + if path + else os.path.join(self.output_dir, parsed.netloc) + ) + + def _read_meta(self, url): + meta_path = os.path.join(self._url_dir(url), "initial.json") + if not os.path.isfile(meta_path): + return None + try: + with open(meta_path) as f: + return json.load(f) + except (json.JSONDecodeError, FileNotFoundError): + return None + + def _content_changed(self, url, content): + new_hash = hashlib.sha256(content.encode()).hexdigest() + previous = self._read_meta(url) + if previous: + return previous.get("content", {}).get("initial_sha256") != new_hash + return True + + def _is_asset(self, resource_type): + return resource_type in ( + "stylesheet", + "image", + "media", + "font", + "script", + "fetch", + "xhr", + "other", + ) + + def _asset_path(self, url, content_type): + parsed = urllib.parse.urlparse(url) + netloc = parsed.netloc or parsed.path.split("/")[0] + path = parsed.path.strip("/") if parsed.path else netloc + if path.startswith(netloc): + path = "/" + path[len(netloc) :] + ext_map = { + "image/png": "png", + "image/jpeg": "jpg", + "image/webp": "webp", + "image/gif": "gif", + "image/svg+xml": "svg", + "image/x-icon": "ico", + "application/javascript": "js", + "text/javascript": "js", + "text/css": "css", + "font/woff2": "woff2", + "font/woff": "woff", + "application/json": "json", + "application/pdf": "pdf", + } + _, ext = os.path.splitext(path) + if ext: + return os.path.join(self.output_dir, netloc, path) + mime_ext = ext_map.get(content_type.split(";")[0].strip().lower()) + if mime_ext: + return os.path.join(self.output_dir, netloc, path, f"index.{mime_ext}") + return os.path.join(self.output_dir, netloc, path, "index.dat") + + async def _on_response(self, response, current_url_ref): + try: + url = response.url + if not self._is_asset(response.request.resource_type): + return + if self.max_assets is not None and self.asset_count >= self.max_assets: + return + content_type = response.headers.get("content-type", "") + asset_path = self._asset_path(url, content_type) + if os.path.isfile(asset_path): + return + self.asset_count += 1 + try: + body = await response.body() + except Exception: + return + try: + os.makedirs(os.path.dirname(asset_path), exist_ok=True) + except OSError: + return + with open(asset_path, "wb") as f: + f.write(body) + asset_hash = hashlib.sha256(body).hexdigest() + meta = { + "url": url, + "request": { + "method": response.request.method, + "headers": dict(response.request.headers), + }, + "response": { + "status": response.status, + "headers": dict(response.headers), + }, + "content": {"sha256": asset_hash}, + "scrape": { + "timestamp": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(), + }, + } + with open(asset_path + ".json", "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) + page_url = current_url_ref.get() + if page_url and page_url in self._pages: + self._pages[page_url]["assets"].append( + { + "url": url, + } + ) + except Exception: + pass + + def _extract_article_links(self, doc, page_url): + links = set() + skip_patterns = [ + re.compile( + r"^/(about|contact|privacy|terms|ad|subscribe|newsletter|" + r"login|signup|careers|feedback|site|help|support|faq|" + r"sitemap|robots|feed|rss|api|static|assets|img|images|" + r"css|js|scripts|fonts|cdn|media|video|audio)$" + ), + re.compile( + r"\.(png|jpg|jpeg|gif|svg|ico|css|js|woff|woff2|ttf|pdf|xml|json|html)$" + ), + re.compile(r"^#"), + re.compile(r"^mailto:"), + re.compile(r"^javascript:"), + re.compile(r"^/$"), + ] + for a in doc("a").items(): + href = a.attr("href") or "" + resolved = urllib.parse.urljoin(page_url, href) + parsed = urllib.parse.urlparse(resolved) + parsed_base = ".".join(parsed.netloc.rstrip(".").split(".")[-2:]) + if parsed_base != self._origin_base: + continue + path = parsed.path.strip("/") + if not path: + continue + if any(s.search(parsed.path) for s in skip_patterns): + continue + segments = [s for s in path.split("/") if s] + if len(segments) >= 2: + links.add(resolved) + links.discard(self.url) + for seen in self.enqueued: + links.discard(seen) + return links + + async def _process_page(self, page, page_url, depth): + headers = {} + previous = self._read_meta(page_url) + if previous: + etag = previous.get("response", {}).get("etag") + if etag: + headers["if-none-match"] = etag + timestamp = previous.get("scrape", {}).get("timestamp") + if timestamp: + headers["if-modified-since"] = timestamp + try: + await page.set_extra_http_headers(headers) + resp = await page.goto(page_url, wait_until="domcontentloaded") + await page.set_extra_http_headers({}) + except Exception as e: + logger.error(f"Failed to navigate to {page_url}: {e}") + return + page_url = resp.url + if resp.request.redirected_from: + redirect_url = resp.request.redirected_from.url + page_dir = self._url_dir(redirect_url) + meta_path = os.path.join(page_dir, "initial.json") + if not os.path.exists(meta_path): + os.makedirs(page_dir, exist_ok=True) + prev_resp = resp.request.redirected_from.existing_response + meta = { + "request": { + "method": resp.request.method, + "url": redirect_url, + "final_url": page_url, + "redirect_chain": [], + "headers": dict(resp.request.headers), + }, + "response": { + "status": prev_resp.status if prev_resp else None, + "headers": dict(resp.headers), + }, + "content": {"sha256": ""}, + "scrape": { + "timestamp": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(), + }, + } + with open(meta_path, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) + if resp.status == 304: + page_dir = self._url_dir(page_url) + initial_path = os.path.join(page_dir, "initial.html") + if os.path.isfile(initial_path): + with open(initial_path, encoding="utf-8") as f: + local_html = f.read() + await self._enqueue_links(local_html, page_url, depth) + return + initial_html = await page.content() + content_hash = hashlib.sha256(initial_html.encode()).hexdigest() + changed = self._content_changed(page_url, initial_html) + + if changed: + at_limit = self.max_pages is not None and self.page_count >= self.max_pages + if at_limit: + return + self.page_count += 1 + logger.info(f"Scraping page {self.page_count}: {page_url}") + page_dir = self._url_dir(page_url) + os.makedirs(page_dir, exist_ok=True) + with open( + os.path.join(page_dir, "initial.html"), "w", encoding="utf-8" + ) as f: + f.write(initial_html) + + await self._wait_for_hydrated(page) + await self._scroll_page(page) + final_html = await page.content() + final_hash = hashlib.sha256(final_html.encode()).hexdigest() + with open(os.path.join(page_dir, "index.html"), "w", encoding="utf-8") as f: + f.write(final_html) + page_data = {"assets": []} + self._pages[page_url] = page_data + meta = { + "request": { + "method": resp.request.method, + "url": page_url, + "final_url": page_url, + "redirect_chain": [], + "headers": dict(resp.request.headers), + }, + "response": { + "status": resp.status, + "headers": dict(resp.headers), + "etag": resp.headers.get("etag"), + }, + "content": { + "sha256": final_hash, + "initial_sha256": content_hash, + }, + "scrape": { + "timestamp": datetime.datetime.now( + datetime.timezone.utc + ).isoformat(), + }, + } + with open( + os.path.join(page_dir, "initial.json"), "w", encoding="utf-8" + ) as f: + json.dump(meta, f, indent=2) + await self._enqueue_links(initial_html, page_url, depth) + await self._enqueue_links(final_html, page_url, depth) + else: + logger.info(f"Page unchanged, skipping: {page_url}") + page_dir = self._url_dir(page_url) + initial_path = os.path.join(page_dir, "initial.html") + if os.path.isfile(initial_path): + with open(initial_path, encoding="utf-8") as f: + local_html = f.read() + await self._enqueue_links(local_html, page_url, depth) + + async def _wait_for_hydrated(self, page): + prev_size = -1 + stable = 0 + for _ in range(10): + html = await page.content() + curr_size = len(html) + if abs(curr_size - prev_size) < 500: + stable += 1 + else: + stable = 0 + prev_size = curr_size + if stable >= 2: + break + await asyncio.sleep(1) + + async def _scroll_page(self, page): + """Scroll to bottom and back to trigger lazy loading.""" + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await asyncio.sleep(1) + await page.evaluate("window.scrollTo(0, 0)") + + async def _enqueue_links(self, html, page_url, depth): + if self.max_pages is not None and self.page_count >= self.max_pages: + return + if depth >= self.max_depth: + return + doc = pq(html) + remaining = ( + (self.max_pages - self.page_count) + if self.max_pages is not None + else UNLIMITED + ) + for link_url in self._extract_article_links(doc, page_url): + if remaining <= 0: + break + if link_url not in self.enqueued: + self.enqueued.add(link_url) + remaining -= 1 + await self._queue.put((link_url, depth + 1)) + + async def _worker(self, worker_id, pages): + page = pages[worker_id] + current_url_ref = {"url": None} + page.on("response", lambda resp, r=current_url_ref: self._on_response(resp, r)) + while True: + if self.max_pages is not None and self.page_count >= self.max_pages: + break + item = await self._queue.get() + if item is None: + break + url, depth = item + if depth >= self.max_depth: + self._queue.task_done() + continue + self._busy += 1 + try: + if current_url_ref["url"] and current_url_ref["url"] in self._pages: + self._pages[current_url_ref["url"]]["assets"] = self._pages[ + current_url_ref["url"] + ].get("assets", []) + current_url_ref["url"] = url + await asyncio.wait_for( + self._process_page(page, url, depth), + timeout=self.page_timeout, + ) + await asyncio.sleep(0.3) + except asyncio.TimeoutError: + logger.warning(f"Timeout processing {url}") + except Exception as e: + logger.error(f"Error processing {url}: {e}") + finally: + self._busy -= 1 + self._queue.task_done() + + async def run(self): + async with async_playwright() as p: + browser = await p.chromium.launch( + headless=True, + args=["--disable-dev-shm-usage"], + executable_path="/usr/bin/chromium-browser", + ) + context = await browser.new_context( + user_agent=( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ), + viewport={"width": 1920, "height": 1080}, + ) + await context.add_init_script( + "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" + ) + pages = [] + for i in range(self.max_parallel): + pg = await context.new_page() + pg.set_default_timeout(self.timeout) + pages.append(pg) + logger.info( + f"Scraping {self.url} (depth={self.max_depth}, parallel={self.max_parallel})" + ) + await self._queue.put((self.url, 0)) + workers = [ + asyncio.create_task(self._worker(i, pages)) + for i in range(self.max_parallel) + ] + watchdog = 0 + try: + while True: + await asyncio.sleep(0.5) + watchdog += 1 + if watchdog >= 300: + logger.warning("Watchdog: queue not draining") + watchdog = 0 + if self._queue.empty() and self._busy == 0: + for _ in range(self.max_parallel): + await self._queue.put(None) + break + if self.max_pages is not None and self.page_count >= self.max_pages: + for _ in range(self.max_parallel): + await self._queue.put(None) + break + await asyncio.gather(*workers) + except Exception: + logger.warning("Browser crashed, stopping scrape") + for _ in range(self.max_parallel): + await self._queue.put(None) + await asyncio.gather(*workers, return_exceptions=True) + logger.info(f"Done: {self.page_count} pages, {self.asset_count} assets") + try: + await browser.close() + except Exception: + pass -- cgit v1.3.1