Crisis V7 MEGA: sufinanciranje_sport + panel + CRM auth
DB: - pgz_sport.sufinanciranje_sport.je_klub flag (RSS programi/totals false) - pgz_sport.sufinanciranje_sport.klub_id matched Endpoints: - /v2/potpore/by-year: samo_klubovi=True default + davatelj filter Frontend: - sport2.html PANEL FORCE HIDE CSS (right:-100vw default) - crm_v2.html: redirect to /login only on actual 401, not on page load
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Media deep crawl — full pages of local portals."""
|
||||
import sys, json, time
|
||||
sys.path.insert(0, "/opt/pgz-sport/scrapers/harvesters")
|
||||
from _common import (fetch, extract_text, extract_title, chunk_text,
|
||||
upsert_facts, find_internal_links, DSN)
|
||||
from urllib.parse import urlparse
|
||||
import psycopg2
|
||||
|
||||
MEDIA = {
|
||||
"novilist_rijeka": ["https://www.novilist.hr/rijeka/", "https://www.novilist.hr/regija/"],
|
||||
"rijekadanas_full": ["https://rijekadanas.com/category/rijeka/", "https://rijekadanas.com/category/pgz/"],
|
||||
"rijekain_full": ["https://rijekain.hr/category/rijeka/"],
|
||||
"primorske_full": ["https://primorskenovice.hr/"],
|
||||
"rkc_blog": ["https://www.rkcrijeka.hr/blog/"],
|
||||
"rijeka2020_arhiva": ["https://rijeka2020.eu/category/news/"],
|
||||
"kulturpunkt_ri": ["https://www.kulturpunkt.hr/tag/rijeka"],
|
||||
"5portala_hr_pgz": ["https://www.5portala.hr/regije/primorsko-goranska/"],
|
||||
}
|
||||
|
||||
|
||||
def crawl(name, urls, max_pages=20):
|
||||
conn = psycopg2.connect(DSN); conn.autocommit = True
|
||||
visited = set(); queue = list(urls); facts = 0
|
||||
while queue and len(visited) < max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in visited: continue
|
||||
visited.add(url)
|
||||
html, status = fetch(url, timeout=15)
|
||||
if not html or status != 200: continue
|
||||
title = extract_title(html); text = extract_text(html)
|
||||
if not text or len(text) < 300: continue
|
||||
ff = []
|
||||
if title and len(title) > 12:
|
||||
ff.append({"fact": f"[{name}] {title}", "url": url, "title": title})
|
||||
for c in chunk_text(text, 800):
|
||||
if len(c) > 120:
|
||||
ff.append({"fact": c, "url": url, "title": title})
|
||||
facts += upsert_facts(conn, ff, source_name=f"media_{name}",
|
||||
category="media_pgz_deep", confidence=0.82)
|
||||
base = urlparse(url).hostname
|
||||
for link in find_internal_links(html, url):
|
||||
if link not in visited and (urlparse(link).hostname or "") == base and len(queue) < 60:
|
||||
queue.append(link)
|
||||
time.sleep(0.6)
|
||||
conn.close()
|
||||
return {"name": name, "visited": len(visited), "facts": facts}
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
for name, urls in MEDIA.items():
|
||||
try:
|
||||
r = crawl(name, urls, max_pages=18)
|
||||
print(f" {name:25} {r['visited']:>3}p {r['facts']:>5}f")
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
print(f" {name:25} FAIL: {str(e)[:60]}")
|
||||
total = sum(r.get("facts", 0) for r in results)
|
||||
print(f"=== TOTAL: {total} ===")
|
||||
print(json.dumps({"media_count": len(results), "total_facts": total}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user