damir
ae9c4e2bfd
DB: pgz_sport.sportski_objekti (103 objekti, 103 s geo, 60 s adresom, 31 tip) API: - /api/v2/sportski-objekti (filter: tip, grad, sport, q) - /api/v2/sportski-objekti/meta (tipovi, gradovi, sportovi, ukupno) Frontend: - /static/objekti.html — Leaflet (OpenStreetMap) interactive map - 3 dropdown filter (tip, grad, sport) + search - Side panel s listom + map markers s ikonama (🏟️⚽🏊⛵🎿🎳⛸️🎯🥌🏃) - Popup: naziv, tip, kapacitet, adresa, upravitelj, izgradeno, sportovi, web link, Google Maps link - /objekti, /sport/objekti, /sport/api/v2/sportski-objekti routes Sidebar app.html: +Sportski objekti link Background: scripts/objekti_enrich_address.py (Nominatim reverse-geocode 60 objekata bez adrese)
69 lines
2.7 KiB
Python
69 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Sport federacije i savezi PGŽ — all sports."""
|
|
import sys, json, time
|
|
sys.path.insert(0, "/opt/pgz-sport/scrapers/harvesters")
|
|
from _common import (fetch, extract_text, extract_title, chunk_text,
|
|
upsert_facts, find_internal_links, DSN)
|
|
from urllib.parse import urlparse
|
|
import psycopg2
|
|
|
|
FEDERATIONS = {
|
|
"zns_pgz": ["https://www.pgzns.hr/"],
|
|
"kss_pgz": ["https://www.kss-pgz.hr/"],
|
|
"handball_pgz": ["https://www.hrs-pgz.hr/"],
|
|
"odbojka_savez_pgz": ["https://www.odbojka-pgz.hr/"],
|
|
"atletika_savez_pgz": ["https://www.atletika-pgz.hr/"],
|
|
"plivanje_savez_pgz": ["https://www.plivanje-pgz.hr/"],
|
|
"skijaski_savez_pgz": ["https://www.ski-pgz.hr/"],
|
|
"tenis_savez_pgz": ["https://www.tenis-pgz.hr/"],
|
|
"judo_savez_pgz": ["https://www.judo-pgz.hr/"],
|
|
"karate_savez_pgz": ["https://www.karate-pgz.hr/"],
|
|
"kuglanje_savez_pgz": ["https://www.kuglanje-pgz.hr/"],
|
|
}
|
|
|
|
|
|
def crawl(name, urls, max_pages=12):
|
|
conn = psycopg2.connect(DSN); conn.autocommit = True
|
|
visited = set(); queue = list(urls); facts = 0
|
|
while queue and len(visited) < max_pages:
|
|
url = queue.pop(0)
|
|
if url in visited: continue
|
|
visited.add(url)
|
|
html, status = fetch(url, timeout=15)
|
|
if not html or status != 200: continue
|
|
title = extract_title(html); text = extract_text(html)
|
|
if not text or len(text) < 200: continue
|
|
ff = []
|
|
if title and len(title) > 8:
|
|
ff.append({"fact": f"{name} - {title}", "url": url, "title": title})
|
|
for c in chunk_text(text, 800):
|
|
if len(c) > 100:
|
|
ff.append({"fact": c, "url": url, "title": title})
|
|
facts += upsert_facts(conn, ff, source_name=name,
|
|
category="sport_federacije_pgz", confidence=0.88)
|
|
base = urlparse(url).hostname
|
|
for link in find_internal_links(html, url):
|
|
if link not in visited and (urlparse(link).hostname or "") == base and len(queue) < 35:
|
|
queue.append(link)
|
|
time.sleep(0.5)
|
|
conn.close()
|
|
return {"name": name, "visited": len(visited), "facts": facts}
|
|
|
|
|
|
def main():
|
|
results = []
|
|
for name, urls in FEDERATIONS.items():
|
|
try:
|
|
r = crawl(name, urls, max_pages=10)
|
|
print(f" {name:25} {r['visited']:>3}p {r['facts']:>5}f")
|
|
results.append(r)
|
|
except Exception as e:
|
|
print(f" {name:25} FAIL: {str(e)[:60]}")
|
|
total = sum(r.get("facts", 0) for r in results)
|
|
print(f"=== TOTAL: {total} ===")
|
|
print(json.dumps({"fed_count": len(results), "total_facts": total}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|