V8 MEGA: meta endpoints + manifestacije + HNS V8 harvester batch
Endpoints: - /v2/potpore/meta — dropdown options (sportovi, vrste, davatelji, godine) - /v2/potpore/by-year — sport, vrsta filters - /v2/manifestacije/meta — mjesta, razine, organizatori - /v2/manifestacije — lista s filterima HNS: - 20 PGŽ priority klubova batch harvester pokrenut (HNK Goranin, HNK Orijent 1919, HNK Rijeka, NK Crikvenica, ...) - ETA 30 min
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EU projekti i fondovi PGZ."""
|
||||
import sys, json, time
|
||||
sys.path.insert(0, "/opt/pgz-sport/scrapers/harvesters")
|
||||
from _common import (fetch, extract_text, extract_title, chunk_text,
|
||||
upsert_facts, find_internal_links, DSN)
|
||||
from urllib.parse import urlparse
|
||||
import psycopg2
|
||||
|
||||
EU = {
|
||||
"eu_fondovi_pgz": ["https://strukturnifondovi.hr/"],
|
||||
"ri_eu_kreativnost": ["https://rijeka2020.eu/"],
|
||||
"rijeka_smartcity": ["https://www.rijeka.hr/smart-city/"],
|
||||
"agencija_unutarnji": ["https://www.amenita.hr/"],
|
||||
"ri_lokalna_akcijska": ["https://lag-rijeka.hr/"],
|
||||
"agencija_jadranska": ["https://www.adriatic-ionian.eu/"],
|
||||
"interreg_kvarner": ["https://www.italy-croatia.eu/"],
|
||||
"horizon_pgz": ["https://horizon-europe.gov.hr/"],
|
||||
"epro_kvarner": ["https://www.epro-kvarner.hr/"],
|
||||
"leader_pgz": ["https://www.leader-rijeka.hr/"],
|
||||
}
|
||||
|
||||
|
||||
def crawl(name, urls, max_pages=10):
|
||||
conn = psycopg2.connect(DSN); conn.autocommit = True
|
||||
visited = set(); queue = list(urls); facts = 0
|
||||
while queue and len(visited) < max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in visited: continue
|
||||
visited.add(url)
|
||||
html, status = fetch(url, timeout=15)
|
||||
if not html or status != 200: continue
|
||||
title = extract_title(html); text = extract_text(html)
|
||||
if not text or len(text) < 200: continue
|
||||
ff = []
|
||||
if title and len(title) > 8:
|
||||
ff.append({"fact": f"{name} - {title}", "url": url, "title": title})
|
||||
for c in chunk_text(text, 800):
|
||||
if len(c) > 100:
|
||||
ff.append({"fact": c, "url": url, "title": title})
|
||||
facts += upsert_facts(conn, ff, source_name=name,
|
||||
category="eu_projekti_pgz", confidence=0.84)
|
||||
base = urlparse(url).hostname
|
||||
for link in find_internal_links(html, url):
|
||||
if link not in visited and (urlparse(link).hostname or "") == base and len(queue) < 25:
|
||||
queue.append(link)
|
||||
time.sleep(0.5)
|
||||
conn.close()
|
||||
return {"name": name, "visited": len(visited), "facts": facts}
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
for name, urls in EU.items():
|
||||
try:
|
||||
r = crawl(name, urls, max_pages=10)
|
||||
print(f" {name:25} {r['visited']:>3}p {r['facts']:>5}f")
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
print(f" {name:25} FAIL: {str(e)[:60]}")
|
||||
total = sum(r.get("facts", 0) for r in results)
|
||||
print(f"=== TOTAL: {total} ===")
|
||||
print(json.dumps({"eu_count": len(results), "total_facts": total}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gov.hr deep — DZS PxWeb, sudreg portal, sukobinteresa."""
|
||||
import sys, json, time
|
||||
sys.path.insert(0, "/opt/pgz-sport/scrapers/harvesters")
|
||||
from _common import (fetch, extract_text, extract_title, chunk_text,
|
||||
upsert_facts, find_internal_links, DSN)
|
||||
from urllib.parse import urlparse
|
||||
import psycopg2
|
||||
|
||||
GOV = {
|
||||
"dzs_kvarner": ["https://podaci.dzs.hr/hr/podaci/stanovnistvo/popis-stanovnistva/"],
|
||||
"sukobinteresa_pgz": ["https://www.sukobinteresa.hr/"],
|
||||
"drzavna_revizija": ["https://www.revizija.hr/"],
|
||||
"drzavna_komisija_kn": ["https://www.dkom.hr/"],
|
||||
"fina_pgz": ["https://www.fina.hr/poslovni-subjekti"],
|
||||
"fina_javne_objave": ["https://www.fina.hr/javne-objave-uprave"],
|
||||
"porezna_uprava_ri": ["https://www.porezna-uprava.hr/"],
|
||||
"ministarstvo_pgz": ["https://www.gov.hr/hr/ministarstva"],
|
||||
"vlada_hr_pgz": ["https://vlada.gov.hr/"],
|
||||
"uprava_polic_pgz": ["https://policija.gov.hr/policijska-uprava-primorsko-goranska/"],
|
||||
"vis_hr": ["https://www.vis-hr.com/"],
|
||||
}
|
||||
|
||||
|
||||
def crawl(name, urls, max_pages=10):
|
||||
conn = psycopg2.connect(DSN); conn.autocommit = True
|
||||
visited = set(); queue = list(urls); facts = 0
|
||||
while queue and len(visited) < max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in visited: continue
|
||||
visited.add(url)
|
||||
html, status = fetch(url, timeout=15)
|
||||
if not html or status != 200: continue
|
||||
title = extract_title(html); text = extract_text(html)
|
||||
if not text or len(text) < 200: continue
|
||||
ff = []
|
||||
if title and len(title) > 8:
|
||||
ff.append({"fact": f"{name} - {title}", "url": url, "title": title})
|
||||
for c in chunk_text(text, 800):
|
||||
if len(c) > 100:
|
||||
ff.append({"fact": c, "url": url, "title": title})
|
||||
facts += upsert_facts(conn, ff, source_name=name,
|
||||
category="gov_registri_pgz", confidence=0.88)
|
||||
base = urlparse(url).hostname
|
||||
for link in find_internal_links(html, url):
|
||||
if link not in visited and (urlparse(link).hostname or "") == base and len(queue) < 30:
|
||||
queue.append(link)
|
||||
time.sleep(0.5)
|
||||
conn.close()
|
||||
return {"name": name, "visited": len(visited), "facts": facts}
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
for name, urls in GOV.items():
|
||||
try:
|
||||
r = crawl(name, urls, max_pages=10)
|
||||
print(f" {name:25} {r['visited']:>3}p {r['facts']:>5}f")
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
print(f" {name:25} FAIL: {str(e)[:60]}")
|
||||
total = sum(r.get("facts", 0) for r in results)
|
||||
print(f"=== TOTAL: {total} ===")
|
||||
print(json.dumps({"gov_count": len(results), "total_facts": total}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Hoteli, restorani, smjestaj PGZ."""
|
||||
import sys, json, time
|
||||
sys.path.insert(0, "/opt/pgz-sport/scrapers/harvesters")
|
||||
from _common import (fetch, extract_text, extract_title, chunk_text,
|
||||
upsert_facts, find_internal_links, DSN)
|
||||
from urllib.parse import urlparse
|
||||
import psycopg2
|
||||
|
||||
HOTELS = {
|
||||
"milenij_hoteli": ["https://www.milenijhoteli.hr/"],
|
||||
"uniline_kvarner": ["https://www.uniline.hr/destinacije/kvarner"],
|
||||
"ri_botel": ["https://www.botel-marina.hr/"],
|
||||
"jadran_hoteli": ["https://www.jadran-crikvenica.hr/"],
|
||||
"valamar_kvarner": ["https://www.valamar.com/hr/odredista/kvarner"],
|
||||
"kongres_centar": ["https://www.adriatic-grandhotel.com/"],
|
||||
"lovran_riviera": ["https://www.liburnia.hr/"],
|
||||
"remisens_grupa": ["https://www.remisens.com/"],
|
||||
"imperijal_opatija": ["https://www.amadriapark.com/"],
|
||||
"hotel_neboder": ["https://www.jadran-hoteli.hr/"],
|
||||
"kanjon_zrmanja": ["https://www.kvarnertravel.hr/"],
|
||||
}
|
||||
|
||||
|
||||
def crawl(name, urls, max_pages=8):
|
||||
conn = psycopg2.connect(DSN); conn.autocommit = True
|
||||
visited = set(); queue = list(urls); facts = 0
|
||||
while queue and len(visited) < max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in visited: continue
|
||||
visited.add(url)
|
||||
html, status = fetch(url, timeout=15)
|
||||
if not html or status != 200: continue
|
||||
title = extract_title(html); text = extract_text(html)
|
||||
if not text or len(text) < 200: continue
|
||||
ff = []
|
||||
if title and len(title) > 8:
|
||||
ff.append({"fact": f"{name} - {title}", "url": url, "title": title})
|
||||
for c in chunk_text(text, 800):
|
||||
if len(c) > 100:
|
||||
ff.append({"fact": c, "url": url, "title": title})
|
||||
facts += upsert_facts(conn, ff, source_name=name,
|
||||
category="hoteli_pgz", confidence=0.83)
|
||||
base = urlparse(url).hostname
|
||||
for link in find_internal_links(html, url):
|
||||
if link not in visited and (urlparse(link).hostname or "") == base and len(queue) < 25:
|
||||
queue.append(link)
|
||||
time.sleep(0.5)
|
||||
conn.close()
|
||||
return {"name": name, "visited": len(visited), "facts": facts}
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
for name, urls in HOTELS.items():
|
||||
try:
|
||||
r = crawl(name, urls, max_pages=8)
|
||||
print(f" {name:25} {r['visited']:>3}p {r['facts']:>5}f")
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
print(f" {name:25} FAIL: {str(e)[:60]}")
|
||||
total = sum(r.get("facts", 0) for r in results)
|
||||
print(f"=== TOTAL: {total} ===")
|
||||
print(json.dumps({"hotel_count": len(results), "total_facts": total}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Politicke stranke i skupstine PGZ."""
|
||||
import sys, json, time
|
||||
sys.path.insert(0, "/opt/pgz-sport/scrapers/harvesters")
|
||||
from _common import (fetch, extract_text, extract_title, chunk_text,
|
||||
upsert_facts, find_internal_links, DSN)
|
||||
from urllib.parse import urlparse
|
||||
import psycopg2
|
||||
|
||||
POL = {
|
||||
"skupstina_pgz": ["https://www.pgz.hr/skupstina-zupanije/"],
|
||||
"rijeka_gradsko_vije": ["https://www.rijeka.hr/gradsko-vijece/"],
|
||||
"stranka_lkr_pgz": ["https://www.lista-rijeka.com/"],
|
||||
"sdp_rijeka": ["https://www.sdp.hr/podruznice/sdp-rijeka/"],
|
||||
"hdz_pgz": ["https://hdz.hr/zupanije/primorsko-goranska/"],
|
||||
"most_pgz": ["https://most.hr/regionalni-uredi"],
|
||||
"akcija_mladih": ["https://akcijamladih.hr/"],
|
||||
"rastimo_zajedno": ["https://rastimozajedno.hr/"],
|
||||
"javnost_pgz": ["https://www.izbori.hr/"],
|
||||
"zupan_obrenovic": ["https://www.pgz.hr/zupan/"],
|
||||
"gradonacelnica_iv_r": ["https://www.rijeka.hr/gradonacelnica/"],
|
||||
}
|
||||
|
||||
|
||||
def crawl(name, urls, max_pages=10):
|
||||
conn = psycopg2.connect(DSN); conn.autocommit = True
|
||||
visited = set(); queue = list(urls); facts = 0
|
||||
while queue and len(visited) < max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in visited: continue
|
||||
visited.add(url)
|
||||
html, status = fetch(url, timeout=15)
|
||||
if not html or status != 200: continue
|
||||
title = extract_title(html); text = extract_text(html)
|
||||
if not text or len(text) < 200: continue
|
||||
ff = []
|
||||
if title and len(title) > 8:
|
||||
ff.append({"fact": f"{name} - {title}", "url": url, "title": title})
|
||||
for c in chunk_text(text, 800):
|
||||
if len(c) > 100:
|
||||
ff.append({"fact": c, "url": url, "title": title})
|
||||
facts += upsert_facts(conn, ff, source_name=name,
|
||||
category="politika_pgz", confidence=0.85)
|
||||
base = urlparse(url).hostname
|
||||
for link in find_internal_links(html, url):
|
||||
if link not in visited and (urlparse(link).hostname or "") == base and len(queue) < 30:
|
||||
queue.append(link)
|
||||
time.sleep(0.5)
|
||||
conn.close()
|
||||
return {"name": name, "visited": len(visited), "facts": facts}
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
for name, urls in POL.items():
|
||||
try:
|
||||
r = crawl(name, urls, max_pages=10)
|
||||
print(f" {name:25} {r['visited']:>3}p {r['facts']:>5}f")
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
print(f" {name:25} FAIL: {str(e)[:60]}")
|
||||
total = sum(r.get("facts", 0) for r in results)
|
||||
print(f"=== TOTAL: {total} ===")
|
||||
print(json.dumps({"pol_count": len(results), "total_facts": total}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user