"""
enrich_router.py — v3 enrichment + forensic scan
Author: dradulic@outlook.com / damir@rinet.one
Date:   2026-05-04 (R2) → 2026-05-05 (R3 CC6 v3)

POST /v2/enrich/{kind}/{eid}
    Inspect the row, scrape the web (Wikipedia HR, sport-pgz.hr search,
    primary club URL if any), regex-extract candidate fields (web/email/
    telefon), optionally synthesise descriptions via DeepSeek, and return
    a *preview* shape with `proposed` updates the operator can apply.

POST /v2/enrich/{kind}/{eid}/apply
    Body shapes:
      None / {}                  → re-run preview, apply every proposed field
      {"fields": {...}}          → apply ONLY those (whitelist + emptiness still enforced)
    Performs UPDATE on the matching table, sets metadata.enriched_at and
    metadata.enrichment_source, writes a row to pgz_sport.enrichment_log,
    returns the after snapshot.

GET  /v2/enrich/log?kind=&target_id=&limit=
    Read recent enrichment-log entries.

POST /v2/forensic/scan
    Search civic.persons by name, return entity links + findings + risk score.

Kinds: klub | savez | sportas
"""
from __future__ import annotations
import os, re, json, time, html, urllib.parse, urllib.request
from datetime import datetime, timezone
from typing import Any, Optional

import psycopg2, psycopg2.extras
from fastapi import APIRouter, HTTPException, Header, Body

router = APIRouter()

_pgh = os.environ.get('PG_HOST', '10.10.0.2')
_pgp = int(os.environ.get('PG_PORT', '6432'))
if _pgh in ('localhost', '127.0.0.1'):
    _pgh = os.environ.get('DB_HOST', '10.10.0.2')
    _pgp = int(os.environ.get('DB_PORT', '6432'))
DB = dict(host=_pgh, port=_pgp,
          dbname=os.environ.get('PG_DB', 'rinet_v3'),
          user=os.environ.get('PG_USER', 'rinet'),
          password=os.environ.get('PG_PASS', 'R1net2026!SecureDB#v7'))

UA = 'pgz-sport-enrich/3.0 (+https://sport.rinet.one)'
TIMEOUT = 6  # seconds — fail-soft

# ---- Sport-aware source selection (Damir 2026-05-10, task 06) ----
# Generic enrichment was only fitted to football. Other sports need their
# own federation roster + terminology. This map drives the per-sport source
# list used by enrich pipelines.
SPORT_FEDERATION_MAP: "dict[str, list[str]]" = {
    "nogomet":    ["hns_semafor", "hns_cff", "sport_pgz"],
    "košarka":    ["hks_cbf", "kosarkapgz", "sport_pgz"],
    "rukomet":    ["hrs", "sport_pgz"],
    "vaterpolo":  ["hvs", "sport_pgz"],
    "atletika":   ["has", "sport_pgz"],
    "plivanje":   ["hps", "sport_pgz"],
    "hokej":      ["hsh", "sport_pgz"],
    "šah":        ["hss_chess", "sport_pgz"],
    "odbojka":    ["hos", "sport_pgz"],
    "tenis":      ["hts", "sport_pgz"],
    "biciklizam": ["hbs", "sport_pgz"],
}

# Per-sport field schemas - used by /select_sources_for_sport callers and
# by the frontend (data-sport-fields="...") to render the right fields.
SPORT_FIELD_SCHEMA: "dict[str, list[str]]" = {
    "nogomet":    ["position", "dominant_foot", "height_cm", "jersey_no", "club"],
    "košarka":    ["position", "height_cm", "weight_kg", "hand", "stats"],
    "rukomet":    ["position", "hand", "stats"],
    "atletika":   ["discipline", "category", "personal_records"],
    "šah":        ["elo_rating", "fide_id", "title"],
    "vaterpolo":  ["position", "hand", "stats"],
    "plivanje":   ["discipline", "personal_records"],
    "hokej":      ["position", "hand", "jersey_no"],
    "odbojka":    ["position", "height_cm", "stats"],
    "tenis":      ["dominant_hand", "atp_wta_rank", "personal_records"],
    "biciklizam": ["discipline", "team", "category"],
}

_DEFAULT_SOURCES = ["sport_pgz", "google", "wikipedia"]
_DEFAULT_FIELDS  = ["club", "position", "stats"]


def select_sources_for_sport(sport):
    """Return the prioritized source list for a given sport string.

    Match strategy: lower + strip -> exact key, else longest-prefix key,
    else default (_DEFAULT_SOURCES). Preserves diacritics so "košarka"
    matches "košarka" and not the prefix "kosarka".
    """
    if not sport:
        return list(_DEFAULT_SOURCES)
    s = sport.lower().strip()
    if s in SPORT_FEDERATION_MAP:
        return list(SPORT_FEDERATION_MAP[s])
    best = None
    for key in SPORT_FEDERATION_MAP:
        if s.startswith(key) and (best is None or len(key) > len(best)):
            best = key
    if best:
        return list(SPORT_FEDERATION_MAP[best])
    return list(_DEFAULT_SOURCES)


def fields_for_sport(sport):
    """Return the per-sport field list (UI hint + enrichment scope)."""
    if not sport:
        return list(_DEFAULT_FIELDS)
    s = sport.lower().strip()
    if s in SPORT_FIELD_SCHEMA:
        return list(SPORT_FIELD_SCHEMA[s])
    for key in SPORT_FIELD_SCHEMA:
        if s.startswith(key):
            return list(SPORT_FIELD_SCHEMA[key])
    return list(_DEFAULT_FIELDS)
# ---- end sport-aware section ----


# Optional JS-aware fallback (Playwright). Lazy-loaded, never required.
import sys as _sys
_sys.path.insert(0, '/opt/pgz-sport')
try:
    from enrichment import playwright_scraper as _pw_scraper
    _HAS_PW = _pw_scraper.HAS_PLAYWRIGHT
except Exception:
    _pw_scraper = None
    _HAS_PW = False

DEEPSEEK_KEY = os.environ.get('DEEPSEEK_API_KEY', '').strip()
DEEPSEEK_URL = os.environ.get('DEEPSEEK_URL',
                              'https://api.deepseek.com/v1/chat/completions')


# ─── DB helpers ──────────────────────────────────────────────────────────
def _db():
    c = psycopg2.connect(**DB); c.autocommit = True; return c

def _fetch_one(sql, p):
    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        cur.execute(sql, p); r = cur.fetchone()
        return dict(r) if r else None


# ─── HTTP helpers ────────────────────────────────────────────────────────
def _http_get(url: str, timeout: int = TIMEOUT) -> Optional[str]:
    if not url: return None
    if not url.startswith('http'): return None
    try:
        req = urllib.request.Request(url, headers={
            'User-Agent': UA, 'Accept-Language': 'hr,en;q=0.8'})
        with urllib.request.urlopen(req, timeout=timeout) as r:
            data = r.read(150000)
            try:    return data.decode('utf-8')
            except: return data.decode('latin-1', 'ignore')
    except Exception:
        return None



# ─── Redis cache + slow-enrichment telemetry (added 2026-05-10) ──────────
ENRICH_CACHE_TTL = int(os.environ.get('ENRICH_CACHE_TTL_SECONDS', '86400'))  # 24h default
ENRICH_SLOW_LOG  = '/var/log/enrich_slow.log'
ENRICH_SLOW_TG_THRESHOLD = float(os.environ.get('ENRICH_SLOW_TG_THRESHOLD', '20.0'))
ENRICH_SLOW_LOG_THRESHOLD = float(os.environ.get('ENRICH_SLOW_LOG_THRESHOLD', '10.0'))

_enrich_redis_singleton = None
def _enrich_redis():
    """Singleton Redis connection for enrichment cache. Returns None on failure (fail-soft).

    Renamed from _redis() to avoid collision with existing _redis_client() function
    elsewhere in this module.
    """
    global _enrich_redis_singleton
    if _enrich_redis_singleton is not None:
        return _enrich_redis_singleton
    try:
        import redis
    except Exception:
        return None
    host = os.environ.get('REDIS_HOST', 'localhost')
    port = int(os.environ.get('REDIS_PORT', '6379'))
    pwd  = (os.environ.get('REDIS_PASS') or os.environ.get('REDIS_PASSWORD') or '').strip().strip("'").strip('"') or None
    for p in (pwd, None):
        try:
            r = redis.Redis(host=host, port=port, password=p,
                            decode_responses=True, socket_connect_timeout=2,
                            socket_timeout=2)
            r.ping()
            _enrich_redis_singleton = r
            return r
        except Exception:
            continue
    return None

def _cache_get(key: str):
    """Fail-soft cache read. Returns parsed JSON or None."""
    r = _enrich_redis()
    if not r: return None
    try:
        v = r.get(key)
        if v is None: return None
        return json.loads(v)
    except Exception:
        return None

def _cache_set(key: str, value, ttl: int = ENRICH_CACHE_TTL):
    """Fail-soft cache write."""
    r = _enrich_redis()
    if not r: return False
    try:
        r.setex(key, ttl, json.dumps(value, default=str, ensure_ascii=False))
        return True
    except Exception:
        return False

def _cache_delete(key: str):
    r = _enrich_redis()
    if not r: return
    try: r.delete(key)
    except Exception: pass

def _enrich_slow_log(kind: str, eid: int, duration_s: float, cached: bool, error: str = None):
    """Append slow-enrichment record to /var/log/enrich_slow.log + Telegram alert if extreme."""
    import logging as _lg
    line = (f"{datetime.now(timezone.utc).isoformat()} "
            f"kind={kind} id={eid} duration={duration_s:.2f}s cached={cached}"
            f"{' error=' + error if error else ''}\n")
    try:
        with open(ENRICH_SLOW_LOG, "a", encoding="utf-8") as f:
            f.write(line)
    except Exception as e:
        _lg.getLogger("enrich").warning(f"slow-log write failed: {e}")
    if duration_s >= ENRICH_SLOW_TG_THRESHOLD:
        try:
            tok = os.environ.get('TG_BOT_TOKEN') or os.environ.get('TELEGRAM_BOT_TOKEN')
            chat = os.environ.get('TG_CHAT_ID') or os.environ.get('TELEGRAM_CHAT_ID')
            if tok and chat:
                msg = f"\u26a0\ufe0f enrich slow: {kind}/{eid} took {duration_s:.1f}s (cached={cached})"
                urllib.request.urlopen(
                    f"https://api.telegram.org/bot{tok}/sendMessage",
                    data=urllib.parse.urlencode({'chat_id': chat, 'text': msg}).encode('utf-8'),
                    timeout=3
                ).read()
        except Exception:
            pass


def _strip_tags(s: str) -> str:
    if not s: return ''
    s = re.sub(r'<script[^>]*>.*?</script>', ' ', s, flags=re.S | re.I)
    s = re.sub(r'<style[^>]*>.*?</style>',   ' ', s, flags=re.S | re.I)
    s = re.sub(r'<[^>]+>', ' ', s)
    s = html.unescape(s)
    s = re.sub(r'\s+', ' ', s).strip()
    return s


def _extract_meta(html_doc: str, url: str) -> dict:
    if not html_doc: return {}
    out = {'url': url, 'fetched_at': int(time.time())}
    m = re.search(r'<title[^>]*>([^<]+)</title>', html_doc, re.I)
    if m: out['title'] = html.unescape(m.group(1).strip())[:300]
    m = re.search(r'<meta\s+name=["\']description["\']\s+content=["\']([^"\']+)["\']', html_doc, re.I)
    if not m:
        m = re.search(r'<meta\s+property=["\']og:description["\']\s+content=["\']([^"\']+)["\']', html_doc, re.I)
    if m: out['description'] = html.unescape(m.group(1).strip())[:600]
    return out


def _fetch_title(url, timeout=5):
    body = _http_get(url, timeout=timeout)
    if not body: return {'url': url, 'error': 'fetch failed'} if url else None
    return _extract_meta(body, url)


# ─── Field extractors ───────────────────────────────────────────────────
RE_EMAIL = re.compile(r'[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}', re.I)
RE_PHONE = re.compile(r'(?:\+?385[\s\-/]*|0)\d[\d\s\-/]{6,12}\d')
RE_URL   = re.compile(r'https?://[^\s"\'<>)\]]+', re.I)

def _find_email(text: str) -> Optional[str]:
    if not text: return None
    bad = ('@example.', '@test.', '@email.', 'wixpress.com',
           'sentry.io', 'jquery.com', 'googleapis', '@2x.', 'noreply@')
    seen = set()
    for m in RE_EMAIL.finditer(text):
        e = m.group(0).lower().rstrip('.,;:)')
        if any(b in e for b in bad): continue
        if e in seen: continue
        seen.add(e); return e
    return None

def _find_phone(text: str) -> Optional[str]:
    if not text: return None
    for m in RE_PHONE.finditer(text):
        raw = m.group(0).strip()
        digits = re.sub(r'\D', '', raw)
        if not (8 <= len(digits) <= 13): continue
        cleaned = re.sub(r'\s+', ' ', raw).strip()
        if raw.startswith('+385'):  return '+385 ' + raw[4:].lstrip().lstrip('-/')
        if raw.startswith('00385'): return '+385 ' + raw[5:].lstrip().lstrip('-/')
        return cleaned
    return None

def _find_official_web(text: str, hint: str = '') -> Optional[str]:
    if not text: return None
    blocked = ('wikipedia.org', 'sport-pgz.hr', 'google.com', 'facebook.com',
               'instagram.com', 'youtube.com', 'twitter.com', 'wikimedia',
               'sportilus.com', 'transfermarkt.com', 'wikidata.org',
               'sudreg.pravosudje.hr', 'gov.hr', 'apis.google.com',
               'rinet.one', 'pgz.hr')
    candidates: list[str] = []
    for m in RE_URL.finditer(text):
        u = m.group(0).rstrip('.,;:)\'"')
        try:
            host = urllib.parse.urlparse(u).hostname or ''
        except Exception:
            continue
        if not host or any(b in host for b in blocked): continue
        candidates.append(u)
    if not candidates: return None
    if hint:
        slug = re.sub(r'[^a-z0-9]', '', hint.lower())[:8]
        for u in candidates:
            host = urllib.parse.urlparse(u).hostname or ''
            if slug and slug in host.replace('-', '').replace('.', ''):
                return u
    return candidates[0]


# ─── External sources ────────────────────────────────────────────────────
def _wiki_variants(query: str) -> list[str]:
    """Generate sensible Wikipedia HR title variants for a query.

    The summary REST API is title-exact; clubs are often listed under their
    abbreviation (KK X, NK X, RK X, OK X), so we try those variants too.
    """
    if not query: return []
    out, seen = [], set()
    raw = query.strip()
    def _push(v):
        if v and v not in seen: seen.add(v); out.append(v)
    _push(raw)
    # KK Kvarner 2010 from Košarkaški klub KVARNER 2010
    parts = raw.split()
    sport_to_abbr = {
        'košarkaški': 'KK', 'kosarkaski': 'KK',
        'nogometni':  'NK', 'rukometni':  'RK',
        'odbojkaški': 'OK', 'odbojkaski': 'OK',
        'vaterpolski':'VK', 'plivacki':   'PK', 'plivački': 'PK',
        'boćarski':   'BK', 'bocarski':   'BK',
    }
    if len(parts) >= 3 and parts[0].lower() in sport_to_abbr and parts[1].lower() == 'klub':
        _push(sport_to_abbr[parts[0].lower()] + ' ' + ' '.join(p.capitalize() if p.isupper() else p for p in parts[2:]))
    return out

def _wiki_summary(query: str) -> Optional[dict]:
    for variant in _wiki_variants(query):
        title = urllib.parse.quote(variant.replace(' ', '_'), safe='')
        body = _http_get(f'https://hr.wikipedia.org/api/rest_v1/page/summary/{title}', timeout=5)
        if not body: continue
        try:
            d = json.loads(body)
        except Exception:
            continue
        if d.get('type') in ('disambiguation', 'no-extract'): continue
        if not d.get('extract'): continue
        return {
            'source': 'wikipedia.hr',
            'url': d.get('content_urls', {}).get('desktop', {}).get('page'),
            'title': d.get('title'),
            'extract': d.get('extract'),
            'description': d.get('description'),
            'matched_variant': variant,
        }
    return None


def _sport_pgz_search(query: str) -> Optional[dict]:
    if not query: return None
    page = _http_get('https://sport-pgz.hr/?s=' + urllib.parse.quote(query), timeout=6)
    if not page:
        # Plain HTTP failed → try JS-rendered fallback if available.
        if _HAS_PW and _pw_scraper is not None:
            return _pw_scraper.scrape_sport_pgz_klub(query)
        return None
    m = re.search(r'<article[^>]*>.*?<a\s+href=["\']([^"\']+)["\'][^>]*rel=["\']bookmark["\'][^>]*>([^<]+)</a>',
                  page, re.S | re.I)
    if not m:
        m = re.search(r'<a\s+href=["\'](https?://sport-pgz\.hr/[^"\']+)["\'][^>]*>([^<]{6,180})</a>', page, re.I)
    if not m:
        # Search page rendered but yielded nothing parseable — try JS fallback.
        if _HAS_PW and _pw_scraper is not None:
            return _pw_scraper.scrape_sport_pgz_klub(query)
        return None
    hit = m.group(1)
    body = _http_get(hit, timeout=6)
    if not body:
        return {'source': 'sport-pgz.hr', 'url': hit, 'title': html.unescape(m.group(2).strip())}
    text = _strip_tags(body)[:4000]
    meta = _extract_meta(body, hit)
    return {
        'source': 'sport-pgz.hr',
        'url': hit,
        'title': meta.get('title') or html.unescape(m.group(2).strip()),
        'extract': meta.get('description') or text[:500],
        'raw_text': text,
    }


# ─── kosarkapgz.com (Košarkaški savez PGŽ) ──────────────────────────────
# Added 2026-05-10. WordPress site with permalinks DISABLED — REST API is
# reachable only via `?rest_route=/...` (the `/wp-json/...` paths 404).
# Use as enrichment source for PGŽ basketball entities (savez/klub/sportaš).
_KOSARKAPGZ_BASE = 'https://kosarkapgz.com'

def _kosarkapgz_is_basketball_pgz(naziv: str, sport: Optional[str]) -> bool:
    """Heuristic: PGŽ basketball entity worth scraping kosarkapgz.com for.

    The pgz_sport.klubovi/savezi tables are PGŽ-scoped by design
    (klubovi.is_pgz default true, savezi.region default 'PGŽ'), so we don't
    re-check region here — a basketball signal is enough.
    """
    def _fold(s: str) -> str:
        return (s or '').lower().replace('ć', 'c').replace('š', 's') \
                                .replace('ž', 'z').replace('č', 'c').replace('đ', 'd')
    n, sp = _fold(naziv), _fold(sport)
    if 'kosark' in sp: return True   # 'košarka' / 'košarkaški'
    if 'kosark' in n:  return True
    if re.match(r'^\s*kk\b', n): return True   # 'KK X' = Košarkaški klub X
    return False

def _kosarkapgz_savez_meta() -> Optional[dict]:
    """Fetch homepage + key WP pages from kosarkapgz.com. Cached 24h.

    Also runs structured extractors on /?page_id=234 (Upravni odbor) and
    /?page_id=70 (Kontakt), and persists the result into civic.entities +
    civic.persons exactly once per cache miss. See _kosarkapgz_persist_kspgz.
    """
    cache_key = 'kosarkapgz:savez_meta:v1'
    cached = _cache_get(cache_key)
    if cached: return cached

    home = _http_get(_KOSARKAPGZ_BASE + '/', timeout=8)
    if not home:
        _cache_set(cache_key, None, ttl=900); return None
    meta = _extract_meta(home, _KOSARKAPGZ_BASE + '/')

    page_ids = {'povijest': 232, 'upravni_odbor': 234, 'kontakt': 70, 'objave': 217}
    page_html: dict[str, str] = {}
    page_text: dict[str, str] = {}
    parts: list[str] = [meta.get('description', '') or '']
    for slug, pid in page_ids.items():
        body = _http_get(f'{_KOSARKAPGZ_BASE}/?page_id={pid}', timeout=8)
        if not body: continue
        page_html[slug] = body
        text = _strip_tags(body)
        if text:
            page_text[slug] = text[:5000]
            parts.append(f'[{slug}] ' + text[:5000])

    # Structured extraction + idempotent persistence.
    osobe  = _kosarkapgz_extract_osobe(page_html.get('upravni_odbor', '')) or []
    adresa = _kosarkapgz_extract_adresa(page_html.get('kontakt', '')) or {}
    persist = _kosarkapgz_persist_kspgz(osobe, adresa) if (osobe or adresa) else {}

    result = {
        'source': 'kosarkapgz.com',
        'url': _KOSARKAPGZ_BASE + '/',
        'title': meta.get('title') or 'Košarkaški savez PGŽ',
        'extract': (meta.get('description') or '')[:600],
        'pages': page_text,
        'structured': {
            'upravni_odbor': osobe,
            'adresa': adresa,
            'persist': persist,
        },
        'raw_text': ('\n\n'.join(p for p in parts if p))[:80000],
    }
    _cache_set(cache_key, result, ttl=86400)
    return result

def _kosarkapgz_search_posts(query: str, limit: int = 5) -> Optional[dict]:
    """WP REST search for posts mentioning `query`. Source-doc shape."""
    if not query: return None
    q = query.strip()
    if len(q) < 3: return None
    cache_key = 'kosarkapgz:search:' + re.sub(r'[^a-z0-9]+', '_', q.lower())[:60] + ':v1'
    cached = _cache_get(cache_key)
    if cached is not None:
        return cached or None

    url = (f'{_KOSARKAPGZ_BASE}/?rest_route=/wp/v2/posts'
           f'&search={urllib.parse.quote(q)}&per_page={max(1, min(limit, 20))}'
           f'&_fields=id,date,slug,title,excerpt,link,categories')
    body = _http_get(url, timeout=8)
    if not body or not body.lstrip().startswith('['):
        _cache_set(cache_key, [], ttl=3600); return None
    try:
        posts = json.loads(body)
    except Exception:
        _cache_set(cache_key, [], ttl=3600); return None
    if not posts:
        _cache_set(cache_key, [], ttl=3600); return None

    articles = []
    for p in posts[:limit]:
        title_html = ((p.get('title') or {}).get('rendered')) or ''
        excerpt_html = ((p.get('excerpt') or {}).get('rendered')) or ''
        articles.append({
            'title':   html.unescape(_strip_tags(title_html))[:300],
            'excerpt': html.unescape(_strip_tags(excerpt_html))[:400],
            'date':    p.get('date'),
            'link':    p.get('link'),
        })
    blob_lines = [
        f"{(a.get('date') or '')[:10]} — {a.get('title','')}\n"
        f"{a.get('excerpt','')}\n{a.get('link','')}"
        for a in articles
    ]
    result = {
        'source': 'kosarkapgz.com',
        'url': _KOSARKAPGZ_BASE + '/',
        'title': f'kosarkapgz.com — {len(articles)} mention(s) for "{q[:60]}"',
        'extract': blob_lines[0] if blob_lines else '',
        'articles': articles,
        'raw_text': ('\n\n'.join(blob_lines))[:60000],
    }
    _cache_set(cache_key, result, ttl=86400)
    return result



# ─── kosarkapgz.com — structured extractors (page_id=234, page_id=70) ────
# Added 2026-05-10. Cheap, deterministic parsers for two specific WP pages
# whose layout is known:
#   ?page_id=234 → "Upravni odbor" — entry-content is a single <ul> of names,
#                 optionally suffixed with " – funkcija" (en/em dash).
#   ?page_id=70  → "Kontakt" — the OG description packs ulica/grad/email/
#                 tel/web in a single string ("Adresa Verdijeva 11/III51000
#                 RijekaHrvatska E-mail: ... Mob: ... Tel: ... http://...").
# Period of office is not published on either page, so persons.period is
# always None today; the field is kept so future scrapers (kszpg.hr archive)
# can fill it in without a schema change.

_KPGZ_DASH_RE  = re.compile(r'\s*[\u2013\u2014\-]\s*')   # en/em/ascii dash
_KPGZ_LI_RE    = re.compile(r'<li[^>]*>(.*?)</li>', re.I | re.S)
_KPGZ_TAG_RE   = re.compile(r'<[^>]+>')
_KPGZ_OG_DESC  = re.compile(r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\']([^"\']+)', re.I)
_KPGZ_EMAIL_RE = re.compile(r'[\w.+-]+@[\w-]+\.[\w.-]+')
_KPGZ_TEL_RE   = re.compile(r'(?i)Tel\s*[:\.\-]?\s*(\+?[\d\s().\-/]{6,})')
_KPGZ_MOB_RE   = re.compile(r'(?i)Mob\s*[:\.\-]?\s*(\+?[\d\s().\-/]{6,})')
_KPGZ_WEB_RE   = re.compile(r'https?://[\w.\-]+(?:/[\w.\-/?#=&%]*)?', re.I)
# "Verdijeva 11/III" → ulica="Verdijeva", broj="11/III"
_KPGZ_ULICA_RE = re.compile(r'(?i)Adresa\s+([A-ZČĆĐŠŽa-zčćđšž\s]+?)\s+([\dA-Za-z][\w./\-]*)')
# "51000 Rijeka"
_KPGZ_POSTAL_RE = re.compile(r'(\d{5})\s+([A-ZČĆĐŠŽa-zčćđšž][A-ZČĆĐŠŽa-zčćđšž\s\-]+?)(?=\s+Hrvatska|\s+E-mail|\s+Mob|\s+Tel|$)')

def _kosarkapgz_extract_osobe(html_doc: str) -> list[dict]:
    """Return [{'ime':, 'prezime':, 'funkcija':, 'period':}] for ?page_id=234."""
    if not html_doc:
        return []
    # Find the entry-content div by hand (depth-balanced) to avoid grabbing
    # JSON-LD / scripts. WordPress always wraps the body in <div class="entry-content">.
    i = html_doc.find('class="entry-content')
    if i < 0:
        return []
    j = html_doc.find('>', i) + 1
    depth, k = 1, j
    while k < len(html_doc) and depth > 0:
        no = html_doc.find('<div', k)
        nc = html_doc.find('</div>', k)
        if nc < 0:
            break
        if 0 <= no < nc:
            depth += 1; k = no + 4
        else:
            depth -= 1; k = nc + 6
    body = html_doc[j:k-6] if depth == 0 else html_doc[j:]
    out: list[dict] = []
    for raw in _KPGZ_LI_RE.findall(body):
        text = html.unescape(_KPGZ_TAG_RE.sub(' ', raw)).strip()
        text = re.sub(r'\s+', ' ', text)
        if not text:
            continue
        # Split on en/em/ASCII dash → "Name Surname" — "funkcija"
        parts = _KPGZ_DASH_RE.split(text, maxsplit=1)
        person = parts[0].strip()
        funkcija = parts[1].strip().lower() if len(parts) > 1 else None
        # Split person → first name(s) + last name on last whitespace.
        toks = person.split()
        if len(toks) < 2:
            continue
        ime = ' '.join(toks[:-1])
        prezime = toks[-1]
        out.append({
            'ime': ime, 'prezime': prezime,
            'funkcija': funkcija, 'period': None,
        })
    return out

def _kosarkapgz_extract_adresa(html_doc: str) -> dict:
    """Return {'ulica','broj','grad','postal_code','telefon','mobitel','email','web'}.

    Prefers the og:description meta on /?page_id=70 — it is one clean string
    the site re-renders verbatim from the page body. Falls back to stripping
    the page itself if the meta is missing.
    """
    if not html_doc:
        return {}
    desc_match = _KPGZ_OG_DESC.search(html_doc)
    text = html.unescape(desc_match.group(1)) if desc_match else _strip_tags(html_doc)
    text = re.sub(r'\s+', ' ', text).strip()
    if not text:
        return {}
    # The og:description glues fields without whitespace
    # ("11/III51000 RijekaHrvatska E-mail: a@b.com, c@d.comMob: ...").
    # Inject separators so the field regexes don't gobble across boundaries.
    text = re.sub(r'(?<=\D)(?=\d{5}\s)', ' ', text)
    text = re.sub(r'(?<=[a-zćčđšž])(?=[A-ZČĆĐŠŽ])', ' ', text)
    text = re.sub(r'(?i)\s*(Hrvatska|E-?mail|Mob|Tel|IBAN|http)', r' \1', text)
    text = re.sub(r'\s+', ' ', text).strip()
    out: dict = {}
    m = _KPGZ_ULICA_RE.search(text)
    if m:
        out['ulica'] = m.group(1).strip()
        out['broj']  = m.group(2).strip()
    m = _KPGZ_POSTAL_RE.search(text)
    if m:
        out['postal_code'] = m.group(1)
        out['grad']        = m.group(2).strip()
    emails = list(dict.fromkeys(_KPGZ_EMAIL_RE.findall(text)))
    if emails:
        out['email'] = emails[0]
        if len(emails) > 1:
            out['email_extra'] = emails[1:]
    m = _KPGZ_TEL_RE.search(text)
    if m:
        out['telefon'] = re.sub(r'\s+', ' ', m.group(1)).strip()
    m = _KPGZ_MOB_RE.search(text)
    if m:
        out['mobitel'] = re.sub(r'\s+', ' ', m.group(1)).strip()
    # Pick the first http(s) URL that isn't a mailto / schema.org probe.
    for u in _KPGZ_WEB_RE.findall(text):
        if 'schema.org' in u: continue
        out['web'] = u.rstrip('.,')
        break
    return out

# Canonical name kept in civic.entities for the Košarkaški savez PGŽ.
# (civic.entities has rows for many county basketball savezi but NOT for the
# PGŽ one as of 2026-05-10 — we insert it on first persist call.)
_KPGZ_KSPGZ_NAME = 'Košarkaški savez Primorsko-goranske županije'
_KPGZ_KSPGZ_ALIASES = (
    _KPGZ_KSPGZ_NAME.lower(),
    'košarkaški savez pgž',
    'kosarkaski savez pgz',
    'košarkaški savez primorsko-goranske županije',
    'kosarkaski savez primorsko-goranske zupanije',
)

def _kosarkapgz_is_kspgz(naziv: str) -> bool:
    """True for the PGŽ-county savez specifically (not 'Grada Rijeke', not HKS)."""
    if not naziv: return False
    n = naziv.lower().strip()
    if any(a in n for a in _KPGZ_KSPGZ_ALIASES):
        return True
    # Permissive fallback: anything matching košarkaški savez + PGŽ token.
    return ('košarkaški savez' in n or 'kosarkaski savez' in n) and (
        'pgž' in n or 'pgz' in n or 'primorsko' in n
    )

def _kosarkapgz_persist_kspgz(osobe: list[dict], adresa: dict) -> dict:
    """Upsert civic.entities row for KSPGŽ + civic.persons for each Upravni
    odbor member. Also updates pgz_sport.savezi (id=6, KSPGŽ) for predsjednik,
    email, telefon, web. All writes carry provenance.source='kosarkapgz.com'
    via SET LOCAL so the trigger logs them correctly.

    Idempotent: persons keyed on (lower(name), entity_id, metadata->>'source').
    """
    # # KSPGZ-SAVEZI-PATCH-20260510
    res = {
        'entity_id': None, 'entity_action': 'noop',
        'persons_inserted': 0, 'persons_updated': 0,
        'savez_updates': {},   # NEW: pgz_sport.savezi field changes
        'errors': [],
    }
    try:
        c = _db()
        c.autocommit = False  # need a transaction for SET LOCAL
        with c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
            # ── provenance attribution for this whole transaction ─────────
            cur.execute("SET LOCAL provenance.source = 'kosarkapgz.com'")
            cur.execute("SET LOCAL provenance.source_url = 'https://kosarkapgz.com/'")
            cur.execute("SET LOCAL provenance.edited_by = 'kosarkapgz_scraper@enrich_router'")
            cur.execute("SET LOCAL provenance.trust_tier = '2'")  # 1=visit, 2=verified site, 3=oss
            from datetime import date as _date
            cur.execute("SET LOCAL provenance.source_date = %s", (_date.today().isoformat(),))
            # ── entity upsert ─────────────────────────────────────────────
            cur.execute(
                "SELECT id, address, city, county, phone, email, website "
                "FROM civic.entities WHERE lower(name) = lower(%s) LIMIT 1",
                [_KPGZ_KSPGZ_NAME],
            )
            row = cur.fetchone()
            if row:
                eid = row['id']
                # Patch any missing address-ish field. Never overwrite existing
                # non-null values — they may have been hand-curated.
                set_clauses, params = [], []
                addr_full = None
                if adresa.get('ulica') and adresa.get('broj'):
                    addr_full = f"{adresa['ulica']} {adresa['broj']}"
                if addr_full and not row.get('address'):
                    set_clauses.append('address = %s'); params.append(addr_full)
                if adresa.get('grad') and not row.get('city'):
                    set_clauses.append('city = %s'); params.append(adresa['grad'])
                if not row.get('county'):
                    set_clauses.append('county = %s'); params.append('Primorsko-goranska')
                if adresa.get('telefon') and not row.get('phone'):
                    set_clauses.append('phone = %s'); params.append(adresa['telefon'])
                if adresa.get('email') and not row.get('email'):
                    set_clauses.append('email = %s'); params.append(adresa['email'])
                if adresa.get('postal_code') and adresa.get('grad'):
                    set_clauses.append("postal_code = %s"); params.append(adresa['postal_code'])
                if not row.get('website'):
                    set_clauses.append('website = %s'); params.append('https://kosarkapgz.com/')
                # Always extend sources[] with our marker.
                set_clauses.append(
                    "sources = (SELECT array_agg(DISTINCT x) FROM unnest(coalesce(sources, ARRAY[]::text[]) || ARRAY['kosarkapgz.com page_id=70']) AS x)"
                )
                set_clauses.append('updated_at = now()')
                params.append(eid)
                cur.execute(
                    f"UPDATE civic.entities SET {', '.join(set_clauses)} WHERE id = %s",
                    params,
                )
                res['entity_id'] = eid
                res['entity_action'] = 'updated'
            else:
                addr_full = None
                if adresa.get('ulica') and adresa.get('broj'):
                    addr_full = f"{adresa['ulica']} {adresa['broj']}"
                cur.execute(
                    "INSERT INTO civic.entities "
                    "  (name, short_name, entity_type, address, city, county, "
                    "   postal_code, phone, email, website, metadata, sources) "
                    "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) "
                    "RETURNING id",
                    [_KPGZ_KSPGZ_NAME, 'KSPGŽ', 'sportski_savez',
                     addr_full, adresa.get('grad'), 'Primorsko-goranska',
                     adresa.get('postal_code'), adresa.get('telefon'),
                     adresa.get('email'), 'https://kosarkapgz.com/',
                     json.dumps({'sport': 'košarka', 'level': 'regional',
                                 'aliases': ['KSPGŽ', 'Košarkaški savez PGŽ']}),
                     ['kosarkapgz.com page_id=70']],
                )
                res['entity_id'] = cur.fetchone()['id']
                res['entity_action'] = 'inserted'

            # ── persons upsert ────────────────────────────────────────────
            eid = res['entity_id']
            SOURCE_KEY = 'kosarkapgz.com page_id=234'
            for p in osobe or []:
                fullname = f"{p['ime']} {p['prezime']}".strip()
                if not fullname:
                    continue
                cur.execute(
                    "SELECT id, function, metadata FROM civic.persons "
                    "WHERE lower(name) = lower(%s) AND entity_id = %s "
                    "  AND coalesce(metadata->>'source','') = %s "
                    "LIMIT 1",
                    [fullname, eid, SOURCE_KEY],
                )
                existing = cur.fetchone()
                meta = {
                    'source': SOURCE_KEY,
                    'source_url': 'https://kosarkapgz.com/?page_id=234',
                    'ime': p.get('ime'),
                    'prezime': p.get('prezime'),
                    'period': p.get('period'),
                    'scraped_at': int(time.time()),
                }
                if existing:
                    cur.execute(
                        "UPDATE civic.persons SET function = %s, metadata = %s, "
                        "  updated_at = now() WHERE id = %s",
                        [p.get('funkcija') or existing.get('function'),
                         json.dumps(meta), existing['id']],
                    )
                    res['persons_updated'] += 1
                else:
                    cur.execute(
                        "INSERT INTO civic.persons "
                        "  (name, function, level, entity_id, county, city, metadata, trust_tier) "
                        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s)",
                        [fullname, p.get('funkcija'), 'savez',
                         eid, 'Primorsko-goranska', adresa.get('grad'),
                         json.dumps(meta), 3],
                    )
                    res['persons_inserted'] += 1

            # ── pgz_sport.savezi update (NEW 2026-05-10) ──────────────────
            # Update predsjednik / tajnik / email / telefon / web on the
            # canonical KSPGŽ row in pgz_sport.savezi (id=6 by default; lookup
            # by naziv if id moves). Provenance GUC was set at top of txn,
            # so trigger trg_provenance_log captures source='kosarkapgz.com'.
            try:
                # Predsjednik = first Upravni odbor member whose funkcija ~ 'predsjednik'
                predsj = None
                for p in (osobe or []):
                    if (p.get('funkcija') or '').lower().startswith('predsj'):
                        predsj = f"{p.get('ime','')} {p.get('prezime','')}".strip()
                        break

                cur.execute("SELECT id, predsjednik, tajnik, email, telefon, web "
                            "FROM pgz_sport.savezi WHERE lower(naziv) IN "
                            "  (lower(%s), lower(%s)) LIMIT 1",
                            ['Košarkaški savez PGŽ', 'Košarkaški savez Primorsko-goranske županije'])
                srow = cur.fetchone()
                if srow:
                    sid = srow['id']
                    sets, params, applied = [], [], {}

                    def _set_if_changed(col, new_val):
                        if not new_val: return
                        cur_val = srow.get(col)
                        if cur_val and str(cur_val).strip() == str(new_val).strip():
                            return
                        sets.append(f"{col} = %s")
                        params.append(new_val)
                        applied[col] = {'before': cur_val, 'after': new_val}

                    _set_if_changed('predsjednik', predsj)
                    # tajnik not reliably published; skip unless future scrape adds it
                    if adresa.get('email'):
                        _set_if_changed('email', adresa['email'])
                    if adresa.get('telefon'):
                        _set_if_changed('telefon', adresa['telefon'])
                    # Web: prefer kosarkapgz.com over generic sport-pgz.hr aggregator
                    if not srow.get('web') or 'sport-pgz.hr' in (srow.get('web') or ''):
                        _set_if_changed('web', 'https://kosarkapgz.com/')

                    if sets:
                        params.append(sid)
                        cur.execute(
                            f"UPDATE pgz_sport.savezi SET {', '.join(sets)} WHERE id = %s",
                            params,
                        )
                        res['savez_updates'] = {'savez_id': sid, 'fields': applied}
                    else:
                        res['savez_updates'] = {'savez_id': sid, 'fields': {}, 'note': 'no diff'}
                else:
                    res['savez_updates'] = {'note': 'KSPGŽ row not found in pgz_sport.savezi'}
            except Exception as e:
                res['errors'].append(f"savez_update: {e}")

            c.commit()
    except Exception as e:
        try: c.rollback()
        except Exception: pass
        res['errors'].append(str(e))
    finally:
        try: c.close()
        except Exception: pass
    # Bust the kosarkapgz cache so next call re-fetches and re-persists
    try:
        _cache_delete('kosarkapgz:savez_meta:v1')
    except Exception:
        pass
    return res


def _fetch_primary_site(url: str) -> Optional[dict]:
    body = _http_get(url, timeout=6)
    if not body: return None
    text = _strip_tags(body)
    meta = _extract_meta(body, url)
    return {
        'source': urllib.parse.urlparse(url).hostname or url,
        'url': url,
        'title': meta.get('title'),
        'extract': meta.get('description') or text[:500],
        'raw_text': text[:8000],
    }


# ─── Anti-halucinacija safeguard (added 2026-05-10 per Damir directive) ────
# When LLM-synthesised descriptions mention famous Croatian sports/political
# names, those mentions MUST be backed by source text — otherwise we strip
# the offending sentence and log to civic.fact_check_log. Triggered after
# Damir caught a "Toni Kukoč briljirao" fabrication for KK Kvarner 2010
# (Kukoč played for Jugoplastika Split, not Kvarner).
FAMOUS_PEOPLE_REQUIRE_PROOF = (
    'Kukoč', 'Petrović', 'Modrić', 'Šuker', 'Boban', 'Kovač',
    'Dalić', 'Mandžukić', 'Rakitić', 'Lovren', 'Vida', 'Perišić',
    'Tuđman', 'Račan', 'Mesić', 'Josipović', 'Milanović',
    'Bilić', 'Štimac', 'Prosinečki', 'Suker',     # Suker = Šuker without diacritic
    'Kostelić', 'Janica', 'Blanka Vlašić', 'Sara Kolak',
    'Sandra Perković', 'Filip Hrgović',
)

def _fact_check_response(text: str, evidence: list[str], kind: Optional[str], naziv: Optional[str]) -> str:
    """Strip sentences mentioning famous people not present in source evidence.
    Logs each strip to civic.fact_check_log. Croatian-diacritic-aware (substring match).
    Fail-soft on logging — if DB write fails the strip still happens.
    """
    if not text:
        return text
    blob = ' '.join(e for e in (evidence or []) if e)
    out = text
    for famous in FAMOUS_PEOPLE_REQUIRE_PROOF:
        if famous not in out:
            continue
        if famous in blob:
            # Mentioned in sources → keep, mark validated in log so we know which famous names DO have proof
            try:
                _c = _db(); _cur = _c.cursor()
                _cur.execute(
                    "INSERT INTO civic.fact_check_log "
                    "(entity_kind, halucinated_text, famous_person, source_validated, sources_searched, metadata) "
                    "VALUES (%s, %s, %s, TRUE, %s, %s::jsonb)",
                    (kind, '(in sources)', famous, len(evidence or []),
                     json.dumps({'naziv': naziv, 'mode': 'kept_validated'}, ensure_ascii=False))
                )
                _cur.close(); _c.close()
            except Exception:
                pass
            continue
        # Famous mention without source backing — strip the sentence(s)
        pattern = re.compile(r'[^.!?]*' + re.escape(famous) + r'[^.!?]*[.!?]?', re.UNICODE)
        stripped_match = pattern.search(out)
        stripped_text = stripped_match.group(0) if stripped_match else ''
        out = pattern.sub('', out)
        try:
            _c = _db(); _cur = _c.cursor()
            _cur.execute(
                "INSERT INTO civic.fact_check_log "
                "(entity_kind, halucinated_text, famous_person, source_validated, sources_searched, metadata) "
                "VALUES (%s, %s, %s, FALSE, %s, %s::jsonb)",
                (kind, (stripped_text or text)[:1000], famous, len(evidence or []),
                 json.dumps({'naziv': naziv, 'mode': 'stripped_no_proof'}, ensure_ascii=False))
            )
            _cur.close(); _c.close()
        except Exception:
            pass
    # Collapse double spaces / orphaned commas left after sentence removal
    out = re.sub(r'\s+', ' ', out)
    out = re.sub(r'\s+([.,;:])', r'\1', out)
    return out.strip()


# ─── DeepSeek (optional, fail-soft) ─────────────────────────────────────
def _deepseek_describe(naziv: str, kind: str, evidence: list[str]) -> Optional[str]:
    if not DEEPSEEK_KEY or not evidence: return None
    joined = "\n---\n".join(e for e in evidence if e)[:6000]
    if not joined.strip(): return None
    prompt = (f"Iz dolje navedenih izvora napiši profesionalni opis za "
              f"{kind} '{naziv}' na hrvatskom jeziku. 3-5 rečenica. "
              f"Bez uvoda 'Evo opisa', samo tekst.\n\nIZVORI:\n{joined}")
    # STRICT FACT RULE — anti-halucinacija system prompt (Damir 2026-05-10).
    # Any famous-name mention without source backing will also be stripped
    # post-response by _fact_check_response().
    system_prompt = (
        "Pišeš sažete činjenične opise sportskih organizacija na hrvatskom. "
        "STRICT FACT RULE: NE SPOMINJI specifične povijesne osobe, datume ili dostignuća "
        "osim ako su DOSLOVNO prisutni u priloženim IZVORIMA. Nikada ne nagađaj o slavnim "
        "sportašima (Kukoč, Petrović, Modrić, Šuker, Boban, Kostelić, Vlašić, ...) "
        "osim ako su izrijekom navedeni u IZVORIMA. Ako nisi siguran, IZOSTAVI. "
        "Ako izvori ne sadrže povijesnu informaciju, drži se sadašnjeg djelovanja kluba."
    )
    payload = {
        "model": "deepseek-chat",
        "messages": [
            {"role": "system", "content": system_prompt},
            {"role": "user",   "content": prompt},
        ],
        "max_tokens": 280, "temperature": 0.3,
    }
    req = urllib.request.Request(
        DEEPSEEK_URL, data=json.dumps(payload).encode('utf-8'),
        headers={'Authorization': 'Bearer ' + DEEPSEEK_KEY,
                 'Content-Type': 'application/json',
                 'User-Agent': UA}, method='POST')
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            d = json.loads(r.read().decode('utf-8'))
        text = d.get('choices', [{}])[0].get('message', {}).get('content', '').strip()
        if not text:
            return None
        # Post-LLM fact validator — strip famous-name mentions lacking source backing
        text = _fact_check_response(text, evidence, kind, naziv)
        return text or None
    except Exception:
        return None


# ─── Row loaders & display name ─────────────────────────────────────────
def _load_row(kind: str, eid: int) -> dict:
    if kind == 'klub':
        row = _fetch_one("""SELECT id, naziv, oib, sport, grad, predsjednik, tajnik,
                                   web, web_stranica, email, telefon, ciljevi, opis_djelatnosti,
                                   sjediste, godina_osnutka, savez_id, scrape_url, source_url,
                                   metadata
                            FROM pgz_sport.klubovi WHERE id=%s""", (eid,))
    elif kind == 'savez':
        row = _fetch_one("""SELECT id, naziv, oib, sport, predsjednik, tajnik, email, telefon, web,
                                   adresa, godina_osnutka, source_url, metadata
                            FROM pgz_sport.savezi WHERE id=%s""", (eid,))
    elif kind == 'sportas':
        row = _fetch_one("""SELECT c.id, c.ime, c.prezime, c.sport, c.klub_id, c.profile_url,
                                   c.slika_url, c.source_url, c.source, c.source_id,
                                   c.hns_igrac_id, c.biografija,
                                   c.datum_rodenja, c.mjesto_rodenja, c.broj_dresa,
                                   c.visina_cm, c.tezina_kg, c.dominantna_noga, c.oib,
                                   c.vanjski_id, c.metadata,
                                   k.sport AS klub_sport, k.naziv AS klub_naziv
                            FROM pgz_sport.clanovi c
                            LEFT JOIN pgz_sport.klubovi k ON k.id = c.klub_id
                            WHERE c.id=%s""", (eid,))
        # Fall back to klub.sport when c.sport is empty
        if row and not row.get('sport') and row.get('klub_sport'):
            row['sport'] = row['klub_sport']
    else:
        raise HTTPException(400, "kind must be klub|savez|sportas")
    if not row:
        raise HTTPException(404, kind + " not found")
    return row


def _display_name(kind: str, row: dict) -> str:
    if kind == 'sportas':
        return ((row.get('ime') or '') + ' ' + (row.get('prezime') or '')).strip()
    return row.get('naziv', '') or ''


# ─── Sport federations map (loaded once, refresh on file mtime) ─────────
_SPORT_FED_PATH = '/opt/pgz-sport/data/sport_federations.json'
_SPORT_FED_CACHE: dict[str, Any] = {'mtime': 0, 'data': {}, 'aliases': {}, 'media': []}


def _load_sport_feds() -> tuple[dict, dict, list]:
    """Return (feds, aliases, local_media) — refreshed when JSON changes."""
    try:
        st = os.stat(_SPORT_FED_PATH)
    except FileNotFoundError:
        return ({}, {}, [])
    if st.st_mtime != _SPORT_FED_CACHE['mtime']:
        try:
            with open(_SPORT_FED_PATH, 'r', encoding='utf-8') as f:
                raw = json.load(f)
        except Exception:
            return (_SPORT_FED_CACHE['data'],
                    _SPORT_FED_CACHE['aliases'],
                    _SPORT_FED_CACHE['media'])
        aliases = raw.pop('_aliases', {}) if isinstance(raw, dict) else {}
        media   = raw.pop('_local_media_pgz', []) if isinstance(raw, dict) else []
        raw.pop('_meta', None)
        _SPORT_FED_CACHE.update(mtime=st.st_mtime, data=raw, aliases=aliases, media=media)
    return (_SPORT_FED_CACHE['data'],
            _SPORT_FED_CACHE['aliases'],
            _SPORT_FED_CACHE['media'])


def _normalize_sport(sport: Optional[str]) -> Optional[str]:
    if not sport: return None
    s = sport.strip().lower()
    feds, aliases, _ = _load_sport_feds()
    while s in aliases:
        nxt = aliases[s]
        if nxt == s: break
        s = nxt
    return s if s in feds else None


def _sport_fed(sport: Optional[str]) -> Optional[dict]:
    """Resolve sport → federations entry (or None)."""
    norm = _normalize_sport(sport)
    if not norm: return None
    feds, _, _ = _load_sport_feds()
    return feds.get(norm)


def _research_links(naziv, kind, grad=None, sport: Optional[str] = None, row: Optional[dict] = None):
    base_q = (naziv or '').strip()
    q = (base_q + ' ' + grad) if grad else base_q
    qenc = urllib.parse.quote(q)
    out = []
    # Prefer DIRECT profile/source link if entity already has one (e.g. HNS Semafor)
    if row:
        direct = row.get('profile_url') or row.get('source_url') or row.get('scrape_url') or row.get('web') or row.get('web_stranica')
        if direct and isinstance(direct, str) and direct.startswith(('http://','https://')):
            try:
                host = urllib.parse.urlparse(direct).hostname or ''
            except Exception:
                host = ''
            label = 'Vanjski profil'
            icon = '🔗'
            if 'hns' in host: label, icon = 'HNS profil', '⚽'
            elif 'transfermarkt' in host: label, icon = 'Transfermarkt', '⚽'
            elif 'wikipedia' in host: label, icon = 'Wikipedia', '📚'
            elif host.endswith('.hr') or host.endswith('.com'): label, icon = 'Službena stranica', '🌐'
            out.append({'label': label, 'icon': icon, 'url': direct, 'is_direct': True})
    out += [
        {'label': 'Google',       'icon': '🔍', 'url': 'https://www.google.com/search?q=' + qenc},
        {'label': 'Wikipedia HR', 'icon': '📚', 'url': 'https://hr.wikipedia.org/w/index.php?search=' + qenc},
        {'label': 'sport-pgz.hr', 'icon': '🏅', 'url': 'https://sport-pgz.hr/?s=' + qenc},
    ]
    if kind == 'klub':
        out.append({'label': 'Sportilus',       'icon': '⬡', 'url': 'https://www.sportilus.com/?s=' + qenc})
        out.append({'label': 'Sudski registar', 'icon': '⚖', 'url': 'https://sudreg.pravosudje.hr/registar/oc/index.html'})

    # Sport-specific federation links (replace static HNS/transfermarkt for sportas)
    fed = _sport_fed(sport) if sport else None
    if kind == 'sportas':
        if fed and isinstance(fed.get('national'), dict):
            nat = fed['national']
            search = (nat.get('search_url') or nat.get('url') or '').replace('{q}', qenc)
            if search:
                out.append({'label': nat.get('name', 'Nacionalni savez'),
                            'icon': '🏆', 'url': search})
        if fed and isinstance(fed.get('pgz'), dict):
            pgz = fed['pgz']
            url = pgz.get('search_url') or pgz.get('url') or ''
            if url:
                out.append({'label': pgz.get('name', 'PGŽ savez'),
                            'icon': '🏟', 'url': url.replace('{q}', qenc)})
        if not fed:
            # No mapping for this sport → keep transfermarkt as legacy fallback
            # Prefer direct /igraci/{id}/{slug} when hns_igrac_id exists
            hns_id = (clan or {}).get('hns_igrac_id') if 'clan' in dir() else None
            if not hns_id:
                # Try get from current clan dict
                try: hns_id = clan.get('hns_igrac_id') if isinstance(clan, dict) else None
                except: pass
            if hns_id:
                # Slugify ime+prezime: "Franko Andrijašević" → "franko-andrijasevic"
                _ime = (clan.get('ime','') if isinstance(clan, dict) else '') or ''
                _prez = (clan.get('prezime','') if isinstance(clan, dict) else '') or ''
                _full = (_ime + ' ' + _prez).strip().lower()
                _slug = _full
                for old_c, new_c in [('č','c'),('ć','c'),('ž','z'),('š','s'),('đ','d'),(' ','-')]:
                    _slug = _slug.replace(old_c, new_c)
                _slug = re.sub(r'[^a-z0-9-]', '', _slug)
                out.append({'label': 'HNS Semafor (profil)', 'icon': '⚽', 'url': f'https://semafor.hns.family/igraci/{hns_id}/{_slug}/'})
            else:
                out.append({'label': 'HNS Semafor (pretraga)', 'icon': '⚽', 'url': 'https://semafor.hns.family/?s=' + qenc})
            out.append({'label': 'transfermarkt','icon': '⚽', 'url': 'https://www.transfermarkt.com/schnellsuche/ergebnis/schnellsuche?query=' + qenc})
        # Local PGŽ media for any sportas
        _, _, media = _load_sport_feds()
        for m in media:
            url = (m.get('search_url') or '').replace('{q}', qenc)
            if url:
                out.append({'label': m.get('name', 'Lokalni medij'),
                            'icon': '📰', 'url': url})
    if kind == 'savez':
        out.append({'label': 'sport-pgz.hr savezi', 'icon': '🏅', 'url': 'https://sport-pgz.hr/savezi'})
    return out


# ─── Proposal pipelines ─────────────────────────────────────────────────
def _name_tokens(naziv: str) -> list[str]:
    """Significant tokens from entity name (≥4 chars, deaccented)."""
    import unicodedata
    s = unicodedata.normalize('NFKD', naziv or '').encode('ascii', 'ignore').decode('ascii').lower()
    toks = [t for t in re.split(r'[^a-z0-9]+', s) if len(t) >= 4]
    stop = {'klub','udruga','sportski','sport','kosarkaski','kosarka','nogometni',
            'rukometni','savez','rijeka','primorsko','goranski','grad','grada','centar'}
    return [t for t in toks if t not in stop] or toks


def _is_relevant(source: dict, tokens: list[str]) -> bool:
    """A source is 'relevant' only if the page actually mentions the entity name."""
    if not tokens: return True
    import unicodedata
    blob = (source.get('title') or '') + ' ' + (source.get('extract') or '') + ' ' + (source.get('raw_text') or '')
    blob = unicodedata.normalize('NFKD', blob.lower()).encode('ascii', 'ignore').decode('ascii')
    return any(t in blob for t in tokens)



# ─── Klub domain guesser (HR slug → candidate URLs → HEAD probe) ────────
import re as _re_klg

def _slugify_klub(naziv: str) -> str:
    if not naziv: return ""
    s = naziv.lower()
    repl = (("č","c"),("ć","c"),("ž","z"),("š","s"),("đ","d"),
            ('"',''),("'",""),("(",""),(")",""),(",",""),(".",""),
            ("/",""),("\\",""))
    for a,b in repl: s = s.replace(a,b)
    s = _re_klg.sub(r"[^a-z0-9]+", "-", s).strip("-")
    return s

def _klub_domain_candidates(naziv: str) -> list[str]:
    """Generate ranked candidate URLs from club name."""
    if not naziv: return []
    s = _slugify_klub(naziv)
    # Strip common prefixes for cleaner domains
    base = s
    for pref in ("hnk-","nk-","rk-","kk-","ok-","bk-","gk-","tk-","ak-","hbk-"):
        if base.startswith(pref):
            base = base[len(pref):]; break
    # also try short prefix-ed variants
    short = base.split("-")[0] if base else ""
    candidates = []
    sports_prefixes = ["nk-","hnk-","rk-","kk-","bk-","ok-","ak-","tk-"]
    # full slug with original prefix
    for tld in (".hr",".com",".eu",".info"):
        candidates.append(f"https://{s}{tld}")
        candidates.append(f"https://www.{s}{tld}")
    # base-only
    for tld in (".hr",".com"):
        candidates.append(f"https://{base}{tld}")
        candidates.append(f"https://www.{base}{tld}")
    # try sport prefixes if name doesn't already have one
    if not any(s.startswith(p) for p in sports_prefixes):
        for sp in sports_prefixes[:5]:
            for tld in (".hr",".com"):
                candidates.append(f"https://{sp}{base}{tld}")
    # dedup, preserve order
    seen, out = set(), []
    for c in candidates:
        if c not in seen:
            seen.add(c); out.append(c)
    return out[:20]

def _probe_klub_url(url: str, naziv_tokens: list, timeout: int = 5) -> Optional[dict]:
    """HEAD/GET probe; return doc with raw_text if URL is alive AND mentions club tokens."""
    try:
        import requests
        r = requests.get(url, timeout=timeout, allow_redirects=True,
                         headers={"User-Agent":"Mozilla/5.0 RinetEnrichBot/1.0"})
        if r.status_code != 200: return None
        if len(r.text) < 200: return None
        text = r.text.lower()
        # Must mention at least one distinctive token from name
        toks = [t.lower() for t in (naziv_tokens or []) if len(t) > 2]
        if toks and not any(t in text for t in toks):
            return None
        return {"source": "domain_probe", "url": r.url, "raw_text": r.text[:50000]}
    except Exception:
        return None

def _guess_klub_domains(naziv: str, tokens: list) -> Optional[dict]:
    """Parallel probe candidates (5 workers, 4s timeout each); first hit wins."""
    from concurrent.futures import ThreadPoolExecutor, as_completed
    candidates = _klub_domain_candidates(naziv)
    if not candidates: return None
    with ThreadPoolExecutor(max_workers=8) as ex:
        futs = {ex.submit(_probe_klub_url, url, tokens, 4): url for url in candidates[:16]}
        try:
            for fut in as_completed(futs, timeout=10):
                try:
                    doc = fut.result()
                    if doc:
                        # Cancel remaining (best effort)
                        for f in futs:
                            if not f.done(): f.cancel()
                        return doc
                except Exception:
                    continue
        except TimeoutError:
            # All probes timed out — graceful None instead of 500
            for f in futs:
                if not f.done(): f.cancel()
            return None
    return None

def _scrape_klub_subpages(base_url: str, tokens: list) -> str:
    """Fetch /kontakt /uprava /o-nama /o-klubu and concat texts.

    Parallelized via ThreadPoolExecutor (8 workers, 3s per-URL timeout, 8s total budget).
    Was sequential 8x4s = up to 32s.
    """
    if not base_url: return ""
    import requests
    from concurrent.futures import ThreadPoolExecutor, as_completed
    base = base_url.rstrip("/")
    paths = ["/kontakt","/uprava","/o-nama","/o-klubu","/predsjednik","/klub","/contact","/about"]
    headers = {"User-Agent":"Mozilla/5.0 RinetEnrichBot/1.0"}

    def _fetch(path):
        try:
            r = requests.get(base + path, timeout=3, allow_redirects=True, headers=headers)
            if r.status_code == 200 and len(r.text) > 200:
                return r.text[:30000]
        except Exception:
            pass
        return None

    accum = []
    with ThreadPoolExecutor(max_workers=8) as ex:
        futs = {ex.submit(_fetch, p): p for p in paths}
        try:
            for fut in as_completed(futs, timeout=8):
                try:
                    txt = fut.result()
                    if txt: accum.append(txt)
                except Exception:
                    continue
        except Exception:
            for f in futs:
                if not f.done(): f.cancel()
    return "\n\n".join(accum)


def _propose_for_klub(row: dict) -> dict:
    naziv = row.get('naziv') or ''
    # Only consider HTTP(S) URLs as valid primary sources — skip placeholder strings like 'godisnjak_2025'
    raw_primary = row.get('web') or row.get('web_stranica') or row.get('source_url') or row.get('scrape_url')
    primary = raw_primary if (raw_primary and isinstance(raw_primary, str) and raw_primary.startswith(('http://','https://'))) else None
    sources, evidence = [], []
    tokens_pre = _name_tokens(naziv)
    pdoc = _fetch_primary_site(primary) if primary else None
    if not pdoc:
        # No valid web in DB — try to guess domain from club name
        pdoc = _guess_klub_domains(naziv, tokens_pre)
        if pdoc:
            # Also fetch subpages for richer evidence
            sub = _scrape_klub_subpages(pdoc.get('url',''), tokens_pre)
            if sub:
                pdoc['raw_text'] = (pdoc.get('raw_text','') + '\n\n' + sub)[:120000]
    elif pdoc:
        # Have primary site — also fetch its subpages
        sub = _scrape_klub_subpages(pdoc.get('url') or primary, tokens_pre)
        if sub:
            pdoc['raw_text'] = (pdoc.get('raw_text','') + '\n\n' + sub)[:120000]
    if pdoc: sources.append(pdoc); evidence.append(pdoc.get('raw_text') or pdoc.get('extract') or '')
    wiki = _wiki_summary(naziv)
    if wiki: sources.append(wiki); evidence.append(wiki.get('extract') or '')
    spz = _sport_pgz_search(naziv)
    if spz: sources.append(spz); evidence.append(spz.get('raw_text') or spz.get('extract') or '')
    # kosarkapgz.com — only if this looks like a PGŽ basketball club
    if _kosarkapgz_is_basketball_pgz(naziv, row.get('sport')):
        kpgz = _kosarkapgz_search_posts(naziv, limit=5)
        if kpgz: sources.append(kpgz); evidence.append(kpgz.get('raw_text') or '')

    tokens = _name_tokens(naziv)
    relevant = [s for s in sources if _is_relevant(s, tokens)]
    relevant_blob = '\n\n'.join((s.get('raw_text') or s.get('extract') or '') for s in relevant)

    proposed: dict[str, Any] = {}
    # web/email/telefon: ONLY from sources actually mentioning the entity
    if not row.get('web'):
        u = _find_official_web(relevant_blob, naziv)
        if u: proposed['web'] = u
    if not row.get('email'):
        e = _find_email(relevant_blob)
        if e: proposed['email'] = e
    if not row.get('telefon'):
        t = _find_phone(relevant_blob)
        if t: proposed['telefon'] = t
    if not row.get('opis_djelatnosti'):
        descr_evidence = [(s.get('raw_text') or s.get('extract') or '') for s in relevant] or evidence
        descr = _deepseek_describe(naziv, 'sportski klub', descr_evidence)
        if not descr:
            for s in (relevant or sources):
                if s.get('extract') and len(s['extract']) >= 80:
                    descr = s['extract']; break
        if descr: proposed['opis_djelatnosti'] = descr.strip()[:2000]
    return {'proposed': proposed, 'sources': sources}


def _propose_for_savez(row: dict) -> dict:
    naziv = row.get('naziv') or ''
    primary = row.get('web') or row.get('source_url')
    sources, evidence = [], []
    pdoc = _fetch_primary_site(primary) if primary else None
    if pdoc: sources.append(pdoc); evidence.append(pdoc.get('raw_text') or '')
    wiki = _wiki_summary(naziv)
    if wiki: sources.append(wiki); evidence.append(wiki.get('extract') or '')
    spz = _sport_pgz_search(naziv)
    if spz: sources.append(spz); evidence.append(spz.get('raw_text') or '')
    # kosarkapgz.com — fetch federation meta for any basketball savez
    if _kosarkapgz_is_basketball_pgz(naziv, row.get('sport')):
        kpgz = _kosarkapgz_savez_meta()
        if kpgz: sources.append(kpgz); evidence.append(kpgz.get('raw_text') or '')

    tokens = _name_tokens(naziv)
    relevant = [s for s in sources if _is_relevant(s, tokens)]
    relevant_blob = '\n\n'.join((s.get('raw_text') or s.get('extract') or '') for s in relevant)

    proposed: dict[str, Any] = {}
    if not row.get('web'):
        u = _find_official_web(relevant_blob, naziv)
        if u: proposed['web'] = u
    if not row.get('email'):
        e = _find_email(relevant_blob)
        if e: proposed['email'] = e
    if not row.get('telefon'):
        t = _find_phone(relevant_blob)
        if t: proposed['telefon'] = t
    return {'proposed': proposed, 'sources': sources}


# ─── HNS Semafor parsing ────────────────────────────────────────────────
_HNS_BASE = 'https://semafor.hns.family'

def _slugify(name: str) -> str:
    import unicodedata
    s = unicodedata.normalize('NFKD', name or '').encode('ascii', 'ignore').decode('ascii').lower()
    return re.sub(r'[^a-z0-9]+', '-', s).strip('-')

def _hns_url_from_row(row: dict) -> Optional[str]:
    """Try to build a semafor.hns.family /igraci/ URL for this row."""
    # 1) Already-set columns
    for k in ('profile_url', 'source_url'):
        u = row.get(k)
        if u and 'semafor.hns.family/igraci/' in (u or ''):
            return u
    # 2) hns_igrac_id column
    pid = row.get('hns_igrac_id')
    if pid:
        slug = _slugify(((row.get('ime') or '') + ' ' + (row.get('prezime') or '')).strip())
        return f'{_HNS_BASE}/igraci/{int(pid)}/{slug}/'
    # 3) vanjski_id JSONB → hns_comet
    vid = row.get('vanjski_id') or {}
    if isinstance(vid, dict):
        comet = vid.get('hns_comet') or vid.get('hns_pid')
        slug  = vid.get('hns_slug') or _slugify(((row.get('ime') or '') + ' ' + (row.get('prezime') or '')).strip())
        if comet:
            try:
                return f'{_HNS_BASE}/igraci/{int(comet)}/{slug}/'
            except Exception:
                pass
    # 4) source='hns_semafor' + source_id
    if (row.get('source') or '').startswith('hns_') and row.get('source_id'):
        try:
            slug = _slugify(((row.get('ime') or '') + ' ' + (row.get('prezime') or '')).strip())
            return f'{_HNS_BASE}/igraci/{int(row["source_id"])}/{slug}/'
        except Exception:
            pass
    return None


def _parse_hns_player(html_doc: str, url: str) -> Optional[dict]:
    """Extract structured fields from a semafor.hns.family player page."""
    if not html_doc: return None
    try:
        from bs4 import BeautifulSoup
    except Exception:
        return _parse_hns_player_regex(html_doc, url)
    soup = BeautifulSoup(html_doc, 'html.parser')
    out: dict[str, Any] = {'source': 'semafor.hns.family', 'url': url}

    # hns_igrac_id from URL
    m = re.search(r'/igraci/(\d+)/', url)
    if m: out['hns_igrac_id'] = int(m.group(1))

    title = soup.find('title')
    if title: out['title'] = title.get_text(strip=True)[:300]

    # Photo
    photo = soup.find('div', class_='photo')
    if photo:
        img = photo.find('img')
        if img and img.get('src'):
            src = img['src']
            if not src.startswith('http'):
                src = urllib.parse.urljoin(url, src)
            out['slika_url'] = src

    # Player number (jersey)
    pn = soup.find('div', class_='playerName')
    if pn:
        h3 = pn.find('h3')
        if h3:
            t = h3.get_text(strip=True)
            if t.isdigit():
                out['broj_dresa'] = int(t)

    # Datum rodjenja
    li = soup.find('li', class_='dob')
    if li:
        h4 = li.find('h4')
        if h4:
            t = h4.get_text(' ', strip=True)
            mm = re.match(r'(\d{1,2})\.(\d{1,2})\.(\d{4})', t)
            if mm:
                from datetime import date as _date
                try:
                    out['datum_rodenja'] = _date(int(mm.group(3)), int(mm.group(2)), int(mm.group(1))).isoformat()
                except Exception:
                    pass

    # Mjesto rodjenja
    li = soup.find('li', class_='pob')
    if li:
        h4 = li.find('h4')
        if h4:
            out['mjesto_rodenja'] = h4.get_text(strip=True)

    # Trenutni klub (info only — we don't reassign klub_id from here)
    klub_link = soup.find('a', href=re.compile(r'/klubovi/(\d+)/'))
    if klub_link:
        h4 = klub_link.find('h4')
        if h4:
            out['trenutni_klub'] = h4.get_text(strip=True)
        m = re.search(r'/klubovi/(\d+)/', klub_link.get('href') or '')
        if m: out['hns_klub_id'] = int(m.group(1))

    # Description (meta)
    meta_d = soup.find('meta', attrs={'name': 'description'})
    if meta_d and meta_d.get('content'):
        out['description'] = meta_d['content'][:600]

    # Make a clean text blob for relevance / DeepSeek
    text = soup.get_text(' ', strip=True)
    out['raw_text'] = re.sub(r'\s+', ' ', text)[:4000]
    out['extract'] = (out.get('description')
                      or (out['raw_text'][:500] if out.get('raw_text') else None))
    return out


def _parse_hns_player_regex(html_doc: str, url: str) -> Optional[dict]:
    """BS4-free fallback parser."""
    out: dict[str, Any] = {'source': 'semafor.hns.family', 'url': url}
    m = re.search(r'/igraci/(\d+)/', url)
    if m: out['hns_igrac_id'] = int(m.group(1))
    m = re.search(r'<div class="photo"><img src="([^"]+)"', html_doc)
    if m:
        src = m.group(1)
        if not src.startswith('http'): src = urllib.parse.urljoin(url, src)
        out['slika_url'] = src
    m = re.search(r'<li class="dob">.*?<h4>(\d{1,2})\.(\d{1,2})\.(\d{4})', html_doc, re.S)
    if m:
        from datetime import date as _date
        try:
            out['datum_rodenja'] = _date(int(m.group(3)), int(m.group(2)), int(m.group(1))).isoformat()
        except Exception:
            pass
    m = re.search(r'<li class="pob"><i></i><h4>([^<]+)</h4>', html_doc)
    if m: out['mjesto_rodenja'] = m.group(1).strip()
    m = re.search(r'<div class="playerName"><h3>(\d+)</h3>', html_doc)
    if m: out['broj_dresa'] = int(m.group(1))
    m = re.search(r'<meta\s+name="description"\s+content="([^"]+)"', html_doc)
    if m: out['description'] = m.group(1)[:600]
    return out


def _hns_fetch_player(url: str) -> Optional[dict]:
    body = _http_get(url, timeout=8)
    if not body:
        # Try Playwright fallback
        if _HAS_PW and _pw_scraper is not None:
            r = _pw_scraper.fetch_rendered(url, timeout_ms=15000)
            if r and r.get('html_len', 0) > 2000:
                # We didn't store html in fetch_rendered — re-fetch text only is enough
                # but we need html for parse. Do a simple HTTP retry with longer timeout.
                body = _http_get(url, timeout=15)
    return _parse_hns_player(body, url) if body else None


# ─── Generic sport-federation scraper ───────────────────────────────────
def _fed_url_from_row(row: dict) -> Optional[str]:
    """If the row already points to a federation profile (source_url /
    profile_url on a known fed host), return it."""
    feds, _, _ = _load_sport_feds()
    fed_hosts = set()
    for entry in feds.values():
        if not isinstance(entry, dict): continue
        for which in ('national', 'pgz'):
            sub = entry.get(which) or {}
            for k in ('url', 'search_url', 'profile_url_pattern'):
                v = sub.get(k)
                if v:
                    try:
                        h = urllib.parse.urlparse(v.replace('{q}', 'x').replace('{slug}', 'x').replace('{hns_pid}', '1')).hostname
                        if h: fed_hosts.add(h)
                    except Exception:
                        pass
    for k in ('source_url', 'profile_url'):
        u = row.get(k)
        if not u: continue
        try:
            h = urllib.parse.urlparse(u).hostname or ''
        except Exception:
            continue
        if h in fed_hosts:
            return u
    return None


def _parse_federation_profile(html_doc: str, url: str, ime: str, prezime: str) -> Optional[dict]:
    """Best-effort parser for a generic sport-federation profile page.

    Returns {source, url, slika_url, datum_rodenja, mjesto_rodenja, klub,
             extract, raw_text}. Tolerant of varied page structures.
    """
    if not html_doc: return None
    host = urllib.parse.urlparse(url).hostname or ''
    out: dict[str, Any] = {
        'source': host,
        'url': url,
    }
    # Title
    m = re.search(r'<title[^>]*>([^<]+)</title>', html_doc, re.I)
    if m: out['title'] = html.unescape(m.group(1).strip())[:300]
    # Meta description
    m = re.search(r'<meta\s+name=["\']description["\']\s+content=["\']([^"\']+)["\']', html_doc, re.I)
    if m: out['description'] = html.unescape(m.group(1).strip())[:600]

    name_tokens = []
    for t in (ime, prezime):
        if t and len(t) >= 3:
            name_tokens.append(re.escape(t))

    # Pick the first content image whose filename contains the player's name,
    # or fall back to the first non-asset image.
    img_candidates = re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', html_doc, re.I)
    chosen_img = None
    for src in img_candidates:
        low = src.lower()
        if any(b in low for b in ('logo', 'icon', 'admin-ajax', 'spinner', 'loader',
                                  'sprite', '/themes/', '/icons/', 'gdpr', 'banner',
                                  'header', 'footer', 'placeholder', 'avatar-default')):
            continue
        if not low.endswith(('.jpg', '.jpeg', '.png', '.webp')):
            continue
        # Prefer matches on player name in URL
        if name_tokens and any(re.search(t, src, re.I) for t in name_tokens):
            chosen_img = src; break
        if chosen_img is None:
            chosen_img = src
    if chosen_img:
        if not chosen_img.startswith('http'):
            chosen_img = urllib.parse.urljoin(url, chosen_img)
        out['slika_url'] = chosen_img

    # Plain text body for evidence + label scraping
    text = re.sub(r'<script[^>]*>.*?</script>', ' ', html_doc, flags=re.S | re.I)
    text = re.sub(r'<style[^>]*>.*?</style>', ' ', text, flags=re.S | re.I)
    text = re.sub(r'<[^>]+>', ' ', text)
    text = html.unescape(re.sub(r'\s+', ' ', text)).strip()
    out['raw_text'] = text[:4000]
    out['extract']  = (out.get('description')
                       or text[max(0, text.find(prezime)-30):max(0, text.find(prezime)-30)+500]
                       or text[:500])

    # Common label-driven fields (HBS layout: "Godina rođenja: 1979.", "Matični klub: …")
    m = re.search(r'Datum\s+ro[đdj]?enja[:\s]+(\d{1,2}[.\-/]\d{1,2}[.\-/]\d{4})', text, re.I)
    if m:
        try:
            from datetime import date as _date
            d = re.split(r'[.\-/]', m.group(1))
            out['datum_rodenja'] = _date(int(d[2]), int(d[1]), int(d[0])).isoformat()
        except Exception:
            pass
    if 'datum_rodenja' not in out:
        m = re.search(r'Godina\s+ro[đdj]?enja[:\s]+(\d{4})', text, re.I)
        if m:
            try:
                from datetime import date as _date
                out['datum_rodenja'] = _date(int(m.group(1)), 1, 1).isoformat()
            except Exception:
                pass
    m = re.search(r'Mjesto\s+ro[đdj]?enja[:\s]+([A-ZČĆŠĐŽ][^,\n.]{2,40})', text)
    if m: out['mjesto_rodenja'] = m.group(1).strip()
    m = re.search(r'Mati[čc]ni\s+klub[:\s]+([^\n]{3,60}?)(?:\s+(?:Sportski|Datum|Liječni|Reprezent|Sezona|Domaće|Nastupi))', text, re.I)
    if m: out['klub_naziv'] = m.group(1).strip().rstrip('.')

    return out


def _slugify_simple(s: str) -> str:
    import unicodedata
    s = unicodedata.normalize('NFKD', s or '').encode('ascii', 'ignore').decode('ascii').lower()
    return re.sub(r'[^a-z0-9]+', '-', s).strip('-')


def scrape_sport_federation(sport: Optional[str], ime: str, prezime: str) -> Optional[dict]:
    """Try to find and parse the athlete's federation profile page."""
    fed = _sport_fed(sport) if sport else None
    if not fed: return None
    nat = (fed or {}).get('national') or {}
    full_name = (ime + ' ' + prezime).strip()

    # 1) Direct profile URL via {slug} pattern (works for HBS at least)
    pattern = nat.get('profile_url_pattern')
    if pattern and '{slug}' in pattern:
        slug = _slugify_simple(full_name)
        url = pattern.replace('{slug}', slug)
        body = _http_get(url, timeout=8)
        if body and prezime.lower() in body.lower():
            return _parse_federation_profile(body, url, ime, prezime)

    # 2) Search URL → first /igraci|/profil|/clan link that mentions the surname
    search = nat.get('search_url')
    if search:
        body = _http_get(search.replace('{q}', urllib.parse.quote(full_name)), timeout=10)
        if body:
            for href_re in (r'href="([^"]*?/igraci/[^"]+)"',
                            r'href="([^"]*?/igrac/[^"]+)"',
                            r'href="([^"]*?/sportasi/[^"]+)"',
                            r'href="([^"]*?/clanovi/[^"]+)"',
                            r'href="([^"]*?/profil/[^"]+)"'):
                for m in re.finditer(href_re, body, re.I):
                    cand = m.group(1)
                    if not cand.startswith('http'):
                        cand = urllib.parse.urljoin(nat.get('url', search), cand)
                    if _slugify_simple(prezime) in _slugify_simple(cand):
                        b2 = _http_get(cand, timeout=8)
                        if b2:
                            return _parse_federation_profile(b2, cand, ime, prezime)
    return None


def _propose_for_sportas(row: dict) -> dict:
    naziv = ((row.get('ime') or '') + ' ' + (row.get('prezime') or '')).strip()
    ime, prezime = (row.get('ime') or ''), (row.get('prezime') or '')
    sport = row.get('sport')
    sources, evidence = [], []
    proposed: dict[str, Any] = {}

    # 1) HNS Semafor — only meaningful when sport is football OR row already
    # carries an HNS link.
    hns_doc: Optional[dict] = None
    if _normalize_sport(sport) == 'nogomet' or _hns_url_from_row(row):
        hns_url = _hns_url_from_row(row)
        if hns_url:
            hns_doc = _hns_fetch_player(hns_url)
            if hns_doc:
                sources.append(hns_doc)
                evidence.append(hns_doc.get('raw_text') or hns_doc.get('extract') or '')

    # 2) Sport-aware federation scrape (HBS, HKS, etc.) — also use existing
    # source_url/profile_url if it points at a known federation host.
    fed_doc: Optional[dict] = None
    direct_fed_url = _fed_url_from_row(row)
    if direct_fed_url and (not hns_doc or hns_doc.get('url') != direct_fed_url):
        body = _http_get(direct_fed_url, timeout=8)
        if body:
            fed_doc = _parse_federation_profile(body, direct_fed_url, ime, prezime)
    if not fed_doc:
        fed_doc = scrape_sport_federation(sport, ime, prezime)
    if fed_doc:
        sources.append(fed_doc)
        evidence.append(fed_doc.get('raw_text') or fed_doc.get('extract') or '')

    # Helper: pick from hns_doc first then fed_doc
    def _pick(field):
        if hns_doc and hns_doc.get(field): return hns_doc[field]
        if fed_doc and fed_doc.get(field): return fed_doc[field]
        return None

    if not row.get('profile_url'):
        v = _pick('url') or (hns_doc and hns_doc.get('url')) or (fed_doc and fed_doc.get('url'))
        if v: proposed['profile_url'] = v
    if not row.get('source_url'):
        v = (hns_doc and hns_doc.get('url')) or (fed_doc and fed_doc.get('url'))
        if v: proposed['source_url'] = v
    if not row.get('slika_url'):
        v = _pick('slika_url')
        if v: proposed['slika_url'] = v
    if not row.get('hns_igrac_id') and hns_doc and hns_doc.get('hns_igrac_id'):
        proposed['hns_igrac_id'] = hns_doc['hns_igrac_id']
    if not row.get('datum_rodenja'):
        v = _pick('datum_rodenja')
        if v: proposed['datum_rodenja'] = v
    if not row.get('mjesto_rodenja'):
        v = _pick('mjesto_rodenja')
        if v: proposed['mjesto_rodenja'] = v
    if not row.get('broj_dresa') and hns_doc and hns_doc.get('broj_dresa'):
        proposed['broj_dresa'] = hns_doc['broj_dresa']

    # 3) Wikipedia HR for biografija
    if not row.get('biografija'):
        wiki = _wiki_summary(naziv)
        if wiki:
            sources.append(wiki)
            evidence.append(wiki.get('extract') or '')

    # 4) kosarkapgz.com — news mentions for PGŽ basketball players
    if _kosarkapgz_is_basketball_pgz('', sport):
        q = (ime + ' ' + prezime).strip()
        kpgz = _kosarkapgz_search_posts(q, limit=5) if q else None
        if kpgz:
            sources.append(kpgz)
            evidence.append(kpgz.get('raw_text') or '')

    # Description: prefer DeepSeek synthesis from all evidence; fallback to first long snippet
    if not row.get('biografija'):
        descr = _deepseek_describe(naziv, f'sportaš ({sport})' if sport else 'sportaš', evidence) if evidence else None
        if not descr:
            for s in sources:
                ext = s.get('extract')
                if ext and len(ext) >= 80:
                    descr = ext; break
        if descr:
            proposed['biografija'] = descr.strip()[:2000]

    return {'proposed': proposed, 'sources': sources}


# ─── Endpoints ──────────────────────────────────────────────────────────
# ─── R4 — POST /v2/enrich/forensic/{finding_id} ─────────────────────────
def _extract_pep_name(finding: dict) -> Optional[str]:
    """Pull the primary person name from a forensic_findings row."""
    title = (finding.get('title') or '').strip()
    desc  = (finding.get('description') or '').strip()
    payload = finding.get('raw_data') or {}
    if isinstance(payload, str):
        try: payload = json.loads(payload)
        except Exception: payload = {}
    if isinstance(payload, dict):
        for k in ('person_name', 'name', 'osoba'):
            v = payload.get(k)
            if v: return str(v).strip()
    # Try entities_involved.entity_name
    ents = finding.get('entities_involved') or []
    if isinstance(ents, str):
        try: ents = json.loads(ents)
        except Exception: ents = []
    if isinstance(ents, list):
        for e in ents:
            if isinstance(e, dict) and e.get('person_name'):
                return str(e['person_name']).strip()
            if isinstance(e, dict) and e.get('entity_name') and ' ' in (e.get('entity_name') or ''):
                # Some entries store person names as entity_name when entity_type='person'
                if (e.get('entity_type') or '').lower() in ('person','osoba'):
                    return str(e['entity_name']).strip()
    # Fallback: extract a "Ime Prezime" from the title
    m = re.search(r'\b([A-ZČĆŠĐŽ][a-zčćšđž]+)\s+([A-ZČĆŠĐŽ][a-zčćšđž]+(?:-[A-ZČĆŠĐŽ][a-zčćšđž]+)?)\b', title + ' ' + desc)
    if m: return f"{m.group(1)} {m.group(2)}"
    return None


def _gather_pep_evidence(name: str) -> list[dict]:
    sources: list[dict] = []
    wiki = _wiki_summary(name)
    if wiki: sources.append(wiki)
    # DDG html-lite as a "Google snippet" replacement (often OK for HR PEPs)
    ddg = 'https://html.duckduckgo.com/html/?q=' + urllib.parse.quote(f'"{name}" PGŽ Hrvatska')
    page = _http_get(ddg, timeout=8)
    if page:
        # First result block
        m = re.search(r'<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([^<]{6,200})</a>', page)
        snippet_m = re.search(r'<a[^>]+class="result__snippet"[^>]*>(.*?)</a>', page, re.S)
        if m:
            sources.append({
                'source': 'duckduckgo',
                'url': html.unescape(m.group(1))[:500],
                'title': html.unescape(m.group(2)).strip()[:300],
                'extract': re.sub(r'<[^>]+>', ' ', snippet_m.group(1)).strip()[:600] if snippet_m else None,
            })
    return sources


def _related_entities_for_pep(name: str) -> list[dict]:
    """Pull civic.persons + their entity links so we have the structured graph."""
    out: list[dict] = []
    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        cur.execute("""SELECT id, name, function, party, county, city, oib, trust_tier
                       FROM civic.persons
                       WHERE upper(name) ILIKE upper(%s)
                       ORDER BY oib NULLS LAST, id LIMIT 10""", ('%'+name+'%',))
        for p in cur.fetchall():
            p = dict(p)
            entry = {
                'kind': 'person',
                'person_id': p['id'], 'person_name': p['name'],
                'function': p.get('function'), 'party': p.get('party'),
                'county':   p.get('county'),   'city': p.get('city'),
                'oib':      p.get('oib'),      'trust_tier': p.get('trust_tier'),
                'entities': [],
            }
            if p.get('oib'):
                cur.execute("""SELECT pel.entity_id, pel.roles, e.name AS entity_name,
                                      e.oib AS entity_oib, e.entity_type, e.city, e.risk_score
                               FROM civic.person_entity_links pel
                               LEFT JOIN civic.entities e ON e.id = pel.entity_id
                               WHERE pel.person_oib=%s LIMIT 30""", (p['oib'],))
                for r in cur.fetchall():
                    entry['entities'].append(dict(r))
            out.append(entry)
    return out


@router.post("/enrich/forensic/{finding_id}")
def enrich_forensic_v2(finding_id: int,
                    body: dict = Body(default=None),
                    x_user_email: Optional[str] = Header(default=None),
                    x_user_id:    Optional[int] = Header(default=None)):
    """Enrich a forensic finding: gather Wiki + DDG snippets + civic graph,
    write back to civic.forensic_findings.related_entities, and seal the
    payload hash on Polygon (or queue for sealing).
    """
    body = body or {}
    explicit_name = (body.get('name') or '').strip() or None

    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        cur.execute("""SELECT id, finding_type, severity, title, description,
                              entities_involved, raw_data, related_entities, enrichment_metadata
                       FROM civic.forensic_findings WHERE id=%s""", (finding_id,))
        finding = cur.fetchone()
    if not finding:
        raise HTTPException(404, "finding not found")
    finding = dict(finding)

    name = explicit_name or _extract_pep_name(finding)
    if not name:
        raise HTTPException(400, "could not derive a person/entity name; pass {name: \"…\"}")

    sources  = _gather_pep_evidence(name)
    related  = _related_entities_for_pep(name)

    payload = {
        'finding_id': finding_id,
        'name': name,
        'sources': [{'source': s.get('source'), 'url': s.get('url'),
                     'title': s.get('title')} for s in sources],
        'related_entities': related,
        'enriched_at': datetime.now(timezone.utc).isoformat(),
    }

    # Persist back to the finding
    enrichment_meta = finding.get('enrichment_metadata') or {}
    if not isinstance(enrichment_meta, dict): enrichment_meta = {}
    history = enrichment_meta.get('history') or []
    history.append({
        'at': payload['enriched_at'],
        'sources': payload['sources'],
        'related_count': len(related),
        'user': x_user_email,
    })
    enrichment_meta['history']      = history[-10:]
    enrichment_meta['enriched_at']  = payload['enriched_at']
    enrichment_meta['enriched_by']  = x_user_email or 'system'
    enrichment_meta['source_count'] = len(sources)

    with _db() as c, c.cursor() as cur:
        cur.execute("""UPDATE civic.forensic_findings
                          SET related_entities    = %s::jsonb,
                              enrichment_metadata = %s::jsonb
                        WHERE id=%s
                        RETURNING id""",
                    (json.dumps(related, default=str, ensure_ascii=False),
                     json.dumps(enrichment_meta, default=str, ensure_ascii=False),
                     finding_id))
        cur.fetchone()

    # Seal the enrichment payload hash on Polygon (or queue if no key)
    seal_result: dict[str, Any] = {}
    try:
        sys_path_added = False
        try:
            from blockchain import seal as _seal_mod  # noqa: E402
        except Exception:
            import sys as _ssys
            _ssys.path.insert(0, '/opt/pgz-sport')
            from blockchain import seal as _seal_mod  # noqa: E402
            sys_path_added = True
        del sys_path_added  # silence linters
        h = _seal_mod.hash_payload(payload)
        seal_result = _seal_mod.seal_to_polygon(
            data_hash=h,
            ref_id=str(finding_id),
            action='forensic.enriched',
            ref_type='forensic_finding',
            payload=payload,
            user_id=x_user_id,
            user_email=x_user_email,
        )
    except Exception as e:
        seal_result = {'error': f'{type(e).__name__}: {e}'}

    return {
        'finding_id': finding_id,
        'name': name,
        'sources': sources,
        'related_entities': related,
        'related_count': len(related),
        'enrichment_metadata': enrichment_meta,
        'seal': seal_result,
    }


from fastapi import Path as _FPath

@router.post("/enrich/{kind:str}/{eid:int}")
def enrich_preview(kind: str = _FPath(..., regex='^(klub|savez|sportas)$'),
                   eid: int = 0,
                   nocache: int = 0):
    """Enrichment preview with Redis cache (24h TTL) + slow-call telemetry.

    Pass ?nocache=1 to bypass cache.
    """
    _t0 = time.time()
    _cache_key = f"enrich:v1:{kind}:{eid}"
    if not nocache:
        _cached = _cache_get(_cache_key)
        if _cached is not None:
            _cached['_cache'] = 'hit'
            _cached['_cache_ttl_s'] = ENRICH_CACHE_TTL
            try: _enrich_slow_log(kind, eid, time.time() - _t0, cached=True)
            except Exception: pass
            return _cached

    row = _load_row(kind, eid)
    if   kind == 'klub':    res = _propose_for_klub(row)
    elif kind == 'savez':   res = _propose_for_savez(row)
    else:                   res = _propose_for_sportas(row)

    if kind == 'klub':
        keys = ['oib','sport','grad','predsjednik','tajnik','web','email','telefon',
                'sjediste','godina_osnutka','ciljevi','opis_djelatnosti']
    elif kind == 'savez':
        keys = ['oib','sport','predsjednik','tajnik','email','telefon','web','adresa','godina_osnutka']
    else:
        keys = ['sport','profile_url','slika_url','hns_igrac_id','biografija',
                'datum_rodenja','mjesto_rodenja','broj_dresa','visina_cm','tezina_kg',
                'dominantna_noga','oib']

    naziv = _display_name(kind, row)
    grad = row.get('grad') if kind == 'klub' else None
    primary = row.get('web') or row.get('web_stranica') or row.get('source_url') or row.get('scrape_url') or row.get('profile_url')

    filled  = sum(1 for k in keys if row.get(k))
    coverage = round(filled / len(keys) * 100)
    missing  = [k for k in keys if not row.get(k)]

    proposed = res['proposed']
    current  = {k: row.get(k) for k in proposed.keys()}
    meta = row.get('metadata') or {}
    if not isinstance(meta, dict): meta = {}

    _result = {
        'kind': kind, 'id': eid, 'naziv': naziv,
        'coverage': coverage, 'filled_fields': filled, 'total_fields': len(keys),
        'missing_fields': missing,
        'live_snippet': _fetch_title(primary) if primary else None,
        'research_links': _research_links(naziv, kind, grad, sport=row.get('sport'), row=row),
        'sport': row.get('sport'),
        'sport_federation': (lambda f: {
            'national': (f.get('national') or {}).get('name') if f else None,
            'national_url': (f.get('national') or {}).get('url') if f else None,
            'pgz': (f.get('pgz') or {}).get('name') if f else None,
        })(_sport_fed(row.get('sport'))),
        'sources': res['sources'],
        'current':  current,
        'proposed': proposed,
        'last_enriched_at': meta.get('enriched_at'),
        'last_enrichment_source': meta.get('enrichment_source'),
        'enriched_at': int(time.time()),
        'apply_url': f'/sport/api/v2/enrich/{kind}/{eid}/apply',
        '_cache': 'miss',
        '_cache_ttl_s': ENRICH_CACHE_TTL,
    }
    _duration = time.time() - _t0
    try:
        _cache_set(_cache_key, _result, ttl=ENRICH_CACHE_TTL)
    except Exception:
        pass
    try:
        if _duration >= ENRICH_SLOW_LOG_THRESHOLD:
            _enrich_slow_log(kind, eid, _duration, cached=False)
    except Exception:
        pass
    return _result


_TABLE_MAP = {
    'klub':    ('pgz_sport.klubovi',
                {'web','email','telefon','predsjednik','tajnik',
                 'opis_djelatnosti','ciljevi','godina_osnutka','sjediste','adresa'}),
    'savez':   ('pgz_sport.savezi',
                {'web','email','telefon','predsjednik','tajnik','adresa','godina_osnutka'}),
    'sportas': ('pgz_sport.clanovi',
                {'biografija','profile_url','source_url','slika_url','hns_igrac_id',
                 'datum_rodenja','mjesto_rodenja','broj_dresa','visina_cm',
                 'tezina_kg','dominantna_noga','oib'}),
}


def _apply_to_db(kind: str, eid: int, fields: dict, sources: list, user_email: Optional[str]):
    if kind not in _TABLE_MAP:
        raise HTTPException(400, "kind must be klub|savez|sportas")
    table, allowed = _TABLE_MAP[kind]

    # Use a manual transaction so SELECT ... FOR UPDATE actually holds the row
    # lock until we COMMIT. _db() returns autocommit=True connections, so we
    # flip it off locally for this function only.
    c = _db()
    c.autocommit = False
    error_msg = None  # populated on UniqueViolation, recorded in enrichment_log
    try:
        with c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
            cur.execute(f"SELECT * FROM {table} WHERE id=%s FOR UPDATE", (eid,))
            before = cur.fetchone()
            if not before: raise HTTPException(404, kind + " not found")
            before = dict(before)

            sets, params, applied = [], [], {}
            for k, v in (fields or {}).items():
                if k not in allowed: continue
                if v is None or str(v).strip() == '': continue
                if before.get(k):
                    continue  # never overwrite existing
                sets.append(f"{k} = %s")
                params.append(v); applied[k] = v

            meta_in = before.get('metadata') or {}
            if not isinstance(meta_in, dict): meta_in = {}
            now_iso = datetime.now(timezone.utc).isoformat()
            meta_in['enriched_at'] = now_iso
            meta_in['enrichment_source'] = [s.get('source') for s in (sources or []) if s.get('source')]
            history = meta_in.get('enrichment_history') or []
            history.append({
                'at': now_iso,
                'fields': list(applied.keys()),
                'sources': meta_in['enrichment_source'],
                'urls':    [s.get('url') for s in (sources or []) if s.get('url')],
                'user':    user_email,
            })
            meta_in['enrichment_history'] = history[-10:]
            sets.append("metadata = %s::jsonb")
            params.append(json.dumps(meta_in, ensure_ascii=False, default=str))

            params.append(eid)
            try:
                cur.execute(f"UPDATE {table} SET {', '.join(sets)} WHERE id=%s RETURNING *", params)
                after = dict(cur.fetchone())
            except psycopg2.errors.UniqueViolation as _uve:
                # The UPDATE rolled back — pretend nothing was applied so the
                # log row, the API response, and the row state all agree.
                c.rollback()
                failed_fields = list(applied.keys())
                applied = {}                       # truth-in-reporting
                constraint = getattr(getattr(_uve, 'diag', None), 'constraint_name', None)
                error_msg = f"unique_violation: {constraint or 'unknown'}"
                import logging as _lg
                _lg.getLogger("enrich").warning(
                    f"UniqueViolation table={table} id={eid} blocked_fields={failed_fields} constraint={constraint}")
                # Fetch current row state for the audit log.
                cur.execute(f"SELECT * FROM {table} WHERE id=%s", (eid,))
                row = cur.fetchone()
                after = dict(row) if row else {}
                # Park a do-not-retry marker on the row so workers stop hammering it.
                try:
                    block_meta = {
                        'reason':     'unique_violation',
                        'constraint': constraint,
                        'fields':     failed_fields,
                        'at':         now_iso,
                    }
                    cur.execute(
                        f"UPDATE {table} "
                        f"SET metadata = COALESCE(metadata,'{{}}'::jsonb) || %s::jsonb "
                        f"WHERE id=%s",
                        (json.dumps({'enrichment_block': block_meta}, default=str), eid))
                except Exception as _be:
                    # Marker is best-effort; never let it block the apply path.
                    import logging as _lg2
                    _lg2.getLogger("enrich").warning(f"enrichment_block write failed table={table} id={eid}: {_be}")

            # Always log — on success error_msg is NULL, on UniqueViolation it
            # carries the constraint name and applied is empty.
            cur.execute(
                """INSERT INTO pgz_sport.enrichment_log
                     (kind, target_id, source, url, fields_set, before_jsonb, after_jsonb, user_email, error)
                   VALUES (%s,%s,%s,%s,%s,%s::jsonb,%s::jsonb,%s,%s)""",
                (kind, eid,
                 ','.join(meta_in['enrichment_source'])[:120] if meta_in['enrichment_source'] else None,
                 (sources[0].get('url') if sources else None),
                 list(applied.keys()) or None,
                 json.dumps({k: before.get(k) for k in (list(applied.keys()) + ['metadata'])},
                            ensure_ascii=False, default=str),
                 json.dumps({k: after.get(k)  for k in (list(applied.keys()) + ['metadata'])},
                            ensure_ascii=False, default=str),
                 user_email,
                 error_msg))
        c.commit()
    except HTTPException:
        try: c.rollback()
        except Exception: pass
        raise
    except Exception:
        try: c.rollback()
        except Exception: pass
        raise
    finally:
        try: c.close()
        except Exception: pass

    snap_keys = ('id','naziv','ime','prezime','web','email','telefon',
                 'opis_djelatnosti','biografija','metadata')
    return {'applied': applied,
            'after':  {k: after.get(k) for k in snap_keys if k in after}}


@router.post("/enrich/{kind:str}/{eid:int}/apply")
def enrich_apply(kind: str = _FPath(..., regex='^(klub|savez|sportas)$'),
                 eid: int = 0,
                 body: dict = Body(default=None),
                 x_user_email: Optional[str] = Header(default=None),
                 x_user_id:    Optional[int] = Header(default=None)):
    body = body or {}
    fields  = body.get('fields')
    sources = body.get('sources')
    if not fields:
        row = _load_row(kind, eid)
        if   kind == 'klub':    res = _propose_for_klub(row)
        elif kind == 'savez':   res = _propose_for_savez(row)
        else:                   res = _propose_for_sportas(row)
        fields  = res['proposed']
        sources = res['sources']
    out = _apply_to_db(kind, eid, fields or {}, sources or [], x_user_email)
    applied = out.get('applied') or {}
    # R4-A3: write to pgz_sport.sys_audit so the audit page sees enrichment events
    try:
        from audit_seal_router import audit_log as _audit_log
        if applied:
            _audit_log(
                action='enrich.apply',
                target_type=kind,
                target_id=eid,
                payload={'applied': applied,
                         'sources': [s.get('url') for s in (sources or []) if isinstance(s, dict)]},
                user_id=x_user_id,
                user_email=x_user_email,
            )
    except Exception:
        pass
    # Invalidate cache so next preview reflects the new DB state
    try:
        _cache_delete(f"enrich:v1:{kind}:{eid}")
    except Exception:
        pass
    return {
        'status':         'success' if applied else 'no_changes',
        'kind':           kind,
        'id':             eid,
        'applied_count':  len(applied),
        'applied_fields': list(applied.keys()),
        **out,
    }


@router.get("/enrich/log")
def enrich_log(kind: Optional[str] = None, target_id: Optional[int] = None, limit: int = 50):
    where, params = [], []
    if kind:      where.append("kind=%s");      params.append(kind)
    if target_id: where.append("target_id=%s"); params.append(target_id)
    sql = ("SELECT id, kind, target_id, source, url, fields_set, user_email, created_at "
           "FROM pgz_sport.enrichment_log "
           + ("WHERE " + " AND ".join(where) + " " if where else "")
           + "ORDER BY id DESC LIMIT %s")
    params.append(min(int(limit or 50), 200))
    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        cur.execute(sql, params)
        rows = [dict(r) for r in cur.fetchall()]
    for r in rows:
        if r.get('created_at'): r['created_at'] = r['created_at'].isoformat()
    return {'count': len(rows), 'rows': rows}


# ─── R3B M2 — SEARCH SUGGEST (autocomplete for Mreža) ───────────────────
@router.get("/search/suggest")
def search_suggest(q: str = '', type: str = '', limit: int = 10):
    """
    Autocomplete suggestions for the Mreža search inputs.
    type ∈ {person, club, company, ''} — empty means all.
    Returns: {query, results: [{id, label, type, sub}]}
    """
    q = (q or '').strip()
    if len(q) < 2:
        return {'query': q, 'results': []}
    limit = max(1, min(50, int(limit)))
    out = []
    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        if type in ('', 'club'):
            cur.execute("""
              SELECT id, naziv AS label, sport, grad
              FROM pgz_sport.klubovi
              WHERE naziv ILIKE %s AND aktivan=TRUE
              ORDER BY length(naziv), naziv LIMIT %s
            """, ('%'+q+'%', limit))
            for r in cur.fetchall():
                out.append({'id':'klub:'+str(r['id']), 'label': r['label'], 'type':'club',
                            'sub': (r.get('sport') or '')+' · '+(r.get('grad') or '')})
            cur.execute("""
              SELECT id, naziv AS label, sport
              FROM pgz_sport.savezi
              WHERE naziv ILIKE %s AND aktivan=TRUE
              ORDER BY length(naziv), naziv LIMIT %s
            """, ('%'+q+'%', limit))
            for r in cur.fetchall():
                out.append({'id':'savez:'+str(r['id']), 'label': r['label'], 'type':'savez',
                            'sub': r.get('sport') or 'savez'})
        if type in ('', 'person'):
            cur.execute("""
              SELECT c.id, c.ime, c.prezime, c.sport, k.naziv AS klub_naziv
              FROM pgz_sport.clanovi c
              LEFT JOIN pgz_sport.klubovi k ON k.id = c.klub_id
              WHERE (COALESCE(c.ime,'') || ' ' || COALESCE(c.prezime,'')) ILIKE %s
              ORDER BY length(COALESCE(c.ime,'')||COALESCE(c.prezime,'')), c.prezime
              LIMIT %s
            """, ('%'+q+'%', limit))
            for r in cur.fetchall():
                out.append({'id':'sportas:'+str(r['id']),
                            'label': (r.get('ime') or '')+' '+(r.get('prezime') or ''),
                            'type':'person',
                            'sub': (r.get('sport') or 'sportaš')+(r.get('klub_naziv') and ' · '+r['klub_naziv'] or '')})
            cur.execute("""
              SELECT id, name AS label, function, oib, county
              FROM civic.persons
              WHERE name ILIKE %s
              ORDER BY oib NULLS LAST, length(name) LIMIT %s
            """, ('%'+q+'%', limit))
            for r in cur.fetchall():
                out.append({'id':'civic_person:'+str(r['id']),
                            'label': r['label'], 'type':'person',
                            'sub': (r.get('function') or 'civic')+' · '+(r.get('county') or '')})
        if type in ('', 'company'):
            cur.execute("""
              SELECT id, name AS label, oib, city, entity_type
              FROM civic.entities
              WHERE name ILIKE %s
              ORDER BY length(name) LIMIT %s
            """, ('%'+q+'%', limit))
            for r in cur.fetchall():
                out.append({'id':'civic_entity:'+str(r['id']),
                            'label': r['label'], 'type':'company',
                            'sub': (r.get('entity_type') or 'tvrtka')+' · '+(r.get('city') or '')})
    return {'query': q, 'results': out[:limit*2]}


# ─── R3B M3 — FORENSIC ENRICH (Wikipedia scrape + persist) ──────────────
@router.post("/forensic/findings/{finding_id}/enrich")
def enrich_forensic(finding_id: int):
    """
    Look up the forensic finding, derive the PEP person name from
    entities_involved or title, hit Wikipedia HR for a summary, and persist
    the enriched payload into civic.forensic_findings.ai_analysis (or back into
    raw_data.enrichment).
    """
    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        cur.execute("""
          SELECT id, finding_type, severity, title, description, entities_involved,
                 raw_data, ai_analysis
          FROM civic.forensic_findings WHERE id=%s
        """, (finding_id,))
        f = cur.fetchone()
        if not f: raise HTTPException(404, "finding not found")
        f = dict(f)

        # Derive person name candidates
        candidates = []
        if isinstance(f.get('entities_involved'), (list, dict)):
            ei = f['entities_involved']
            if isinstance(ei, dict):
                for k in ('person','name','osoba','PEP','pep'):
                    if ei.get(k): candidates.append(str(ei[k]))
                # Also try persons: [...] list
                for p in (ei.get('persons') or ei.get('osobe') or []):
                    if isinstance(p, dict) and p.get('name'): candidates.append(p['name'])
                    elif isinstance(p, str): candidates.append(p)
            elif isinstance(ei, list):
                for it in ei:
                    if isinstance(it, dict):
                        for k in ('name','person','label'):
                            if it.get(k): candidates.append(str(it[k])); break
                    elif isinstance(it, str):
                        candidates.append(it)
        if not candidates and f.get('title'):
            # Heuristic: extract first capitalised "Ime Prezime" pair
            m = re.search(r'\b([A-ZŠĐČĆŽ][a-zšđčćž]{2,})\s+([A-ZŠĐČĆŽ][a-zšđčćž]{2,})', f['title'])
            if m: candidates.append(m.group(0))

        wiki = None
        used_query = None
        for q in candidates[:3]:
            wiki = _wiki_summary(q)
            if wiki:
                used_query = q
                break

        # Build enrichment payload
        enrichment = {
            'queried': candidates[:5],
            'used_query': used_query,
            'wiki': wiki,
            'enriched_at': datetime.now(timezone.utc).isoformat(),
        }

        # Persist into raw_data.enrichment
        raw = f.get('raw_data')
        if raw is None: raw = {}
        if not isinstance(raw, dict): raw = {'_legacy': raw}
        raw['enrichment'] = enrichment

        cur.execute("""
          UPDATE civic.forensic_findings
          SET raw_data = %s::jsonb,
              ai_analysis = COALESCE(ai_analysis, %s)
          WHERE id = %s
        """, (json.dumps(raw, default=str, ensure_ascii=False),
              (wiki or {}).get('extract'),
              finding_id))
        c.commit()

    return {
        'finding_id': finding_id,
        'queried': candidates[:5],
        'used_query': used_query,
        'wiki': wiki,
        'persisted': True,
    }


# ─── R3B P4 — FORENSIC SCAN (kept from prior version) ───────────────────
@router.post("/forensic/scan")
def forensic_scan(req: dict = Body(...)):
    name = (req.get('name') or '').strip()
    if len(name) < 3:
        raise HTTPException(400, "name must be at least 3 chars")
    with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
        cur.execute("""
          SELECT id, name, function, party, county, city, oib, trust_tier
          FROM civic.persons
          WHERE upper(name) ILIKE upper(%s)
          ORDER BY oib NULLS LAST, id LIMIT 25
        """, ('%' + name + '%',))
        persons = [dict(r) for r in cur.fetchall()]
        for p in persons:
            p['links'] = []; p['findings'] = []
            if p.get('oib'):
                cur.execute("""
                  SELECT pel.entity_id, pel.roles, e.name AS entity_name, e.oib AS entity_oib,
                         e.entity_type, e.city, e.risk_score
                  FROM civic.person_entity_links pel
                  LEFT JOIN civic.entities e ON e.id = pel.entity_id
                  WHERE pel.person_oib = %s LIMIT 50
                """, (p['oib'],))
                p['links'] = [dict(r) for r in cur.fetchall()]
                cur.execute("""
                  SELECT id, finding_type, severity, title, severity_score, created_at
                  FROM civic.forensic_findings
                  WHERE entities_involved::text ILIKE %s
                  ORDER BY severity_score DESC, created_at DESC LIMIT 30
                """, ('%' + p['oib'] + '%',))
                p['findings'] = [dict(r) for r in cur.fetchall()]
            if not p['findings']:
                cur.execute("""
                  SELECT id, finding_type, severity, title, severity_score, created_at
                  FROM civic.forensic_findings
                  WHERE title ILIKE %s OR description ILIKE %s
                  ORDER BY severity_score DESC, created_at DESC LIMIT 30
                """, ('%' + p['name'] + '%', '%' + p['name'] + '%'))
                p['findings'] = [dict(r) for r in cur.fetchall()]
    total_links = total_findings = crit_findings = 0
    for p in persons:
        total_links += len(p.get('links') or [])
        for f in p.get('findings') or []:
            total_findings += 1
            if f.get('severity') in ('CRITICAL', 'HIGH'): crit_findings += 1
        score = 0
        if (p.get('function') or '').strip(): score += 30
        if (p.get('party') or '').strip():    score += 15
        score += min(40, len(p.get('links') or []) * 5)
        score += min(40, len(p.get('findings') or []) * 10)
        score += sum(20 for f in (p.get('findings') or []) if f.get('severity') in ('CRITICAL', 'HIGH'))
        p['risk_score'] = min(100, score)
    overall = max((p.get('risk_score', 0) for p in persons), default=0)
    return {'query': name, 'matched_persons': len(persons),
            'overall_risk_score': overall, 'total_links': total_links,
            'total_findings': total_findings, 'critical_findings': crit_findings,
            'persons': persons, 'scanned_at': int(time.time())}



# ─── SB-3 — Bulk enrichment ─────────────────────────────────────────────
_BULK_KEY_MAP = {
    'klub':    ('pgz_sport.klubovi',
                ('oib','sport','grad','predsjednik','tajnik','web','email','telefon',
                 'sjediste','godina_osnutka','ciljevi','opis_djelatnosti')),
    'savez':   ('pgz_sport.savezi',
                ('oib','sport','predsjednik','tajnik','email','telefon','web',
                 'adresa','godina_osnutka')),
    'sportas': ('pgz_sport.clanovi',
                ('sport','profile_url','slika_url','hns_igrac_id','biografija',
                 'datum_rodenja','mjesto_rodenja','broj_dresa')),
}


def _coverage_sql(prefix: str, keys: tuple[str, ...]) -> str:
    parts = [f"(CASE WHEN {prefix}{k} IS NOT NULL AND ({prefix}{k}::text) <> '' THEN 1 ELSE 0 END)"
             for k in keys]
    return f"((({' + '.join(parts)})::numeric * 100) / {len(keys)})"


def _bulk_pick(kind: str, limit: int, coverage_max: int) -> list[int]:
    if kind not in _BULK_KEY_MAP:
        raise HTTPException(400, "kind must be klub|savez|sportas")
    table, keys = _BULK_KEY_MAP[kind]
    cov = _coverage_sql('', keys)
    extra_where = ''
    if kind == 'klub':
        extra_where = "AND aktivan = TRUE"
    elif kind == 'sportas':
        extra_where = "AND aktivan = TRUE"
    sql = (f"SELECT id FROM {table} "
           f"WHERE 1=1 {extra_where} "
           f"AND {cov} < %s "
           f"ORDER BY random() LIMIT %s")
    with _db() as c, c.cursor() as cur:
        cur.execute(sql, (coverage_max, limit))
        return [r[0] for r in cur.fetchall()]


@router.post("/enrich/bulk")
def enrich_bulk(body: dict = Body(default=None),
                x_user_email: Optional[str] = Header(default=None),
                x_user_id:    Optional[int] = Header(default=None)):
    """Run preview+apply over N random under-enriched rows of one kind.

    Body: {kind: 'klub'|'savez'|'sportas', limit: 50, coverage_max: 70}
    Returns aggregate stats. Synchronous (use polling, not SSE).
    """
    body = body or {}
    kind = (body.get('kind') or '').strip().lower()
    if kind not in _BULK_KEY_MAP:
        raise HTTPException(400, "kind must be klub|savez|sportas")
    limit         = max(1, min(int(body.get('limit') or 50), 200))
    coverage_max  = max(0, min(int(body.get('coverage_max') or 70), 100))

    ids = _bulk_pick(kind, limit, coverage_max)
    items: list[dict] = []
    fields_total = 0
    started = time.time()

    for eid in ids:
        try:
            row = _load_row(kind, eid)
            if   kind == 'klub':    res = _propose_for_klub(row)
            elif kind == 'savez':   res = _propose_for_savez(row)
            else:                   res = _propose_for_sportas(row)
            proposed = res.get('proposed') or {}
            srcs     = res.get('sources')  or []
            if not proposed:
                items.append({'id': eid, 'applied': 0, 'fields': []})
                continue
            out = _apply_to_db(kind, eid, proposed, srcs, x_user_email)
            applied = out.get('applied') or {}
            fields_total += len(applied)
            items.append({'id': eid, 'applied': len(applied), 'fields': list(applied.keys())})
            try:
                from audit_seal_router import audit_log as _audit_log
                if applied:
                    _audit_log(action='enrich.bulk.apply',
                               target_type=kind, target_id=eid,
                               payload={'applied': applied},
                               user_id=x_user_id, user_email=x_user_email)
            except Exception:
                pass
        except HTTPException as e:
            items.append({'id': eid, 'error': e.detail})
        except Exception as e:
            items.append({'id': eid, 'error': f'{type(e).__name__}: {e}'})

    return {
        'status':       'success',
        'kind':         kind,
        'requested':    limit,
        'processed':    len(items),
        'fields_total': fields_total,
        'elapsed_s':    round(time.time() - started, 1),
        'items':        items,
    }


# ─── SB-4 — Worker status / control ─────────────────────────────────────
_REDIS_KEYS = {
    'heartbeat':   'cc:pgz-enricher:heartbeat',
    'pause':       'cc:pgz-enricher:pause',
    'run_now':     'cc:pgz-enricher:run_now',
    'last_cycle':  'cc:pgz-enricher:last_cycle',
    'confidence':  'cc:pgz-enricher:confidence',
    'fields_24h':  'cc:pgz-enricher:fields_24h',
}


def _redis_client():
    try:
        import redis
    except Exception:
        return None
    host = os.environ.get('REDIS_HOST', 'localhost')
    port = int(os.environ.get('REDIS_PORT', '6379'))
    pwd  = (os.environ.get('REDIS_PASS') or '').strip().strip("'").strip('"') or None
    # Try with password first (prod); fall back to anonymous (dev box) on AUTH failure.
    for p in (pwd, None):
        try:
            r = redis.Redis(host=host, port=port, password=p,
                            decode_responses=True, socket_connect_timeout=2)
            r.ping()
            return r
        except Exception:
            continue
    return None


@router.get("/enrich/worker/status")
def enrich_worker_status():
    r = _redis_client()
    out = {'available': bool(r)}
    if not r:
        return out
    try:
        hb = r.get(_REDIS_KEYS['heartbeat'])
        out['heartbeat']        = int(hb) if hb else None
        out['heartbeat_age_s']  = (int(time.time()) - int(hb)) if hb else None
        out['paused']           = (r.get(_REDIS_KEYS['pause']) or '0') == '1'
        out['run_now_pending']  = (r.get(_REDIS_KEYS['run_now']) or '0') == '1'
        last = r.get(_REDIS_KEYS['last_cycle'])
        if last:
            try:    out['last_cycle'] = json.loads(last)
            except: out['last_cycle'] = last
        conf = r.get(_REDIS_KEYS['confidence'])
        out['confidence_threshold'] = float(conf) if conf else 0.7
        f24 = r.get(_REDIS_KEYS['fields_24h'])
        out['fields_24h'] = int(f24) if f24 and f24.isdigit() else 0
    except Exception as e:
        out['error'] = f'{type(e).__name__}: {e}'
    # Recent enrichment_log rows for live activity
    try:
        with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
            cur.execute("""SELECT id, kind, target_id, source, fields_set, user_email, created_at
                           FROM pgz_sport.enrichment_log
                           ORDER BY id DESC LIMIT 25""")
            rows = []
            for rr in cur.fetchall():
                rr = dict(rr)
                if rr.get('created_at'): rr['created_at'] = rr['created_at'].isoformat()
                rows.append(rr)
            out['recent'] = rows
    except Exception:
        out['recent'] = []
    return out


@router.post("/enrich/worker/pause")
def enrich_worker_pause(body: dict = Body(default=None)):
    body = body or {}
    pause = bool(body.get('paused', True))
    r = _redis_client()
    if not r: raise HTTPException(503, 'redis unavailable')
    r.set(_REDIS_KEYS['pause'], '1' if pause else '0')
    return {'status': 'success', 'paused': pause}


@router.post("/enrich/worker/run-now")
def enrich_worker_run_now():
    r = _redis_client()
    if not r: raise HTTPException(503, 'redis unavailable')
    r.set(_REDIS_KEYS['run_now'], '1')
    return {'status': 'success', 'queued': True}


@router.post("/enrich/worker/confidence")
def enrich_worker_confidence(body: dict = Body(...)):
    try:
        v = float(body.get('value'))
    except Exception:
        raise HTTPException(400, 'value must be number 0..1')
    if not (0.0 <= v <= 1.0):
        raise HTTPException(400, 'value out of range 0..1')
    r = _redis_client()
    if not r: raise HTTPException(503, 'redis unavailable')
    r.set(_REDIS_KEYS['confidence'], str(v))
    return {'status': 'success', 'confidence_threshold': v}