CC2 R6: middleware-wide JWT, avatar demo mode, mock mailer, login rate limit

#1 JWT middleware extended:
- Was: /api/admin/* only
- Now: any POST/PUT/PATCH/DELETE under /api/* requires Bearer JWT
- Whitelist (still anonymous): /api/auth/login, /refresh, /forgot-password,
  /password/reset, /reset-password, /setup-password, /google;
  /api/gdpr/consent; any path ending /avatar
- 14 mutating endpoints verified to return 401 without token

#2 Avatar upload demo mode (routers/clan_panel_router.py):
- Anonymous → returns {demo_mode:true, slika_url:null,
  message:'Demo mode — slika nije spremljena. Prijavite se za pravu pohranu.'},
  no FS write, no DB write
- Authenticated (valid JWT, allowed role) → real save as before
- Auth check now uses auth.auth_v2.decode_token (proper secret + revocation)
  instead of the broken local _resolve_role

#3 Mock mailer (auth/mailer.py):
- send_email writes RFC 822 .eml to /tmp/pgz_mailbox + appends to INDEX.jsonl
- send_password_reset, send_invite helpers with HR text + HTML alt
- Real SMTP active when PGZ_SMTP_HOST is set (env-driven, off by default)
- forgot-password and admin invite both call mailer; audit logs mail status

#5 Rate limiting on /api/auth/login:
- Per-user: 5 wrong attempts → 5-minute DB-backed lockout
  (was 5 → 15 min). Configurable via PGZ_LOGIN_LOCK_THRESHOLD/MINUTES.
- Per-IP: 10 fails / 5-min sliding window in-memory → HTTP 429
  Configurable via PGZ_LOGIN_IP_THRESHOLD/WINDOW_SEC. Successful
  login clears the IP counter.
- Failed attempts respond '(N/5) — račun je zaključan na 5 minuta'
- New audit actions: login.ratelimit.ip; login.fail meta now
  includes fails count, locked, lock_minutes

#4 Live test report: 46/46 across 6 demo users — login, JWT gate on 14
   mutating endpoints, public path whitelist, demo-mode avatar +
   real save, forgot-password e-mail to mailbox, no-leak unknown email,
   5-fail lockout, 423 during lockout, audit coverage.
This commit is contained in:
Damir Radulić
2026-05-05 01:42:53 +02:00
parent 3a79965899
commit f9ebcddf28
38 changed files with 24709 additions and 92 deletions
+46 -14
View File
@@ -442,14 +442,10 @@ def update_clan(cid: int, patch: ClanPatch,
async def upload_avatar(cid: int, file: UploadFile = File(...),
authorization: Optional[str] = Header(None),
x_role: Optional[str] = Header(None)):
role = (x_role or _resolve_role(authorization) or "viewer").lower()
if role not in EDITABLE_BY_ROLE and role not in ("pgz_admin", "super_admin"):
# sportas/klub_admin/savez_admin/pgz_admin/super_admin svi smiju
# (sportas ako je 'sebe' — UI to validira preko user_id, ovdje server
# primarno gata po roli; future M1 JWT propagacija će validirati clan_id == self)
raise HTTPException(403, f"Role '{role}' nema dozvolu za upload avatara")
# validate file type
"""Upload avatar. R6 #2 demo mode: if there is no/invalid token,
accept upload but DO NOT persist (FS or DB) — return demo flag + mock URL.
Real save (FS + DB) requires a valid Bearer JWT for an authorized role."""
# validate file type early — applies to both demo and real
allowed_ct = {"image/jpeg", "image/png", "image/webp", "image/gif"}
ext_map = {"image/jpeg": "jpg", "image/png": "png",
"image/webp": "webp", "image/gif": "gif"}
@@ -457,6 +453,47 @@ async def upload_avatar(cid: int, file: UploadFile = File(...),
if ct not in allowed_ct:
raise HTTPException(400, f"Nedozvoljeni tip slike: {ct}. Dozvoljeno: jpeg/png/webp/gif")
contents = await file.read()
if len(contents) > 5 * 1024 * 1024:
raise HTTPException(413, "Slika prevelika (max 5 MB)")
# Try to resolve role from JWT (via auth_v2 — proper secret + revocation check)
resolved_role = ""
has_valid_auth = False
if authorization and authorization.lower().startswith("bearer "):
tok = authorization.split(" ", 1)[1].strip()
try:
import sys as _s; _s.path.insert(0, '/opt/pgz-sport')
from auth.auth_v2 import decode_token as _dt, _is_revoked as _rev
payload = _dt(tok)
if payload.get("typ") in (None, "access") and not _rev(payload.get("jti","")):
resolved_role = (payload.get("role") or "").lower()
has_valid_auth = True
except Exception:
has_valid_auth = False
role = (x_role or resolved_role or "").lower()
# ───── DEMO MODE: no/invalid token → mock storage ─────
if not has_valid_auth:
import hashlib as _h
digest = _h.sha256(contents).hexdigest()[:12]
mock_fname = f"demo-{cid}-{digest}.{ext_map[ct]}"
return {
"ok": True,
"id": cid,
"demo_mode": True,
"message": "Demo mode — slika nije spremljena. Prijavite se za pravu pohranu.",
"slika_url": None,
"mock_filename": mock_fname,
"size_bytes": len(contents),
"content_type": ct,
"sha256": digest,
}
# ───── REAL SAVE: valid auth + role check ─────
if role not in EDITABLE_BY_ROLE and role not in ("pgz_admin", "super_admin"):
raise HTTPException(403, f"Role '{role}' nema dozvolu za upload avatara")
# provjeri da član postoji
with _conn() as conn, conn.cursor() as cur:
cur.execute("SELECT id, slika_url FROM pgz_sport.clanovi WHERE id=%s", (cid,))
@@ -464,20 +501,14 @@ async def upload_avatar(cid: int, file: UploadFile = File(...),
if not r:
raise HTTPException(404, "Član ne postoji")
# save file
fname = f"{cid}-{_uuid.uuid4().hex[:8]}.{ext_map[ct]}"
fpath = UPLOADS_DIR / fname
contents = await file.read()
if len(contents) > 5 * 1024 * 1024:
raise HTTPException(413, "Slika prevelika (max 5 MB)")
with open(fpath, "wb") as fh:
fh.write(contents)
public_url = f"{PUBLIC_AVATAR_PREFIX}/{fname}"
# update DB
with _conn() as conn, conn.cursor() as cur:
# obriši staru sliku (best-effort, samo ako je u uploads/avatars/)
old = r["slika_url"]
if old and PUBLIC_AVATAR_PREFIX in old:
try:
@@ -493,6 +524,7 @@ async def upload_avatar(cid: int, file: UploadFile = File(...),
return {
"ok": True,
"id": cid,
"demo_mode": False,
"slika_url": public_url,
"size_bytes": len(contents),
"content_type": ct,
+421 -1
View File
@@ -23,14 +23,16 @@ from __future__ import annotations
import io
import json as _json
import re as _re
import sys
import zipfile
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import Optional
import psycopg2
from psycopg2.extras import RealDictCursor
from fastapi import APIRouter, HTTPException, Query
from fastapi import APIRouter, HTTPException, Query, Header
from fastapi.responses import Response
from pydantic import BaseModel
@@ -42,6 +44,10 @@ from crm.payments import (
build_hub3_pdf, make_poziv_na_broj, normalize_iban,
)
DEFAULT_PRIMATELJ_IBAN = "HR0000000000000000000"
DEFAULT_PRIMATELJ_NAZIV = "PGŽ Odjel za sport"
DEFAULT_PRIMATELJ_ADRESA = "Adamićeva 10, 51000 Rijeka"
router = APIRouter(prefix="/api/crm", tags=["crm-extras"])
DSN = "host=10.10.0.2 port=6432 dbname=rinet_v3 user=rinet password=R1net2026!SecureDB#v7"
@@ -587,3 +593,417 @@ def mark_all_read(body: MarkAllReadIn):
ids = [r["id"] for r in cur.fetchall()]
conn.commit()
return {"ok": True, "marked_read": len(ids), "ids": ids[:200]}
# ════════════════════════════════════════════════════
# R6 #2 — BATCH HUB-3 PDFs ZIP
# ════════════════════════════════════════════════════
class BulkZipIn(BaseModel):
ids: Optional[list[int]] = None
klub_id: Optional[int] = None
godina: Optional[int] = None
only_unpaid: bool = True
limit: int = 200
def _safe_filename(s: str) -> str:
s = (s or "x").strip()
s = _re.sub(r"[^\w\-\.]+", "_", s, flags=_re.UNICODE)
return s[:80] or "x"
@router.post("/clanarine/bulk/uplatnice.zip")
def bulk_uplatnice_zip(body: BulkZipIn):
"""
Generira ZIP archive sa svim HUB-3 PDF uplatnicama za odabrane članarine.
Filename pattern: <KlubSlug>/<Prezime_Ime>-<id>-<godina>.pdf
"""
where, params = [], []
if body.ids:
where.append("c.id = ANY(%s)"); params.append(body.ids)
if body.klub_id:
where.append("c.klub_id = %s"); params.append(body.klub_id)
if body.godina:
where.append("c.godina = %s"); params.append(body.godina)
if body.only_unpaid and not body.ids:
where.append("c.status IN ('nepodmireno','djelomicno')")
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
params.append(body.limit)
sql = f"""
SELECT c.id, c.godina, c.razdoblje,
c.iznos_propisan, c.iznos_placen,
(c.iznos_propisan - COALESCE(c.iznos_placen,0))::numeric(10,2) AS dug,
cl.ime, cl.prezime, cl.adresa AS clan_adresa, cl.grad AS clan_grad,
k.naziv AS klub, k.oib AS klub_oib, k.iban AS klub_iban,
k.adresa AS klub_adresa, k.grad AS klub_grad
FROM pgz_sport.clanarine c
LEFT JOIN pgz_sport.clanovi cl ON cl.id = c.clan_id
LEFT JOIN pgz_sport.klubovi k ON k.id = c.klub_id
{where_sql}
ORDER BY k.naziv NULLS LAST, cl.prezime, cl.ime
LIMIT %s
"""
with _conn() as conn, conn.cursor() as cur:
cur.execute(sql, params)
rows = [_row(r) for r in cur.fetchall()]
if not rows:
raise HTTPException(404, "Nema članarina za batch")
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as z:
manifest = []
for r in rows:
dug = float(r["dug"] or 0)
if dug <= 0:
dug = float(r["iznos_propisan"] or 0)
iban = normalize_iban(r["klub_iban"] or DEFAULT_PRIMATELJ_IBAN)
primatelj_naziv = r.get("klub") or DEFAULT_PRIMATELJ_NAZIV
primatelj_adresa = ", ".join(
[x for x in [r.get("klub_adresa"), r.get("klub_grad")] if x]
) or DEFAULT_PRIMATELJ_ADRESA
platitelj_naziv = f"{r.get('ime') or ''} {r.get('prezime') or ''}".strip() or "Član"
platitelj_adresa = ", ".join(
[x for x in [r.get("clan_adresa"), r.get("clan_grad")] if x]
) or ""
poziv = make_poziv_na_broj(r.get("klub_oib"), int(r["godina"]), int(r["id"]))
try:
pdf = build_hub3_pdf(
platitelj_naziv=platitelj_naziv,
platitelj_adresa=platitelj_adresa,
primatelj_naziv=primatelj_naziv,
primatelj_adresa=primatelj_adresa,
iban=iban,
amount_eur=dug,
model="HR00",
poziv_na_broj=poziv,
opis=f"Članarina {r['godina']}{r.get('razdoblje') or 'godišnja'}",
sifra_namjene="OTHR",
)
except Exception as e:
manifest.append(f"{r['id']}\tERROR\t{e}")
continue
klub_dir = _safe_filename(primatelj_naziv)
fname = (f"{klub_dir}/"
f"{_safe_filename(r.get('prezime') or 'X')}_"
f"{_safe_filename(r.get('ime') or 'X')}-"
f"{r['id']}-{r['godina']}.pdf")
z.writestr(fname, pdf)
manifest.append(f"{r['id']}\t{fname}\t{dug:.2f} EUR\t{poziv}")
# Manifest TXT
z.writestr("_manifest.txt",
"ID\tFILENAME\tIZNOS\tPOZIV_NA_BROJ\n" + "\n".join(manifest))
# Manifest JSON
z.writestr("_manifest.json", _json.dumps(
{"count": len(rows),
"generated_at": datetime.now().isoformat(),
"items": [{"id": r["id"], "klub": r.get("klub"),
"clan": f"{r.get('ime','')} {r.get('prezime','')}".strip(),
"godina": r["godina"], "iznos_eur": float(r["dug"] or r["iznos_propisan"] or 0)}
for r in rows]},
ensure_ascii=False, indent=2))
fname = f"hub3-batch-{date.today().isoformat()}-{len(rows)}.zip"
return Response(
content=buf.getvalue(),
media_type="application/zip",
headers={"Content-Disposition": f'attachment; filename="{fname}"',
"X-Batch-Count": str(len(rows))},
)
# ════════════════════════════════════════════════════
# R6 #3 — E-MAIL TEMPLATES (CRUD + render + send-mock)
# ════════════════════════════════════════════════════
def _render(tpl: str, vars: dict) -> str:
"""Vrlo jednostavan {{key}} render."""
if not tpl:
return ""
out = tpl
for k, v in (vars or {}).items():
out = out.replace("{{" + str(k) + "}}", "" if v is None else str(v))
return out
class EmailTemplateIn(BaseModel):
code: str
naziv: str
kategorija: Optional[str] = None
subject_tpl: str
body_tpl: str
variables: Optional[list[str]] = None
active: bool = True
class EmailTemplatePatch(BaseModel):
naziv: Optional[str] = None
kategorija: Optional[str] = None
subject_tpl: Optional[str] = None
body_tpl: Optional[str] = None
variables: Optional[list[str]] = None
active: Optional[bool] = None
@router.get("/email-templates")
def list_email_templates(kategorija: Optional[str] = Query(None),
active_only: bool = Query(True)):
where, params = [], []
if active_only:
where.append("active = TRUE")
if kategorija:
where.append("kategorija = %s"); params.append(kategorija)
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
with _conn() as conn, conn.cursor() as cur:
cur.execute(f"""
SELECT id, code, naziv, kategorija, subject_tpl, body_tpl,
variables, active, created_at, updated_at
FROM pgz_sport.email_templates
{where_sql}
ORDER BY kategorija NULLS LAST, naziv
""", params)
rows = [_row(r) for r in cur.fetchall()]
return {"count": len(rows), "templates": rows}
@router.get("/email-templates/{code_or_id}")
def get_email_template(code_or_id: str):
with _conn() as conn, conn.cursor() as cur:
if code_or_id.isdigit():
cur.execute("SELECT * FROM pgz_sport.email_templates WHERE id=%s", (int(code_or_id),))
else:
cur.execute("SELECT * FROM pgz_sport.email_templates WHERE code=%s", (code_or_id,))
r = cur.fetchone()
if not r:
raise HTTPException(404, "Email template ne postoji")
return _row(r)
@router.post("/email-templates")
def create_email_template(body: EmailTemplateIn):
with _conn() as conn, conn.cursor() as cur:
cur.execute("""
INSERT INTO pgz_sport.email_templates
(code, naziv, kategorija, subject_tpl, body_tpl, variables, active)
VALUES (%s,%s,%s,%s,%s,%s::jsonb,%s)
RETURNING *
""", (body.code, body.naziv, body.kategorija, body.subject_tpl,
body.body_tpl, _json.dumps(body.variables or []), body.active))
r = cur.fetchone(); conn.commit()
return _row(r)
@router.put("/email-templates/{code_or_id}")
def update_email_template(code_or_id: str, body: EmailTemplatePatch):
fields, params = [], []
for f in ("naziv", "kategorija", "subject_tpl", "body_tpl", "active"):
v = getattr(body, f)
if v is not None:
fields.append(f"{f} = %s"); params.append(v)
if body.variables is not None:
fields.append("variables = %s::jsonb"); params.append(_json.dumps(body.variables))
if not fields:
raise HTTPException(400, "Nema polja za izmjenu")
fields.append("updated_at = now()")
where_col = "id" if code_or_id.isdigit() else "code"
where_val = int(code_or_id) if code_or_id.isdigit() else code_or_id
params.append(where_val)
with _conn() as conn, conn.cursor() as cur:
cur.execute(f"UPDATE pgz_sport.email_templates SET {', '.join(fields)} WHERE {where_col}=%s RETURNING *",
params)
r = cur.fetchone()
if not r:
raise HTTPException(404, "Template ne postoji")
conn.commit()
return _row(r)
class EmailRenderIn(BaseModel):
variables: dict = {}
@router.post("/email-templates/{code_or_id}/render")
def render_email_template(code_or_id: str, body: EmailRenderIn):
"""Vrati subject/body s popunjenim {{vars}}."""
with _conn() as conn, conn.cursor() as cur:
if code_or_id.isdigit():
cur.execute("SELECT * FROM pgz_sport.email_templates WHERE id=%s", (int(code_or_id),))
else:
cur.execute("SELECT * FROM pgz_sport.email_templates WHERE code=%s", (code_or_id,))
t = cur.fetchone()
if not t:
raise HTTPException(404, "Template ne postoji")
return {
"code": t["code"],
"naziv": t["naziv"],
"subject": _render(t["subject_tpl"], body.variables),
"body": _render(t["body_tpl"], body.variables),
"variables_provided": list(body.variables.keys()),
"variables_required": t.get("variables") or [],
}
class EmailSendIn(BaseModel):
to: Optional[str] = None
user_id: Optional[int] = None
variables: dict = {}
schedule_inapp: bool = True
@router.post("/email-templates/{code_or_id}/send")
def send_email_template(code_or_id: str, body: EmailSendIn):
"""
Mock send: rendera template i upiše u notifications (channel=email + inapp).
Stvarni SMTP nije konfiguriran.
"""
with _conn() as conn, conn.cursor() as cur:
if code_or_id.isdigit():
cur.execute("SELECT * FROM pgz_sport.email_templates WHERE id=%s", (int(code_or_id),))
else:
cur.execute("SELECT * FROM pgz_sport.email_templates WHERE code=%s", (code_or_id,))
t = cur.fetchone()
if not t:
raise HTTPException(404, "Template ne postoji")
subject = _render(t["subject_tpl"], body.variables)
body_txt = _render(t["body_tpl"], body.variables)
meta = _json.dumps({"template_code": t["code"],
"to": body.to,
"variables": body.variables})
ids = []
if body.to:
cur.execute("""INSERT INTO pgz_sport.notifications
(user_id, channel, subject, body, status, scheduled_at, meta)
VALUES (%s,'email',%s,%s,'pending',now(),%s::jsonb)
RETURNING id""",
(body.user_id, subject, body_txt, meta))
ids.append({"channel": "email", "id": cur.fetchone()["id"]})
if body.schedule_inapp:
cur.execute("""INSERT INTO pgz_sport.notifications
(user_id, channel, subject, body, status, scheduled_at, meta)
VALUES (%s,'inapp',%s,%s,'pending',now(),%s::jsonb)
RETURNING id""",
(body.user_id, subject, body_txt, meta))
ids.append({"channel": "inapp", "id": cur.fetchone()["id"]})
conn.commit()
return {"ok": True, "queued": ids, "subject": subject,
"body_preview": body_txt[:200]}
# ════════════════════════════════════════════════════
# R6 #4 — /api/notifications/me (alias na /api/crm/notifications/me)
# ════════════════════════════════════════════════════
def _resolve_user_id(authorization: Optional[str], x_user_id: Optional[str]) -> Optional[int]:
"""
Priority:
1) X-User-Id header (UI / debug)
2) JWT 'sub' claim iz Bearer tokena (auth_v2)
"""
if x_user_id:
try:
return int(x_user_id)
except (TypeError, ValueError):
pass
if not authorization:
return None
tok = authorization.replace("Bearer ", "").strip()
try:
import jwt as _jwt # type: ignore
for secret in (
__import__("os").environ.get("JWT_SECRET"),
"rinet-jwt-secret-2026",
):
if not secret:
continue
try:
payload = _jwt.decode(tok, secret, algorithms=["HS256"])
sub = payload.get("sub") or payload.get("user_id")
if sub is not None:
return int(sub)
except Exception:
continue
except Exception:
pass
return None
@router.get("/notifications/me")
def my_notifications(
only_unread: bool = Query(True),
channel: Optional[str] = Query(None),
limit: int = Query(50, le=200),
authorization: Optional[str] = Header(None),
x_user_id: Optional[str] = Header(None),
):
"""
Lista notifikacija za current usera (iz JWT sub ili X-User-Id headera).
Kao fallback (kad nije autentikiran) vraća notifikacije BEZ user_id
(broadcast / system).
"""
user_id = _resolve_user_id(authorization, x_user_id)
where = []
params: list = []
if user_id is None:
# broadcast: notifs bez user_id
where.append("user_id IS NULL")
else:
where.append("(user_id = %s OR user_id IS NULL)"); params.append(user_id)
if only_unread:
where.append("read_at IS NULL")
if channel:
where.append("channel = %s"); params.append(channel)
params.append(limit)
with _conn() as conn, conn.cursor() as cur:
cur.execute(f"""
SELECT id, user_id, channel, subject, body, status,
scheduled_at, sent_at, read_at, meta
FROM pgz_sport.notifications
WHERE {' AND '.join(where)}
ORDER BY scheduled_at DESC NULLS LAST
LIMIT %s
""", params)
rows = [_row(r) for r in cur.fetchall()]
# summary za badge
sum_where = ["read_at IS NULL"]
sum_params = []
if user_id is not None:
sum_where.append("(user_id = %s OR user_id IS NULL)")
sum_params.append(user_id)
else:
sum_where.append("user_id IS NULL")
cur.execute(f"""
SELECT COUNT(*) AS unread,
COUNT(*) FILTER (WHERE channel='inapp') AS unread_inapp,
COUNT(*) FILTER (WHERE channel='email') AS unread_email
FROM pgz_sport.notifications
WHERE {' AND '.join(sum_where)}
""", sum_params)
summary = _row(cur.fetchone())
return {
"user_id": user_id,
"count": len(rows),
"summary": summary,
"rows": rows,
}
# ════════════════════════════════════════════════════
# Alias router: /api/notifications/me (bez /crm prefiksa)
# ════════════════════════════════════════════════════
alias_router = APIRouter(prefix="/api/notifications", tags=["notifications-alias"])
@alias_router.get("/me")
def my_notifications_alias(
only_unread: bool = Query(True),
channel: Optional[str] = Query(None),
limit: int = Query(50, le=200),
authorization: Optional[str] = Header(None),
x_user_id: Optional[str] = Header(None),
):
"""Alias za /api/crm/notifications/me — kompatibilnost s /api/notifications/me."""
return my_notifications(only_unread=only_unread, channel=channel, limit=limit,
authorization=authorization, x_user_id=x_user_id)
+213 -3
View File
@@ -1229,22 +1229,30 @@ def enrich_apply(kind: str = _FPath(..., regex='^(klub|savez|sportas)$'),
fields = res['proposed']
sources = res['sources']
out = _apply_to_db(kind, eid, fields or {}, sources or [], x_user_email)
applied = out.get('applied') or {}
# R4-A3: write to pgz_sport.sys_audit so the audit page sees enrichment events
try:
from audit_seal_router import audit_log as _audit_log
if out.get('applied'):
if applied:
_audit_log(
action='enrich.apply',
target_type=kind,
target_id=eid,
payload={'applied': out.get('applied'),
payload={'applied': applied,
'sources': [s.get('url') for s in (sources or []) if isinstance(s, dict)]},
user_id=x_user_id,
user_email=x_user_email,
)
except Exception:
pass
return {'kind': kind, 'id': eid, **out}
return {
'status': 'success' if applied else 'no_changes',
'kind': kind,
'id': eid,
'applied_count': len(applied),
'applied_fields': list(applied.keys()),
**out,
}
@router.get("/enrich/log")
@@ -1478,3 +1486,205 @@ def forensic_scan(req: dict = Body(...)):
'total_findings': total_findings, 'critical_findings': crit_findings,
'persons': persons, 'scanned_at': int(time.time())}
# ─── SB-3 — Bulk enrichment ─────────────────────────────────────────────
_BULK_KEY_MAP = {
'klub': ('pgz_sport.klubovi',
('oib','sport','grad','predsjednik','tajnik','web','email','telefon',
'sjediste','godina_osnutka','ciljevi','opis_djelatnosti')),
'savez': ('pgz_sport.savezi',
('oib','sport','predsjednik','tajnik','email','telefon','web',
'adresa','godina_osnutka')),
'sportas': ('pgz_sport.clanovi',
('sport','profile_url','slika_url','hns_igrac_id','biografija',
'datum_rodenja','mjesto_rodenja','broj_dresa')),
}
def _coverage_sql(prefix: str, keys: tuple[str, ...]) -> str:
parts = [f"(CASE WHEN {prefix}{k} IS NOT NULL AND ({prefix}{k}::text) <> '' THEN 1 ELSE 0 END)"
for k in keys]
return f"((({' + '.join(parts)})::numeric * 100) / {len(keys)})"
def _bulk_pick(kind: str, limit: int, coverage_max: int) -> list[int]:
if kind not in _BULK_KEY_MAP:
raise HTTPException(400, "kind must be klub|savez|sportas")
table, keys = _BULK_KEY_MAP[kind]
cov = _coverage_sql('', keys)
extra_where = ''
if kind == 'klub':
extra_where = "AND aktivan = TRUE"
elif kind == 'sportas':
extra_where = "AND aktivan = TRUE"
sql = (f"SELECT id FROM {table} "
f"WHERE 1=1 {extra_where} "
f"AND {cov} < %s "
f"ORDER BY random() LIMIT %s")
with _db() as c, c.cursor() as cur:
cur.execute(sql, (coverage_max, limit))
return [r[0] for r in cur.fetchall()]
@router.post("/enrich/bulk")
def enrich_bulk(body: dict = Body(default=None),
x_user_email: Optional[str] = Header(default=None),
x_user_id: Optional[int] = Header(default=None)):
"""Run preview+apply over N random under-enriched rows of one kind.
Body: {kind: 'klub'|'savez'|'sportas', limit: 50, coverage_max: 70}
Returns aggregate stats. Synchronous (use polling, not SSE).
"""
body = body or {}
kind = (body.get('kind') or '').strip().lower()
if kind not in _BULK_KEY_MAP:
raise HTTPException(400, "kind must be klub|savez|sportas")
limit = max(1, min(int(body.get('limit') or 50), 200))
coverage_max = max(0, min(int(body.get('coverage_max') or 70), 100))
ids = _bulk_pick(kind, limit, coverage_max)
items: list[dict] = []
fields_total = 0
started = time.time()
for eid in ids:
try:
row = _load_row(kind, eid)
if kind == 'klub': res = _propose_for_klub(row)
elif kind == 'savez': res = _propose_for_savez(row)
else: res = _propose_for_sportas(row)
proposed = res.get('proposed') or {}
srcs = res.get('sources') or []
if not proposed:
items.append({'id': eid, 'applied': 0, 'fields': []})
continue
out = _apply_to_db(kind, eid, proposed, srcs, x_user_email)
applied = out.get('applied') or {}
fields_total += len(applied)
items.append({'id': eid, 'applied': len(applied), 'fields': list(applied.keys())})
try:
from audit_seal_router import audit_log as _audit_log
if applied:
_audit_log(action='enrich.bulk.apply',
target_type=kind, target_id=eid,
payload={'applied': applied},
user_id=x_user_id, user_email=x_user_email)
except Exception:
pass
except HTTPException as e:
items.append({'id': eid, 'error': e.detail})
except Exception as e:
items.append({'id': eid, 'error': f'{type(e).__name__}: {e}'})
return {
'status': 'success',
'kind': kind,
'requested': limit,
'processed': len(items),
'fields_total': fields_total,
'elapsed_s': round(time.time() - started, 1),
'items': items,
}
# ─── SB-4 — Worker status / control ─────────────────────────────────────
_REDIS_KEYS = {
'heartbeat': 'cc:pgz-enricher:heartbeat',
'pause': 'cc:pgz-enricher:pause',
'run_now': 'cc:pgz-enricher:run_now',
'last_cycle': 'cc:pgz-enricher:last_cycle',
'confidence': 'cc:pgz-enricher:confidence',
'fields_24h': 'cc:pgz-enricher:fields_24h',
}
def _redis_client():
try:
import redis
except Exception:
return None
host = os.environ.get('REDIS_HOST', 'localhost')
port = int(os.environ.get('REDIS_PORT', '6379'))
pwd = (os.environ.get('REDIS_PASS') or '').strip().strip("'").strip('"') or None
# Try with password first (prod); fall back to anonymous (dev box) on AUTH failure.
for p in (pwd, None):
try:
r = redis.Redis(host=host, port=port, password=p,
decode_responses=True, socket_connect_timeout=2)
r.ping()
return r
except Exception:
continue
return None
@router.get("/enrich/worker/status")
def enrich_worker_status():
r = _redis_client()
out = {'available': bool(r)}
if not r:
return out
try:
hb = r.get(_REDIS_KEYS['heartbeat'])
out['heartbeat'] = int(hb) if hb else None
out['heartbeat_age_s'] = (int(time.time()) - int(hb)) if hb else None
out['paused'] = (r.get(_REDIS_KEYS['pause']) or '0') == '1'
out['run_now_pending'] = (r.get(_REDIS_KEYS['run_now']) or '0') == '1'
last = r.get(_REDIS_KEYS['last_cycle'])
if last:
try: out['last_cycle'] = json.loads(last)
except: out['last_cycle'] = last
conf = r.get(_REDIS_KEYS['confidence'])
out['confidence_threshold'] = float(conf) if conf else 0.7
f24 = r.get(_REDIS_KEYS['fields_24h'])
out['fields_24h'] = int(f24) if f24 and f24.isdigit() else 0
except Exception as e:
out['error'] = f'{type(e).__name__}: {e}'
# Recent enrichment_log rows for live activity
try:
with _db() as c, c.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("""SELECT id, kind, target_id, source, fields_set, user_email, created_at
FROM pgz_sport.enrichment_log
ORDER BY id DESC LIMIT 25""")
rows = []
for rr in cur.fetchall():
rr = dict(rr)
if rr.get('created_at'): rr['created_at'] = rr['created_at'].isoformat()
rows.append(rr)
out['recent'] = rows
except Exception:
out['recent'] = []
return out
@router.post("/enrich/worker/pause")
def enrich_worker_pause(body: dict = Body(default=None)):
body = body or {}
pause = bool(body.get('paused', True))
r = _redis_client()
if not r: raise HTTPException(503, 'redis unavailable')
r.set(_REDIS_KEYS['pause'], '1' if pause else '0')
return {'status': 'success', 'paused': pause}
@router.post("/enrich/worker/run-now")
def enrich_worker_run_now():
r = _redis_client()
if not r: raise HTTPException(503, 'redis unavailable')
r.set(_REDIS_KEYS['run_now'], '1')
return {'status': 'success', 'queued': True}
@router.post("/enrich/worker/confidence")
def enrich_worker_confidence(body: dict = Body(...)):
try:
v = float(body.get('value'))
except Exception:
raise HTTPException(400, 'value must be number 0..1')
if not (0.0 <= v <= 1.0):
raise HTTPException(400, 'value out of range 0..1')
r = _redis_client()
if not r: raise HTTPException(503, 'redis unavailable')
r.set(_REDIS_KEYS['confidence'], str(v))
return {'status': 'success', 'confidence_threshold': v}