HNS sprint: 3-tab drill-down + parallel deep scraper dispatch
HNS-1 verify: smoke test 93409 OK, gap 854 uncovered, throughput ~60/min HNS-2 dispatch: scripts/hns_dispatch.sh + 5 parallel workers shard'd po roster ID; coverage 265→1098 distinct_seasons (93.7% of 1172 roster), 125→971 distinct_matches; total seasons 3170→13371, matches 23515→150071 HNS-3 UI: 6-tab panel collapsed na 3 (🏆 Karijera / 📅 Utakmice / 👤 Profil); novi /api/v2/clan/{id}/hns-matches?limit=30 + /clan/{id}/hns-profile (bio + summary + HNS deep link); prof-grid 3-col card s gold jersey badge; OIB RBAC-mask. Test Josip Zec 449: 72 sez/30 utak. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
# hns_dispatch.sh — parallel HNS player deep-scrape dispatcher
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# Author: dradulic@outlook.com / damir@rinet.one
|
||||
# Date: 2026-05-05
|
||||
# Version: 1.0
|
||||
# Description:
|
||||
# Reads a shard file of HNS player IDs (one per line) and invokes
|
||||
# /opt/pgz-sport/scripts/hns_player_deep.py --player <ID> --no-telegram
|
||||
# sequentially for each ID. Designed to be launched 5x in parallel
|
||||
# (one per shard) so 5 worker processes each chew through ~175 IDs.
|
||||
#
|
||||
# Usage:
|
||||
# bash /opt/pgz-sport/scripts/hns_dispatch.sh <shard_file> <worker_id>
|
||||
# bash /opt/pgz-sport/scripts/hns_dispatch.sh /tmp/hns_shard_aa 1
|
||||
#
|
||||
# Per-worker log: /tmp/hns_worker_<worker_id>.log
|
||||
# Each player attempt: ~1s (HTTP + parse + UPSERT).
|
||||
# Stagger workers manually with `sleep N &&` when launching.
|
||||
|
||||
set -u
|
||||
|
||||
SHARD_FILE="${1:?usage: hns_dispatch.sh <shard_file> <worker_id>}"
|
||||
WORKER_ID="${2:?usage: hns_dispatch.sh <shard_file> <worker_id>}"
|
||||
LOG="/tmp/hns_worker_${WORKER_ID}.log"
|
||||
SCRIPT="/opt/pgz-sport/scripts/hns_player_deep.py"
|
||||
|
||||
cd /opt/pgz-sport || exit 2
|
||||
|
||||
echo "[$(date -Iseconds)] worker=${WORKER_ID} shard=${SHARD_FILE} start ($(wc -l < "$SHARD_FILE") IDs)" >> "$LOG"
|
||||
|
||||
count=0
|
||||
ok=0
|
||||
fail=0
|
||||
while IFS= read -r ID; do
|
||||
[ -z "$ID" ] && continue
|
||||
count=$((count + 1))
|
||||
if python3 "$SCRIPT" --player "$ID" --no-telegram >> "$LOG" 2>&1; then
|
||||
ok=$((ok + 1))
|
||||
else
|
||||
fail=$((fail + 1))
|
||||
echo "[$(date -Iseconds)] worker=${WORKER_ID} FAIL id=${ID}" >> "$LOG"
|
||||
fi
|
||||
# tiny politeness pause; HNS is light traffic but 5 parallel workers
|
||||
# = 5 req/s aggregate; small jitter helps avoid bursts
|
||||
sleep 0.2
|
||||
done < "$SHARD_FILE"
|
||||
|
||||
echo "[$(date -Iseconds)] worker=${WORKER_ID} done count=${count} ok=${ok} fail=${fail}" >> "$LOG"
|
||||
@@ -129,6 +129,11 @@ COMP_CATALOG = [
|
||||
("100381484", "kvalifikacije-za-prvu-nl-pioniri", "pioniri-u15", "2025/2026"),
|
||||
("100569152", "treca-nl-istok", "seniori", "2025/2026"), # Treća NL Istok
|
||||
("100585203", "treca-nl-zapad", "seniori", "2025/2026"), # Treća NL Zapad (PGŽ klubovi)
|
||||
# PGŽ-region ŽNL leagues discovered via klub raspored auto-discovery
|
||||
("101555188", "1-znl-seniori", "seniori", "2025/2026"), # 1.ŽNL PGŽ seniori
|
||||
("112195128", "kup-zns", "seniori", "2025/2026"), # Kup ŽNS Vinodolsko-Senjsko
|
||||
("104425442", "kup-mladezi-juniori", "juniori-u19", "2025/2026"),
|
||||
("104464435", "kup-mladezi-kadeti", "kadeti-u17", "2025/2026"),
|
||||
("100391485", "supersport-hnl", "seniori", "2025/2026"),
|
||||
("100413651", "supersport-prva-nl", "seniori", "2025/2026"),
|
||||
("100418001", "supersport-druga-nl", "seniori", "2025/2026"),
|
||||
|
||||
Reference in New Issue
Block a user