Compare commits

...

10 Commits

Author SHA1 Message Date
Corback b9961fe220 Add action_queue + lore_avancee SQL migration 2026-06-15 17:39:39 +00:00
Corback 150e003f18 Add fallout-visu compose 2026-06-15 17:39:39 +00:00
Corback d4007aec00 Add fallout.coyoteos.ovh Flask visu - VaultCom terminal UI 2026-06-15 17:39:38 +00:00
Corback 1962bbb899 Add ingest_v2.log volume mount 2026-06-15 17:22:02 +00:00
Corback 7f5531e3b9 Fix fallout-ingest: prefer v2 log, expose version field 2026-06-15 17:22:01 +00:00
Corback af0c0d5f33 Add MAO ingest - ChromaDB mao_docs collection 2026-06-15 17:14:58 +00:00
Corback 5e7afaafae Add fallout ingest v2 - structural chunking + systeme metadata 2026-06-15 17:14:57 +00:00
Corback 25b46ff146 Add /admin/run-ingest endpoint for fallout retry 2026-06-15 16:07:41 +00:00
Corback d9bf4b7d28 Add docker-compose.yml 2026-06-15 16:02:16 +00:00
Corback 33c1429c52 Add index.html 2026-06-15 16:02:15 +00:00
8 changed files with 2201 additions and 4 deletions
+30 -4
View File
@@ -108,6 +108,17 @@ def docker_reboot_host():
docker_call("POST", "/containers/host-reboot/start", {})
return status in (201, 409)
def docker_run_host_cmd(name, cmd_list):
"""Run a command on the Ampère host via privileged nsenter container."""
docker_call("DELETE", f"/containers/{name}?force=1")
full_cmd = ["nsenter","-t","1","-m","-u","-i","-n","-p","--"] + cmd_list
config = {"Image": "alpine", "Cmd": full_cmd,
"HostConfig": {"Privileged": True, "PidMode": "host", "AutoRemove": True}}
status, body = docker_call("POST", f"/containers/create?name={name}", config)
if status in (201, 409):
docker_call("POST", f"/containers/{name}/start", {})
return status in (201, 409)
# ── SSH helpers (Guardian / Vigile) ───────────────────────────────────────
SSH_KEYS = {
"guardian": {"key": "/ssh/guardian.key", "host": "82.70.226.48"},
@@ -190,11 +201,16 @@ def get_fallout_ingest():
try:
with open("/data/fallout_state.json") as f: state=json.load(f)
except: pass
running = _log_running("/data/fallout_ingest.log")
# Préfère le log v2 s'il existe, sinon log v1
import os as _os
log_v2 = "/data/fallout_ingest_v2.log"
log_v1 = "/data/fallout_ingest.log"
log_path = log_v2 if _os.path.exists(log_v2) else log_v1
running = _log_running(log_path)
current_pdf=None; current_chunk=None; total_chunks=None
chunks_done=[]; log_tail=[]
chunks_done=[]; log_tail=[]; version="v2" if log_path==log_v2 else "v1"
try:
with open("/data/fallout_ingest.log") as f: lines=f.readlines()
with open(log_path) as f: lines=f.readlines()
log_tail=[l.rstrip() for l in lines[-30:] if l.strip()]
for l in reversed(lines):
l=l.rstrip()
@@ -207,7 +223,7 @@ def get_fallout_ingest():
if m2: chunks_done.append(float(m2.group(1)))
except: pass
avg_t=round(sum(chunks_done)/len(chunks_done),1) if chunks_done else None
return {"running":running,"done":len(state.get("ingested",[])),"failed":len(state.get("failed",[])),
return {"running":running,"version":version,"done":len(state.get("ingested",[])),"failed":len(state.get("failed",[])),
"total":17,"current_pdf":current_pdf if running else None,
"current_chunk":current_chunk,"total_chunks":total_chunks,"avg_embed_s":avg_t,"log_tail":log_tail}
@@ -358,6 +374,16 @@ class Handler(BaseHTTPRequestHandler):
ok = ssh_reboot(vm)
self.send_json({"ok":ok,"vm":vm})
elif p == "/admin/run-ingest":
body = self.read_body()
script = body.get("script","fallout")
if script == "fallout":
cmd = ["sh","-c","PYTHONUNBUFFERED=1 python3 -u /home/ubuntu/fallout/ingest.py >> /home/ubuntu/fallout/ingest.log 2>&1 &"]
else:
self.send_json({"ok":False,"error":"unknown script"}); return
ok = docker_run_host_cmd("host-ingest", cmd)
self.send_json({"ok":ok,"script":script})
else:
self.send_response(404); self.end_headers()
+27
View File
@@ -0,0 +1,27 @@
services:
dash:
image: python:3.11-alpine
container_name: dash
restart: unless-stopped
working_dir: /app
volumes:
- ./app.py:/app/app.py:ro
- ./index.html:/app/index.html:ro
- /home/ubuntu/wiki/pipeline_state.json:/data/pipeline_state.json:ro
- /home/ubuntu/wiki/pipeline.log:/data/pipeline.log:ro
- /home/ubuntu/fallout/ingest.log:/data/fallout_ingest.log:ro
- /home/ubuntu/fallout/ingest_v2.log:/data/fallout_ingest_v2.log:ro
- /home/ubuntu/fallout/ingest_state.json:/data/fallout_state.json:ro
- /proc:/proc:ro
- /var/run/docker.sock:/var/run/docker.sock
- /home/ubuntu/.ssh/guardian.key:/ssh/guardian.key:ro
- /home/ubuntu/.ssh/vigile_backup:/ssh/vigile.key:ro
command: python app.py
networks:
- proxy-nw
extra_hosts:
- "host.docker.internal:host-gateway"
networks:
proxy-nw:
external: true
+687
View File
@@ -0,0 +1,687 @@
"""
fallout.coyoteos.ovh — Venice of Wasteland — Visualisation simulation
Flask app sur Ampère, lit PostgreSQL Vigile via SSH tunnel
"""
import subprocess, time, threading, os, json
from flask import Flask, render_template_string, jsonify, abort
app = Flask(__name__)
# ── SSH tunnel → PostgreSQL Vigile ────────────────────────────────────────────
TUNNEL_LOCAL_PORT = 15432
_tunnel_proc = None
def start_tunnel():
global _tunnel_proc
cmd = [
"ssh", "-N", "-L", f"{TUNNEL_LOCAL_PORT}:localhost:5432",
"-i", "/ssh/vigile.key",
"-o", "StrictHostKeyChecking=no",
"-o", "ServerAliveInterval=30",
"-o", "ServerAliveCountMax=3",
"ubuntu@79.72.30.231"
]
_tunnel_proc = subprocess.Popen(cmd)
time.sleep(3)
print(f"SSH tunnel → Vigile:5432 via local:{TUNNEL_LOCAL_PORT}")
def get_conn():
import psycopg2
return psycopg2.connect(
host="localhost", port=TUNNEL_LOCAL_PORT,
dbname="fallout", user="fallout",
password="VeniceOfWasteland2026!",
connect_timeout=5
)
def query(sql, params=None):
try:
conn = get_conn()
cur = conn.cursor()
cur.execute(sql, params or ())
cols = [d[0] for d in cur.description]
rows = [dict(zip(cols, r)) for r in cur.fetchall()]
conn.close()
return rows
except Exception as e:
print(f"DB error: {e}")
return []
def execute(sql, params=None):
try:
conn = get_conn()
cur = conn.cursor()
cur.execute(sql, params or ())
conn.commit()
conn.close()
return True
except Exception as e:
print(f"DB error: {e}")
return False
# ── Templates ─────────────────────────────────────────────────────────────────
BASE_CSS = """
<style>
@import url('https://fonts.googleapis.com/css2?family=VT323&family=Share+Tech+Mono&display=swap');
:root {
--green: #39ff14;
--green-dim: #1a7a08;
--green-dark: #0d3d04;
--amber: #ffb000;
--red: #ff3c3c;
--bg: #050a05;
--card-bg: #0a150a;
--border: #1f4d1f;
}
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
background: var(--bg);
color: var(--green);
font-family: 'Share Tech Mono', monospace;
font-size: 14px;
min-height: 100vh;
position: relative;
}
body::before {
content: '';
position: fixed; inset: 0; pointer-events: none; z-index: 9999;
background: repeating-linear-gradient(0deg, transparent, transparent 2px, rgba(0,0,0,0.08) 2px, rgba(0,0,0,0.08) 4px);
}
h1, h2, h3, .title { font-family: 'VT323', monospace; letter-spacing: 2px; }
h1 { font-size: 2.4rem; color: var(--green); text-shadow: 0 0 10px var(--green); }
h2 { font-size: 1.6rem; color: var(--green); margin-bottom: 12px; }
h3 { font-size: 1.2rem; color: var(--amber); }
a { color: var(--green-dim); text-decoration: none; }
a:hover { color: var(--green); text-shadow: 0 0 6px var(--green); }
.layout { display: flex; min-height: 100vh; }
.sidebar {
width: 220px; flex-shrink: 0;
background: var(--card-bg); border-right: 1px solid var(--border);
padding: 20px 0;
}
.sidebar .logo { padding: 0 20px 20px; border-bottom: 1px solid var(--border); }
.sidebar .logo .sub { font-size: 11px; color: var(--green-dim); }
.sidebar nav a {
display: block; padding: 10px 20px;
color: var(--green-dim); font-size: 13px;
border-left: 3px solid transparent;
transition: all 0.15s;
}
.sidebar nav a:hover, .sidebar nav a.active {
color: var(--green); border-left-color: var(--green);
background: rgba(57,255,20,0.04);
text-shadow: 0 0 6px var(--green);
}
.sidebar nav .section-label {
padding: 16px 20px 4px; font-size: 10px;
color: var(--green-dim); letter-spacing: 2px; text-transform: uppercase;
}
.main { flex: 1; padding: 30px; overflow: auto; }
.topbar {
display: flex; justify-content: space-between; align-items: center;
margin-bottom: 28px; padding-bottom: 16px;
border-bottom: 1px solid var(--border);
}
.badge {
display: inline-block; padding: 3px 10px;
border: 1px solid; border-radius: 2px;
font-size: 11px; letter-spacing: 1px; text-transform: uppercase;
}
.badge-green { border-color: var(--green); color: var(--green); }
.badge-amber { border-color: var(--amber); color: var(--amber); }
.badge-red { border-color: var(--red); color: var(--red); }
.badge-dim { border-color: var(--border); color: var(--green-dim); }
.card {
background: var(--card-bg); border: 1px solid var(--border);
border-radius: 2px; padding: 20px; margin-bottom: 16px;
}
.card-header { border-bottom: 1px solid var(--border); padding-bottom: 10px; margin-bottom: 14px; }
.grid { display: grid; gap: 16px; }
.grid-2 { grid-template-columns: 1fr 1fr; }
.grid-3 { grid-template-columns: 1fr 1fr 1fr; }
.grid-5 { grid-template-columns: repeat(5, 1fr); }
.stat-bar { margin: 6px 0; }
.stat-bar label { display: flex; justify-content: space-between; margin-bottom: 3px; font-size: 12px; }
.bar-track { background: var(--green-dark); border-radius: 1px; height: 6px; }
.bar-fill { height: 6px; background: var(--green); border-radius: 1px; transition: width 0.3s; }
.bar-fill.danger { background: var(--red); }
.bar-fill.warning { background: var(--amber); }
table { width: 100%; border-collapse: collapse; }
th { text-align: left; padding: 8px 12px; color: var(--green-dim); font-size: 11px; letter-spacing: 1px; text-transform: uppercase; border-bottom: 1px solid var(--border); }
td { padding: 8px 12px; border-bottom: 1px solid rgba(31,77,31,0.4); font-size: 13px; }
tr:hover td { background: rgba(57,255,20,0.03); }
.timeline { list-style: none; }
.timeline li {
position: relative; padding: 10px 10px 10px 24px;
border-left: 1px solid var(--border); margin-left: 8px;
}
.timeline li::before {
content: ''; position: absolute; left: -8px;
background: var(--bg); color: var(--green-dim); font-size: 11px;
}
.timeline .meta { font-size: 11px; color: var(--green-dim); margin-bottom: 4px; }
.timeline .action { color: var(--amber); }
.timeline .result { color: var(--green); margin-top: 4px; font-size: 13px; }
.special-grid { display: grid; grid-template-columns: repeat(7, 1fr); gap: 8px; margin: 12px 0; }
.special-box { text-align: center; padding: 10px 6px; border: 1px solid var(--border); }
.special-box .label { font-size: 10px; color: var(--green-dim); }
.special-box .val { font-size: 1.8rem; font-family: 'VT323', monospace; color: var(--green); }
.hp-display { display: flex; align-items: center; gap: 12px; margin: 12px 0; }
.hp-num { font-family: 'VT323', monospace; font-size: 2rem; }
.hp-num.danger { color: var(--red); }
.hp-num.warning { color: var(--amber); }
.empty { text-align: center; padding: 40px; color: var(--green-dim); }
.empty .ico { font-size: 2rem; margin-bottom: 8px; }
.status-alive { color: var(--green); }
.status-injured { color: var(--amber); }
.status-dead { color: var(--red); }
.status-pending { color: var(--amber); }
.status-resolving { color: #00bfff; }
.status-resolved { color: var(--green-dim); }
.ticker {
background: var(--card-bg); border: 1px solid var(--border);
padding: 8px 14px; margin-bottom: 16px; font-size: 12px;
color: var(--green-dim); white-space: nowrap; overflow: hidden;
}
.blink { animation: blink 1s step-end infinite; }
@keyframes blink { 50% { opacity: 0; } }
@media (max-width: 900px) {
.grid-5 { grid-template-columns: 1fr 1fr; }
.grid-3 { grid-template-columns: 1fr 1fr; }
.sidebar { display: none; }
}
</style>
"""
def nav(active=""):
return f"""
<div class="sidebar">
<div class="logo">
<h1 style="font-size:1.4rem">☢ VAULTCOM</h1>
<div class="sub">Venice of Wasteland</div>
<div class="sub">La Corboration</div>
</div>
<nav>
<div class="section-label">Simulation</div>
<a href="/" class="{'active' if active=='home' else ''}">⬡ Mission Control</a>
<a href="/world" class="{'active' if active=='world' else ''}">◈ État du Monde</a>
<div class="section-label">Agents</div>
<a href="/characters" class="{'active' if active=='chars' else ''}">◉ Personnages</a>
<a href="/actions" class="{'active' if active=='actions' else ''}">▶ File d'Actions</a>
<div class="section-label">Archives</div>
<a href="/lore" class="{'active' if active=='lore' else ''}">◎ Lore Avancée</a>
<a href="/events" class="{'active' if active=='events' else ''}">⊟ Journal</a>
</nav>
</div>
"""
SHELL = """<!DOCTYPE html>
<html lang="fr">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>VaultCom — {{ title }}</title>
""" + BASE_CSS + """
</head>
<body>
<div class="layout">
{{ nav | safe }}
<div class="main">
<div class="topbar">
<h2>{{ title }}</h2>
<div style="display:flex;gap:10px;align-items:center">
{% if session %}
<span class="badge badge-green">Jour {{ session.day_current }}/{{ session.day_total }}</span>
<span class="badge badge-dim">{{ session.mode | upper }}</span>
{% else %}
<span class="badge badge-red">DB OFFLINE</span>
{% endif %}
<span style="font-size:11px;color:var(--green-dim)" id="clock"></span>
</div>
</div>
{{ content | safe }}
</div>
</div>
<script>
function tick() { document.getElementById('clock').textContent = new Date().toLocaleTimeString('fr-FR'); }
tick(); setInterval(tick, 1000);
</script>
</body>
</html>"""
def render(title, content, active="", session=None):
return render_template_string(SHELL, title=title, content=content,
nav=nav(active), session=session)
# ── Helpers ───────────────────────────────────────────────────────────────────
def get_session():
rows = query("SELECT * FROM sessions ORDER BY id DESC LIMIT 1")
return rows[0] if rows else None
def hp_class(cur, mx):
if mx == 0: return "warning"
pct = cur / mx
return "danger" if pct < 0.33 else ("warning" if pct < 0.66 else "")
def bar_class(cur, mx):
if mx == 0: return "warning"
pct = cur / mx
return "danger" if pct < 0.33 else ("warning" if pct < 0.66 else "")
# ── Routes ────────────────────────────────────────────────────────────────────
@app.route("/")
def index():
sess = get_session()
chars = query("SELECT * FROM characters WHERE session_id = %s ORDER BY faction, name",
(sess["id"],)) if sess else []
recent = query("""SELECT e.*, c.name as char_name FROM events e
JOIN characters c ON c.id = e.character_id
WHERE e.session_id = %s ORDER BY e.day_in_game DESC, e.hour_in_game DESC LIMIT 12""",
(sess["id"],)) if sess else []
factions = query("""SELECT DISTINCT faction, COUNT(*) as nb,
SUM(CASE WHEN status='alive' THEN 1 ELSE 0 END) as alive
FROM characters WHERE session_id = %s GROUP BY faction ORDER BY faction""",
(sess["id"],)) if sess else []
pending = query("SELECT COUNT(*) as n FROM action_queue WHERE session_id = %s AND status='pending'",
(sess["id"],))[0]["n"] if sess else 0
lore_last = query("""SELECT * FROM lore_avancee WHERE session_id = %s
ORDER BY day_start DESC LIMIT 1""", (sess["id"],)) if sess else []
# Characters grid
char_cards = ""
if chars:
for c in chars:
hp_pct = int(c["hp_current"] / max(c["hp_max"], 1) * 100)
bc = bar_class(c["hp_current"], c["hp_max"])
nc = hp_class(c["hp_current"], c["hp_max"])
st_cls = f"status-{c['status']}"
char_cards += f"""
<a href="/character/{c['id']}" style="text-decoration:none">
<div class="card" style="cursor:pointer">
<div class="card-header" style="display:flex;justify-content:space-between">
<h3>{c['name']}</h3>
<span class="badge badge-dim {st_cls}">{c['status'].upper()}</span>
</div>
<div style="font-size:11px;color:var(--green-dim);margin-bottom:8px">{c['faction'] or ''} · {c['location'] or '?'}</div>
<div class="stat-bar">
<label><span>HP</span><span class="{nc}">{c['hp_current']}/{c['hp_max']}</span></label>
<div class="bar-track"><div class="bar-fill {bc}" style="width:{hp_pct}%"></div></div>
</div>
<div style="font-size:11px;color:var(--green-dim);margin-top:6px">
CAPS: {c['caps']} · XP: {c['xp']}
</div>
</div>
</a>"""
else:
char_cards = '<div class="empty"><div class="ico">◉</div>Aucun agent initialisé</div>'
# Recent events
ev_html = ""
if recent:
ev_html = '<ul class="timeline">'
for e in recent:
ev_html += f"""
<li>
<div class="meta">Jour {e['day_in_game']}h{e['hour_in_game']:02d} · {e['char_name']} · {e['location'] or '?'}</div>
<div class="action">▶ {e['action'][:120]}{'' if len(e['action'])>120 else ''}</div>
{'<div class="result">⟹ ' + (e['result'][:150] if e['result'] else '') + '</div>' if e['result'] else ''}
</li>"""
ev_html += '</ul>'
else:
ev_html = '<div class="empty"><div class="ico">⊟</div>Journal vide — simulation non démarrée</div>'
# Faction summary
fac_html = ""
for f in factions:
fac_html += f'<tr><td>{f["faction"] or "?"}</td><td>{f["alive"]}</td><td>{f["nb"]}</td></tr>'
# Lore entry
lore_html = ""
if lore_last:
l = lore_last[0]
lore_html = f"""
<div class="card">
<div class="card-header">
<h3>{l['title'] or f"Résumé jours {l['day_start']}{l['day_end']}"}</h3>
<div style="font-size:11px;color:var(--green-dim)">Jours {l['day_start']}{l['day_end']}</div>
</div>
<div style="line-height:1.7;color:var(--green)">{l['content'][:600]}{'' if len(l['content'])>600 else ''}</div>
</div>"""
else:
lore_html = '<div class="empty"><div class="ico">◎</div>Pas encore de lore synthétisé</div>'
content = f"""
<div class="ticker">
▌ VAULTCOM ACTIVE ▌ SESSION: {sess['name'] if sess else 'N/A'}
AGENTS: {len(chars)} ▌ ACTIONS EN ATTENTE: {pending}
<span class="blink">█</span>
</div>
<div class="grid grid-5" style="margin-bottom:20px">{char_cards}</div>
<div class="grid grid-2">
<div class="card">
<div class="card-header"><h3>Journal récent</h3></div>
{ev_html}
</div>
<div>
<div class="card" style="margin-bottom:16px">
<div class="card-header"><h3>Factions actives</h3></div>
<table>
<tr><th>Faction</th><th>Vivants</th><th>Total</th></tr>
{fac_html if fac_html else '<tr><td colspan="3" style="color:var(--green-dim);text-align:center">—</td></tr>'}
</table>
</div>
<div>
<h3 style="margin-bottom:10px">Dernière synthèse narrative</h3>
{lore_html}
</div>
</div>
</div>
"""
return render("Mission Control", content, "home", sess)
@app.route("/characters")
def characters():
sess = get_session()
chars = query("SELECT * FROM characters WHERE session_id = %s ORDER BY faction, name",
(sess["id"],)) if sess else []
rows = ""
for c in chars:
hp_pct = int(c["hp_current"] / max(c["hp_max"], 1) * 100)
bc = bar_class(c["hp_current"], c["hp_max"])
rows += f"""
<tr>
<td><a href="/character/{c['id']}">{c['name']}</a></td>
<td style="color:var(--amber)">{c['faction'] or ''}</td>
<td>{c['location'] or '?'}</td>
<td>
<div class="bar-track" style="width:80px;display:inline-block">
<div class="bar-fill {bc}" style="width:{hp_pct}%"></div>
</div>
<span style="font-size:11px;margin-left:6px">{c['hp_current']}/{c['hp_max']}</span>
</td>
<td>{c['caps']}</td>
<td class="status-{c['status']}">{c['status'].upper()}</td>
<td style="font-size:11px;color:var(--green-dim)">{c['xp']} XP</td>
</tr>"""
content = f"""
<div class="card">
<table>
<tr><th>Nom</th><th>Faction</th><th>Lieu</th><th>HP</th><th>Caps</th><th>Status</th><th>XP</th></tr>
{rows if rows else '<tr><td colspan="7" class="empty">Aucun personnage</td></tr>'}
</table>
</div>"""
return render("Agents", content, "chars", sess)
@app.route("/character/<int:cid>")
def character(cid):
sess = get_session()
chars = query("SELECT * FROM characters WHERE id = %s", (cid,))
if not chars: abort(404)
c = chars[0]
inv = query("SELECT * FROM inventory WHERE character_id = %s ORDER BY item_name", (cid,))
events = query("""SELECT * FROM events WHERE character_id = %s
ORDER BY day_in_game, hour_in_game""", (cid,))
SPECIALS = [("S","strength"),("P","perception"),("E","endurance"),
("C","charisma"),("I","intelligence"),("A","agility"),("L","luck")]
sp_html = "".join(f"""
<div class="special-box">
<div class="val">{c[key]}</div>
<div class="label">{abbr}</div>
</div>""" for abbr, key in SPECIALS)
hp_pct = int(c["hp_current"] / max(c["hp_max"], 1) * 100)
bc = bar_class(c["hp_current"], c["hp_max"])
nc = hp_class(c["hp_current"], c["hp_max"])
inv_rows = "".join(f"<tr><td>{i['item_name']}</td><td>{i.get('quantity',1)}</td><td style='color:var(--green-dim);font-size:12px'>{i.get('description','')}</td></tr>"
for i in inv) if inv else "<tr><td colspan='3' style='color:var(--green-dim)'>Inventaire vide</td></tr>"
ev_html = ""
if events:
ev_html = '<ul class="timeline">'
for e in events:
ev_html += f"""
<li>
<div class="meta">Jour {e['day_in_game']}h{e['hour_in_game']:02d} · {e['location'] or '?'}</div>
<div class="action">▶ {e['action']}</div>
{'<div class="result">⟹ ' + (e['result'] or '') + '</div>' if e['result'] else ''}
</li>"""
ev_html += '</ul>'
else:
ev_html = '<div class="empty">Aucune action enregistrée</div>'
bio_html = f'<div style="color:var(--green-dim);line-height:1.7;margin-top:8px">{c["bio"] or "Dossier vide."}</div>'
content = f"""
<div class="card" style="margin-bottom:20px">
<div style="display:flex;justify-content:space-between;align-items:flex-start">
<div>
<h2 style="font-size:2rem">{c['name']}</h2>
<div style="color:var(--amber);margin:4px 0">{c['faction'] or ''}</div>
<div style="color:var(--green-dim);font-size:12px">Localisation: {c['location'] or '?'} · Caps: {c['caps']} · XP: {c['xp']}</div>
</div>
<span class="badge badge-dim status-{c['status']}">{c['status'].upper()}</span>
</div>
<div class="hp-display">
<span class="hp-num {nc}">{c['hp_current']}</span>
<div style="flex:1">
<div style="font-size:11px;color:var(--green-dim);margin-bottom:4px">/{c['hp_max']} HP</div>
<div class="bar-track"><div class="bar-fill {bc}" style="width:{hp_pct}%"></div></div>
</div>
</div>
<div class="special-grid">{sp_html}</div>
{bio_html}
</div>
<div class="grid grid-2">
<div class="card">
<div class="card-header"><h3>Inventaire</h3></div>
<table>
<tr><th>Objet</th><th>Qté</th><th>Description</th></tr>
{inv_rows}
</table>
</div>
<div class="card">
<div class="card-header"><h3>Journal d'actions</h3></div>
{ev_html}
</div>
</div>"""
return render(c["name"], content, "chars", sess)
@app.route("/actions")
def actions():
sess = get_session()
rows = query("""
SELECT aq.*, c.name as char_name, t.name as target_name
FROM action_queue aq
LEFT JOIN characters c ON c.id = aq.character_id
LEFT JOIN characters t ON t.id = aq.target_char_id
WHERE aq.session_id = %s
ORDER BY aq.day_in_game DESC, aq.hour_in_game DESC, aq.id DESC
LIMIT 100""", (sess["id"],)) if sess else []
def status_badge(s):
cls = {"pending":"amber","resolving":"green","resolved":"dim","cancelled":"red"}.get(s,"dim")
return f'<span class="badge badge-{cls}">{s.upper()}</span>'
trs = "".join(f"""
<tr>
<td>J{r['day_in_game']}h{r['hour_in_game']:02d}</td>
<td><a href="/character/{r['character_id']}">{r['char_name'] or '?'}</a></td>
<td>{r['location'] or '?'}</td>
<td>{r['action_text'][:80]}{'' if len(r['action_text'])>80 else ''}</td>
<td>{r['target_name'] or ''}</td>
<td>{r['scene_id'] or ''}</td>
<td>{status_badge(r['status'])}</td>
<td style="font-size:12px;color:var(--green-dim)">{(r['result_text'] or '')[:60]}</td>
</tr>""" for r in rows) if rows else "<tr><td colspan='8' class='empty'>Aucune action</td></tr>"
content = f"""
<div class="card">
<table>
<tr><th>Moment</th><th>Agent</th><th>Lieu</th><th>Action</th><th>Cible</th><th>Scène</th><th>Status</th><th>Résultat</th></tr>
{trs}
</table>
</div>"""
return render("File d'Actions", content, "actions", sess)
@app.route("/world")
def world():
sess = get_session()
if not sess:
return render("État du Monde", '<div class="empty">DB offline</div>', "world")
day = sess["day_current"]
states = query("""SELECT * FROM world_state WHERE session_id = %s AND day_in_game = %s
ORDER BY location""", (sess["id"], day))
factions = query("""SELECT * FROM faction_relations WHERE session_id = %s
ORDER BY faction_a, faction_b""", (sess["id"],))
def stab_color(s):
if s >= 70: return "var(--green)"
if s >= 40: return "var(--amber)"
return "var(--red)"
world_rows = "".join(f"""
<tr>
<td style="color:var(--amber)">{s['location']}</td>
<td>{s['controlling_faction'] or ''}</td>
<td>
<div class="bar-track" style="width:80px;display:inline-block">
<div class="bar-fill" style="width:{s['stability']}%;background:{stab_color(s['stability'])}"></div>
</div>
<span style="font-size:11px;margin-left:6px;color:{stab_color(s['stability'])}">{s['stability']}%</span>
</td>
<td>{s['population'] or '?'}</td>
<td style="font-size:12px;color:var(--green-dim)">{s['notes'] or ''}</td>
</tr>""" for s in states) if states else "<tr><td colspan='5' class='empty'>Aucune donnée</td></tr>"
fac_rows = "".join(f"""
<tr>
<td style="color:var(--amber)">{f['faction_a']}</td>
<td>{f['faction_b']}</td>
<td style="color:{'var(--green)' if f['relation_score']>0 else 'var(--red)' if f['relation_score']<0 else 'var(--green-dim)'}">{'+' if f['relation_score']>0 else ''}{f['relation_score']}</td>
<td style="font-size:12px;color:var(--green-dim)">{f.get('notes','')}</td>
</tr>""" for f in factions) if factions else "<tr><td colspan='4' class='empty'>Aucune donnée</td></tr>"
content = f"""
<div class="grid grid-2">
<div class="card">
<div class="card-header"><h3>Zones — Jour {day}</h3></div>
<table>
<tr><th>Lieu</th><th>Faction</th><th>Stabilité</th><th>Pop.</th><th>Notes</th></tr>
{world_rows}
</table>
</div>
<div class="card">
<div class="card-header"><h3>Relations entre factions</h3></div>
<table>
<tr><th>Faction A</th><th>Faction B</th><th>Score</th><th>Notes</th></tr>
{fac_rows}
</table>
</div>
</div>"""
return render("État du Monde", content, "world", sess)
@app.route("/lore")
def lore():
sess = get_session()
entries = query("""SELECT * FROM lore_avancee WHERE session_id = %s
ORDER BY day_start DESC""", (sess["id"],)) if sess else []
cards = ""
if entries:
for e in entries:
facs = ", ".join(e["factions_involved"] or []) or ""
locs = ", ".join(e["locations_involved"] or []) or ""
cards += f"""
<div class="card">
<div class="card-header" style="display:flex;justify-content:space-between">
<h3>{e['title'] or f"Chronique jours {e['day_start']}{e['day_end']}"}</h3>
<span class="badge badge-dim">J{e['day_start']}{e['day_end']}</span>
</div>
<div style="font-size:11px;color:var(--green-dim);margin-bottom:12px">
Factions: {facs} · Lieux: {locs}
</div>
<div style="line-height:1.8;color:var(--green)">{e['content']}</div>
</div>"""
else:
cards = '<div class="empty"><div class="ico">◎</div>Aucune synthèse narrative générée<br><span style="font-size:12px">Le MJ produira des entrées toutes les 24h de jeu simulé</span></div>'
return render("Lore Avancée", cards, "lore", sess)
@app.route("/events")
def events():
sess = get_session()
evs = query("""SELECT e.*, c.name as char_name FROM events e
JOIN characters c ON c.id = e.character_id
WHERE e.session_id = %s
ORDER BY e.day_in_game DESC, e.hour_in_game DESC
LIMIT 200""", (sess["id"],)) if sess else []
rows = "".join(f"""
<tr>
<td>J{e['day_in_game']}h{e['hour_in_game']:02d}</td>
<td><a href="/character/{e['character_id']}">{e['char_name']}</a></td>
<td style="color:var(--green-dim)">{e['location'] or '?'}</td>
<td style="color:var(--amber)">{e['action'][:80]}{'' if len(e['action'])>80 else ''}</td>
<td style="font-size:12px">{(e['result'] or '')[:100]}{'' if e['result'] and len(e['result'])>100 else ''}</td>
</tr>""" for e in evs) if evs else "<tr><td colspan='5' class='empty'>Journal vide</td></tr>"
content = f"""
<div class="card">
<table>
<tr><th>Moment</th><th>Agent</th><th>Lieu</th><th>Action</th><th>Résultat</th></tr>
{rows}
</table>
</div>"""
return render("Journal des Événements", content, "events", sess)
@app.route("/api/summary")
def api_summary():
"""Pour le widget dashboard dash.coyoteos.ovh"""
sess = get_session()
if not sess:
return jsonify({"ok": False})
chars = query("SELECT status, COUNT(*) n FROM characters WHERE session_id=%s GROUP BY status", (sess["id"],))
pending = query("SELECT COUNT(*) n FROM action_queue WHERE session_id=%s AND status='pending'", (sess["id"],))
return jsonify({
"ok": True,
"session": sess["name"],
"day": sess["day_current"],
"day_total": sess["day_total"],
"chars": {r["status"]: r["n"] for r in chars},
"pending_actions": pending[0]["n"] if pending else 0,
})
# ── Main ──────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
t = threading.Thread(target=start_tunnel, daemon=True)
t.start()
print("VaultCom :5000")
app.run(host="0.0.0.0", port=5000, debug=False)
+21
View File
@@ -0,0 +1,21 @@
services:
fallout-visu:
image: python:3.11-alpine
container_name: fallout-visu
restart: unless-stopped
working_dir: /app
volumes:
- ./app.py:/app/app.py:ro
- /home/ubuntu/.ssh/vigile_backup:/ssh/vigile.key:ro
command: >
sh -c "apk add --no-cache openssh-client &&
pip install flask psycopg2-binary --quiet &&
python app.py"
networks:
- proxy-nw
environment:
- PYTHONUNBUFFERED=1
networks:
proxy-nw:
external: true
+46
View File
@@ -0,0 +1,46 @@
-- Migration fallout DB — tables manquantes
-- À exécuter sur Vigile : docker exec -i fallout_db psql -U fallout -d fallout
-- File d'actions (coeur du moteur de simulation)
CREATE TABLE IF NOT EXISTS action_queue (
id SERIAL PRIMARY KEY,
session_id INTEGER REFERENCES sessions(id),
day_in_game INTEGER NOT NULL,
hour_in_game INTEGER NOT NULL DEFAULT 0,
character_id INTEGER REFERENCES characters(id),
action_text TEXT NOT NULL,
location VARCHAR(100),
target_char_id INTEGER REFERENCES characters(id),
target_location VARCHAR(100),
scene_id VARCHAR(50), -- groupe les actions qui se croisent
status VARCHAR(20) DEFAULT 'pending', -- pending|resolving|resolved|cancelled
priority INTEGER DEFAULT 5,
result_text TEXT, -- rempli après résolution par le MJ
created_at TIMESTAMP DEFAULT now(),
resolved_at TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_aq_session_day_hour ON action_queue(session_id, day_in_game, hour_in_game);
CREATE INDEX IF NOT EXISTS idx_aq_status ON action_queue(status);
CREATE INDEX IF NOT EXISTS idx_aq_scene ON action_queue(scene_id);
-- Lore avancée — résumés narratifs générés par le LLM toutes les 24h de jeu
CREATE TABLE IF NOT EXISTS lore_avancee (
id SERIAL PRIMARY KEY,
session_id INTEGER REFERENCES sessions(id),
day_start INTEGER NOT NULL,
day_end INTEGER NOT NULL,
title VARCHAR(200),
content TEXT NOT NULL, -- texte narratif généré par MJ 14b
factions_involved TEXT[],
locations_involved TEXT[],
chroma_id VARCHAR(100), -- ref chunk fallout_canon dans ChromaDB
created_at TIMESTAMP DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_la_session ON lore_avancee(session_id, day_start);
-- Vérification
SELECT table_name FROM information_schema.tables
WHERE table_schema = 'public'
ORDER BY table_name;
+719
View File
@@ -0,0 +1,719 @@
<!DOCTYPE html>
<html lang="fr">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>CoyoteOS — Dashboard</title>
<style>
:root {
--bg:#0a0a0f; --surface:#111118; --surface2:#18181f; --border:#1f1f2e;
--text:#e2e2f0; --muted:#6b6b88;
--green:#22c55e; --orange:#f97316; --red:#ef4444;
--blue:#3b82f6; --purple:#a855f7; --accent:#7c3aed;
}
*{box-sizing:border-box;margin:0;padding:0}
body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,sans-serif;min-height:100vh}
/* Header */
header{display:flex;align-items:center;justify-content:space-between;padding:1rem 2rem;
border-bottom:1px solid var(--border);background:linear-gradient(90deg,#0d0d18,#110d1a);
position:sticky;top:0;z-index:100}
.logo{font-size:1.1rem;font-weight:700;letter-spacing:.05em;color:#fff}
.logo span{color:var(--accent)}
.live-dot{width:8px;height:8px;border-radius:50%;background:var(--green);
box-shadow:0 0 8px var(--green);animation:pulse 2s infinite;display:inline-block}
@keyframes pulse{0%,100%{opacity:1;transform:scale(1)}50%{opacity:.5;transform:scale(.8)}}
.last-update{font-size:.72rem;color:var(--muted)}
/* Layout */
main{padding:1.5rem 2rem;max-width:1600px;margin:0 auto;display:flex;flex-direction:column;gap:1.2rem}
/* Section title */
.section-title{font-size:.68rem;font-weight:700;color:var(--muted);text-transform:uppercase;
letter-spacing:.15em;margin-bottom:.8rem;display:flex;align-items:center;gap:.6rem}
.section-title::after{content:"";flex:1;height:1px;background:var(--border)}
/* Cards */
.card{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:1.2rem 1.4rem}
/* Grids */
.grid-3{display:grid;grid-template-columns:repeat(3,1fr);gap:1rem}
.grid-2{display:grid;grid-template-columns:repeat(2,1fr);gap:1rem}
@media(max-width:1000px){.grid-3{grid-template-columns:repeat(2,1fr)}}
@media(max-width:650px){.grid-3,.grid-2{grid-template-columns:1fr}}
/* Progress bars */
.bar-bg{background:#1a1a2a;border-radius:6px;height:5px;overflow:hidden}
.bar-fill{height:100%;border-radius:6px;transition:width .6s ease}
/* Colors */
.c-green{color:var(--green)} .c-orange{color:var(--orange)} .c-red{color:var(--red)}
.c-blue{color:var(--blue)} .c-purple{color:var(--purple)} .c-muted{color:var(--muted)}
/* Pipeline card */
.pipeline-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;overflow:hidden}
.pipeline-header{display:flex;align-items:center;justify-content:space-between;
padding:1rem 1.4rem;border-bottom:1px solid var(--border);
background:linear-gradient(135deg,#0d0d18,#160d24);cursor:pointer;user-select:none}
.pipeline-header h2{font-size:.85rem;font-weight:600;color:#c4b5fd}
.pipeline-body{padding:1.2rem 1.4rem;display:flex;flex-direction:column;gap:1rem}
.pipeline-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:.8rem}
.pipeline-stat{text-align:center}
.pipeline-stat .n{font-size:2.2rem;font-weight:700}
.pipeline-stat .l{font-size:.65rem;color:var(--muted);text-transform:uppercase;letter-spacing:.1em;margin-top:.2rem}
.current-pdf{background:var(--surface2);border:1px solid var(--border);border-radius:8px;padding:.8rem 1rem}
.pdf-name{font-size:.9rem;font-weight:600;color:#c4b5fd;margin-bottom:.4rem;word-break:break-all}
.chunk-progress{display:flex;align-items:center;gap:.8rem;margin-top:.6rem}
.chunk-bar-wrap{flex:1;background:#1a1a2a;border-radius:4px;height:8px;overflow:hidden}
.chunk-bar{height:100%;background:linear-gradient(90deg,var(--accent),var(--purple));border-radius:4px;transition:width .4s}
.chunk-label{font-size:.75rem;color:var(--muted);white-space:nowrap}
.log-box{background:#080810;border:1px solid var(--border);border-radius:8px;padding:.8rem;
font-family:'Cascadia Code','Consolas',monospace;font-size:.7rem;max-height:160px;overflow-y:auto;line-height:1.6}
.log-ok{color:var(--green)} .log-err{color:var(--red)} .log-info{color:#6b6b88}
.log-cur{color:#c4b5fd} .log-chunk{color:var(--blue)} .log-dl{color:var(--orange)}
/* VM cards */
.vm-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;overflow:hidden}
.vm-header{padding:.8rem 1.2rem;border-bottom:1px solid var(--border);display:flex;align-items:center;gap:.7rem}
.vm-dot{width:9px;height:9px;border-radius:50%}
.vm-dot.up{background:var(--green);box-shadow:0 0 6px var(--green)}
.vm-dot.down{background:var(--red)}
.vm-name{font-size:.85rem;font-weight:600}
.vm-ip{font-size:.7rem;color:var(--muted);margin-left:auto;font-family:monospace}
.vm-body{padding:1rem 1.2rem;display:flex;flex-direction:column;gap:.5rem}
.vm-row{display:flex;align-items:center;justify-content:space-between}
.vm-row .lbl{font-size:.72rem;color:var(--muted)}
.vm-row .val{font-size:.85rem;font-weight:600}
.mini-bar-wrap{flex:1;margin:0 .8rem;background:#1a1a2a;border-radius:3px;height:4px;overflow:hidden}
.mini-bar{height:100%;border-radius:3px;transition:width .5s}
/* Summary */
.summary-card{background:linear-gradient(135deg,#0f0a1e,#0a1020);border:1px solid #2a1a4a;
border-radius:12px;padding:1.4rem;display:grid;grid-template-columns:repeat(3,1fr);gap:1rem;text-align:center}
.sum-item .n{font-size:1.8rem;font-weight:700}
.sum-item .l{font-size:.65rem;color:var(--muted);text-transform:uppercase;letter-spacing:.1em;margin-top:.3rem}
.sum-divider{border-left:1px solid var(--border)}
/* Sites */
.sites-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:.8rem}
.site-btn{background:var(--surface);border:1px solid var(--border);border-radius:10px;
padding:1rem;text-decoration:none;color:var(--text);
display:flex;flex-direction:column;align-items:center;gap:.5rem;
transition:border-color .2s,background .2s}
.site-btn:hover{background:var(--surface2);border-color:var(--accent)}
.site-btn .site-icon{font-size:1.5rem}
.site-btn .site-name{font-size:.8rem;font-weight:600;text-align:center}
.site-btn .site-status{display:flex;align-items:center;gap:.4rem;font-size:.68rem}
.sdot{width:7px;height:7px;border-radius:50%;flex-shrink:0}
.sdot-up{background:var(--green);box-shadow:0 0 5px var(--green)}
.sdot-down{background:var(--red);box-shadow:0 0 5px var(--red)}
/* Badge */
.badge{display:inline-flex;align-items:center;gap:.35rem;border-radius:20px;
padding:.2rem .7rem;font-size:.68rem;font-weight:600}
.badge-running{background:#0a2a0a;border:1px solid #22c55e44;color:var(--green)}
.badge-idle{background:#1a1a1a;border:1px solid #33333366;color:var(--muted)}
.badge-done{background:#0a1a2a;border:1px solid #3b82f644;color:var(--blue)}
/* Collapsible */
.coll-hdr{display:flex;align-items:center;justify-content:space-between;
cursor:pointer;user-select:none;margin-top:.6rem;padding-top:.6rem;border-top:1px solid var(--border)}
.coll-hdr .coll-lbl{font-size:.65rem;color:var(--muted);text-transform:uppercase;letter-spacing:.1em}
.coll-arrow{font-size:.7rem;color:var(--muted);transition:transform .2s}
.coll-arrow.open{transform:rotate(180deg)}
.coll-body{overflow:hidden;transition:max-height .3s ease;max-height:0}
.coll-body.open{max-height:600px}
.coll-inner{padding-top:.4rem}
/* Container list */
.ct-row{display:flex;align-items:center;gap:.4rem;padding:.12rem 0;font-size:.7rem}
.ct-dot{width:6px;height:6px;border-radius:50%;flex-shrink:0}
.ct-dot.up{background:var(--green);box-shadow:0 0 4px var(--green)}
.ct-dot.down{background:var(--red)}
.ct-name{color:var(--text);flex:1}
.ct-img{color:var(--muted);font-size:.63rem}
/* Ollama models */
.ollama-model{display:flex;align-items:center;gap:.5rem;padding:.2rem 0;font-size:.72rem}
.ollama-model .m-name{color:var(--green);font-weight:600;flex:1}
.ollama-model .m-size{color:var(--muted)}
/* AdminSys */
.admin-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:1rem}
.admin-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;overflow:hidden}
.admin-card-header{padding:.8rem 1.2rem;background:linear-gradient(135deg,#120a1e,#0a1220);
border-bottom:1px solid var(--border);display:flex;align-items:center;gap:.6rem}
.admin-card-header .a-title{font-size:.85rem;font-weight:600;flex:1}
.admin-card-body{padding:.8rem 1.2rem;display:flex;flex-direction:column;gap:.4rem}
/* Reboot buttons */
.reboot-row{display:flex;gap:.6rem;flex-wrap:wrap;padding:.4rem 0}
.btn-reboot{background:#1a0a0a;border:1px solid #ef444444;color:var(--red);
border-radius:8px;padding:.4rem .9rem;font-size:.75rem;font-weight:600;cursor:pointer;
transition:background .2s;white-space:nowrap}
.btn-reboot:hover{background:#2a0a0a}
/* Group container controls */
.group-card{background:var(--surface2);border:1px solid var(--border);border-radius:8px;margin-bottom:.4rem}
.group-hdr{display:flex;align-items:center;gap:.5rem;padding:.5rem .8rem;cursor:pointer}
.group-icon{font-size:.9rem}
.group-name{font-size:.78rem;font-weight:600;flex:1}
.group-badge{font-size:.62rem;color:var(--muted);background:#1a1a2a;border-radius:4px;padding:.1rem .4rem}
.btn-group-toggle{font-size:.67rem;color:var(--muted);background:transparent;border:1px solid var(--border);
border-radius:4px;padding:.15rem .5rem;cursor:pointer;transition:background .15s}
.btn-group-toggle:hover{background:var(--border)}
.group-containers{padding:0 .8rem .5rem;display:flex;flex-direction:column;gap:.25rem}
.ct-control-row{display:flex;align-items:center;gap:.4rem}
.ct-control-name{flex:1;font-size:.72rem;color:var(--text)}
.btn-ct{font-size:.62rem;border:none;border-radius:4px;padding:.2rem .5rem;cursor:pointer;font-weight:600;transition:opacity .15s}
.btn-ct:hover{opacity:.8}
.btn-ct.stop{background:#2a0a0a;color:var(--red)}
.btn-ct.start{background:#0a2a0a;color:var(--green)}
.btn-ct.restart{background:#0a1a2a;color:var(--blue)}
.btn-ct:disabled{opacity:.4;cursor:not-allowed}
/* Toast */
.toast{position:fixed;bottom:1.5rem;right:1.5rem;background:#1a1a2a;border:1px solid var(--border);
border-radius:8px;padding:.7rem 1.2rem;font-size:.8rem;z-index:999;
animation:fadeIn .3s ease;pointer-events:none}
@keyframes fadeIn{from{opacity:0;transform:translateY(8px)}to{opacity:1;transform:translateY(0)}}
</style>
</head>
<body>
<header>
<div class="logo"><span>Coyote</span>OS — Dashboard</div>
<div style="display:flex;align-items:center;gap:1.2rem">
<span><span class="live-dot"></span><span style="font-size:.8rem;margin-left:.4rem">Live</span></span>
<span class="last-update" id="last-update">Chargement...</span>
</div>
</header>
<main>
<!-- ── PIPELINES ── -->
<div>
<div class="section-title">Pipelines actifs</div>
<div class="grid-2">
<!-- MAO -->
<div class="pipeline-card">
<div class="pipeline-header" onclick="toggleColl(this.nextElementSibling)">
<h2>qwen2.5 — Wiki MAO</h2>
<div style="display:flex;align-items:center;gap:.6rem">
<span id="pipeline-badge" class="badge badge-idle">● Vérification...</span>
<span style="color:var(--muted);font-size:.8rem"></span>
</div>
</div>
<div class="coll-body open" style="max-height:600px">
<div class="pipeline-body">
<div class="pipeline-grid">
<div class="pipeline-stat"><div class="n c-green" id="p-done"></div><div class="l">Terminés</div></div>
<div class="pipeline-stat"><div class="n" style="color:#888">77</div><div class="l">Total FR/EN</div></div>
<div class="pipeline-stat"><div class="n c-orange" id="p-failed"></div><div class="l">Échecs</div></div>
<div class="pipeline-stat"><div class="n c-blue" id="p-speed"></div><div class="l">Moy. s/chunk</div></div>
</div>
<div>
<div style="display:flex;justify-content:space-between;margin-bottom:.4rem">
<span style="font-size:.72rem;color:var(--muted)">Progression</span>
<span style="font-size:.72rem;font-weight:600" id="p-pct">0%</span>
</div>
<div class="bar-bg" style="height:8px"><div class="bar-fill" id="p-bar" style="width:0%;background:var(--purple)"></div></div>
</div>
<div class="current-pdf">
<div class="pdf-name" id="p-name">Attente...</div>
<div style="font-size:.7rem;color:var(--muted)" id="p-eta"></div>
<div class="chunk-progress">
<div class="chunk-bar-wrap"><div class="chunk-bar" id="p-chunk-bar" style="width:0%"></div></div>
<span class="chunk-label" id="p-chunk-label">chunk —/—</span>
</div>
</div>
<div>
<div style="font-size:.67rem;color:var(--muted);margin-bottom:.3rem;text-transform:uppercase;letter-spacing:.1em">Log</div>
<div class="log-box" id="log-box">Chargement...</div>
</div>
</div>
</div>
</div>
<!-- Fallout -->
<div class="pipeline-card">
<div class="pipeline-header" style="background:linear-gradient(135deg,#0d1010,#0d1a10)" onclick="toggleColl(this.nextElementSibling)">
<h2 style="color:#4ade80">nomic-embed-text — Fallout ChromaDB</h2>
<div style="display:flex;align-items:center;gap:.6rem">
<span id="fallout-badge" class="badge badge-idle">● Vérification...</span>
<span style="color:var(--muted);font-size:.8rem"></span>
</div>
</div>
<div class="coll-body open" style="max-height:600px">
<div class="pipeline-body">
<div class="pipeline-grid">
<div class="pipeline-stat"><div class="n c-green" id="f-done"></div><div class="l">Indexés</div></div>
<div class="pipeline-stat"><div class="n" style="color:#888">17</div><div class="l">Total</div></div>
<div class="pipeline-stat"><div class="n c-orange" id="f-failed"></div><div class="l">Échecs</div></div>
<div class="pipeline-stat"><div class="n c-blue" id="f-speed"></div><div class="l">Moy. s/embed</div></div>
</div>
<div>
<div style="display:flex;justify-content:space-between;margin-bottom:.4rem">
<span style="font-size:.72rem;color:var(--muted)">Progression</span>
<span style="font-size:.72rem;font-weight:600" id="f-pct">0%</span>
</div>
<div class="bar-bg" style="height:8px"><div class="bar-fill" id="f-bar" style="width:0%;background:var(--green)"></div></div>
</div>
<div class="current-pdf">
<div class="pdf-name" id="f-name" style="color:#4ade80">Attente...</div>
<div class="chunk-progress">
<div class="chunk-bar-wrap"><div class="chunk-bar" id="f-chunk-bar" style="width:0%;background:linear-gradient(90deg,#22c55e,#4ade80)"></div></div>
<span class="chunk-label" id="f-chunk-label">chunk —/—</span>
</div>
</div>
<div>
<div style="font-size:.67rem;color:var(--muted);margin-bottom:.3rem;text-transform:uppercase;letter-spacing:.1em">Log</div>
<div class="log-box" id="f-log-box">Chargement...</div>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- ── VMs ── -->
<div>
<div class="section-title">État des VMs</div>
<div class="grid-3">
<!-- Ampère -->
<div class="vm-card">
<div class="vm-header">
<div class="vm-dot up" id="vm-ampere-dot"></div>
<div><div class="vm-name">Ampère</div><div style="font-size:.65rem;color:var(--muted)">ARM 4c / 24 Go — Cerveau</div></div>
<div class="vm-ip">82.70.224.131</div>
</div>
<div class="vm-body">
<div class="vm-row"><span class="lbl">CPU</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-a-cpu-bar" style="width:0%"></div></div>
<span class="val" id="vm-a-cpu"></span></div>
<div class="vm-row"><span class="lbl">RAM</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-a-mem-bar" style="width:0%;background:var(--blue)"></div></div>
<span class="val c-blue" id="vm-a-mem"></span></div>
<div class="vm-row"><span class="lbl">Disque</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-a-disk-bar" style="width:0%;background:var(--purple)"></div></div>
<span class="val c-purple" id="vm-a-disk"></span></div>
<div class="vm-row"><span class="lbl">Uptime</span><span class="val c-green" id="vm-a-uptime"></span></div>
<div class="vm-row"><span class="lbl">Réseau ↓/↑</span><span class="val" id="vm-a-net" style="font-size:.78rem"></span></div>
<!-- Ollama -->
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
<span class="coll-lbl">Ollama — modèles actifs</span><span class="coll-arrow open"></span>
</div>
<div class="coll-body open"><div class="coll-inner" id="vm-a-ollama">Chargement...</div></div>
<!-- Containers -->
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
<span class="coll-lbl">Containers Docker (live)</span><span class="coll-arrow open"></span>
</div>
<div class="coll-body open"><div class="coll-inner" id="vm-a-containers">Chargement...</div></div>
</div>
</div>
<!-- Guardian -->
<div class="vm-card">
<div class="vm-header">
<div class="vm-dot up" id="vm-guardian-dot"></div>
<div><div class="vm-name">Guardian</div><div style="font-size:.65rem;color:var(--muted)">AMD 1c / 1 Go — Bouclier</div></div>
<div class="vm-ip">82.70.226.48</div>
</div>
<div class="vm-body">
<div class="vm-row"><span class="lbl">CPU</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-g-cpu-bar" style="width:0%"></div></div>
<span class="val" id="vm-g-cpu"></span></div>
<div class="vm-row"><span class="lbl">RAM</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-g-mem-bar" style="width:0%;background:var(--blue)"></div></div>
<span class="val c-blue" id="vm-g-mem"></span></div>
<div class="vm-row"><span class="lbl">Disque</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-g-disk-bar" style="width:0%;background:var(--purple)"></div></div>
<span class="val c-purple" id="vm-g-disk"></span></div>
<div class="vm-row"><span class="lbl">Uptime</span><span class="val c-green" id="vm-g-uptime"></span></div>
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
<span class="coll-lbl">Containers Docker</span><span class="coll-arrow open"></span>
</div>
<div class="coll-body open"><div class="coll-inner">
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">pdffetch</span><span class="ct-img">python:alpine</span></div>
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">adguardhome</span><span class="ct-img">adguard/adguardhome</span></div>
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">wg-easy</span><span class="ct-img">ghcr.io/wg-easy</span></div>
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">node-exporter</span><span class="ct-img">prom/node-exporter</span></div>
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">portainer_agent</span><span class="ct-img">portainer/agent</span></div>
</div></div>
</div>
</div>
<!-- Vigile -->
<div class="vm-card">
<div class="vm-header">
<div class="vm-dot up" id="vm-vigile-dot"></div>
<div><div class="vm-name">Vigile</div><div style="font-size:.65rem;color:var(--muted)">AMD 1c / 1 Go — Sentinelle</div></div>
<div class="vm-ip">79.72.30.231</div>
</div>
<div class="vm-body">
<div class="vm-row"><span class="lbl">CPU</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-v-cpu-bar" style="width:0%"></div></div>
<span class="val" id="vm-v-cpu"></span></div>
<div class="vm-row"><span class="lbl">RAM</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-v-mem-bar" style="width:0%;background:var(--blue)"></div></div>
<span class="val c-blue" id="vm-v-mem"></span></div>
<div class="vm-row"><span class="lbl">Disque</span>
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-v-disk-bar" style="width:0%;background:var(--purple)"></div></div>
<span class="val c-purple" id="vm-v-disk"></span></div>
<div class="vm-row"><span class="lbl">Uptime</span><span class="val c-green" id="vm-v-uptime"></span></div>
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
<span class="coll-lbl">Containers Docker</span><span class="coll-arrow open"></span>
</div>
<div class="coll-body open"><div class="coll-inner">
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">fallout_db</span><span class="ct-img">postgres:16-alpine</span></div>
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">node-exporter</span><span class="ct-img">prom/node-exporter</span></div>
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">portainer_agent</span><span class="ct-img">portainer/agent</span></div>
</div></div>
</div>
</div>
</div>
</div>
<!-- ── RÉSUMÉ 30j ── -->
<div>
<div class="section-title">Totaux infrastructure — 30 derniers jours</div>
<div class="summary-card">
<div class="sum-item"><div class="n c-blue" id="sum-rx"></div><div class="l">Entrant ↓</div></div>
<div class="sum-item sum-divider"><div class="n c-purple" id="sum-tx"></div><div class="l">Sortant ↑</div></div>
<div class="sum-item sum-divider"><div class="n" id="sum-total" style="color:#c4b5fd"></div><div class="l">Total 3 VMs</div></div>
</div>
<div style="margin-top:.4rem;font-size:.68rem;color:var(--muted);text-align:right">Source : Prometheus PromQL</div>
</div>
<!-- ── ADMINSYS ── -->
<div>
<div class="section-title">AdminSys</div>
<div class="admin-grid">
<!-- Reboot VMs -->
<div class="admin-card">
<div class="admin-card-header">
<span style="font-size:1.1rem">🔄</span>
<span class="a-title">Redémarrage VMs</span>
</div>
<div class="admin-card-body" style="padding:1rem 1.2rem">
<div style="font-size:.72rem;color:var(--muted);margin-bottom:.6rem">⚠️ Double-cliquez pour confirmer le reboot</div>
<div class="reboot-row">
<button class="btn-reboot" ondblclick="rebootVM('ampere',this)">⚡ Ampère</button>
<button class="btn-reboot" ondblclick="rebootVM('guardian',this)">⚡ Guardian</button>
<button class="btn-reboot" ondblclick="rebootVM('vigile',this)">⚡ Vigile</button>
</div>
</div>
</div>
<!-- Container groups -->
<div class="admin-card" style="grid-column:span 2">
<div class="admin-card-header">
<span style="font-size:1.1rem">🐋</span>
<span class="a-title">Containers Ampère — Groupes</span>
<span style="font-size:.68rem;color:var(--muted)">Cliquer sur un groupe pour déployer</span>
</div>
<div class="admin-card-body" id="groups-body">Chargement...</div>
</div>
</div>
</div>
<!-- ── SITES ── -->
<div>
<div class="section-title">Accès rapides — Services CoyoteOS</div>
<div class="sites-grid" id="sites-grid"></div>
</div>
</main>
<!-- Toast -->
<div id="toast" class="toast" style="display:none"></div>
<script>
// ── Helpers ──────────────────────────────────────────────────────────────
function fmt(b){if(!b)return"N/A";if(b>=1e12)return(b/1e12).toFixed(2)+" To";if(b>=1e9)return(b/1e9).toFixed(1)+" Go";if(b>=1e6)return(b/1e6).toFixed(1)+" Mo";if(b>=1e3)return(b/1e3).toFixed(0)+" Ko";return b+" o"}
function fmtSpd(b){if(b>=1e6)return(b/1e6).toFixed(1)+" Mo/s";if(b>=1e3)return(b/1e3).toFixed(0)+" Ko/s";return b+" o/s"}
function pctColor(p){if(p>=90)return"var(--red)";if(p>=70)return"var(--orange)";return"var(--green)"}
function setBar(id,pct,color){const el=document.getElementById(id);if(el){el.style.width=pct+"%";if(color)el.style.background=color}}
function setText(id,val){const el=document.getElementById(id);if(el)el.textContent=val}
function toast(msg, color="var(--green)"){
const t=document.getElementById("toast");
t.textContent=msg; t.style.color=color; t.style.display="block";
setTimeout(()=>t.style.display="none",3000);
}
function toggleColl(body){
const open=body.classList.toggle("open");
const arrow=body.previousElementSibling?.querySelector(".coll-arrow");
if(arrow)arrow.classList.toggle("open",open);
}
// ── Metrics Ampère ───────────────────────────────────────────────────────
async function refreshMetrics(){
try{
const d=await fetch("/metrics").then(r=>r.json());
const cpu=d.cpu.percent; const mem=d.memory.percent; const dsk=d.disk.percent;
setText("vm-a-cpu",cpu+"%"); document.getElementById("vm-a-cpu").style.color=pctColor(cpu);
setBar("vm-a-cpu-bar",cpu,pctColor(cpu));
setText("vm-a-mem",mem+"%"); setBar("vm-a-mem-bar",mem,null);
setText("vm-a-disk",dsk+"%"); document.getElementById("vm-a-disk").style.color=pctColor(dsk);
setBar("vm-a-disk-bar",dsk,pctColor(dsk));
const u=d.uptime; setText("vm-a-uptime",u.days+"j "+u.hours+"h "+u.minutes+"m");
const net=d.network; setText("vm-a-net","↓"+fmtSpd(net.rx_bps)+" ↑"+fmtSpd(net.tx_bps));
setText("last-update","Mis à jour "+new Date().toLocaleTimeString("fr-FR"));
}catch(e){document.getElementById("vm-ampere-dot").className="vm-dot down"}
}
// ── Métriques Guardian/Vigile via Prometheus ────────────────────────────
async function refreshVmMetrics(instance, prefix){
try{
const d=await fetch(`/vm-metrics?instance=${instance}`).then(r=>r.json());
if(!d.available) return;
const cpu=d.cpu||0; const mem=d.mem_pct||0; const dsk=d.disk_pct||0;
setText(`vm-${prefix}-cpu`,cpu+"%"); document.getElementById(`vm-${prefix}-cpu`).style.color=pctColor(cpu);
setBar(`vm-${prefix}-cpu-bar`,cpu,pctColor(cpu));
setText(`vm-${prefix}-mem`,mem+"%"); setBar(`vm-${prefix}-mem-bar`,mem,null);
setText(`vm-${prefix}-disk`,dsk+"%"); document.getElementById(`vm-${prefix}-disk`).style.color=pctColor(dsk);
setBar(`vm-${prefix}-disk-bar`,dsk,pctColor(dsk));
if(d.uptime_human) setText(`vm-${prefix}-uptime`,d.uptime_human);
document.getElementById(`vm-${prefix}-dot`).className="vm-dot up";
}catch(e){}
}
// ── Ollama ───────────────────────────────────────────────────────────────
async function refreshOllama(){
try{
const models=await fetch("/ollama-ps").then(r=>r.json());
const el=document.getElementById("vm-a-ollama");
if(!models.length){el.innerHTML='<div style="font-size:.7rem;color:var(--muted)">Aucun modèle chargé</div>';return}
el.innerHTML=models.map(m=>`<div class="ollama-model">
<span class="m-name">${m.name}</span>
<span class="m-size">${m.size_gb}Go</span>
</div>`).join("");
}catch(e){}
}
// ── Docker Ampère ────────────────────────────────────────────────────────
async function refreshDocker(){
try{
const cs=await fetch("/docker-ps").then(r=>r.json());
const el=document.getElementById("vm-a-containers");
if(!cs.length){el.innerHTML='<div class="ct-row" style="color:var(--muted)">Aucun container</div>';return}
el.innerHTML=cs.map(c=>{
const up=c.state==="running";
return`<div class="ct-row"><div class="ct-dot ${up?"up":"down"}"></div><span class="ct-name" style="color:${up?"var(--text)":"var(--muted)"}">${c.name}</span><span class="ct-img">${c.image}</span></div>`;
}).join("");
}catch(e){}
}
// ── Pipelines ────────────────────────────────────────────────────────────
function logLine(l){
l=l.trim(); if(!l)return"";
if(l.match(/OK \(\d/))return`<div class="log-ok">${l}</div>`;
if(l.match(/ERREUR|Error/i))return`<div class="log-err">${l}</div>`;
if(l.match(/^\[.+\.pdf\]/))return`<div class="log-cur">${l}</div>`;
if(l.match(/chunk \d+\/\d+/))return`<div class="log-chunk">${l}</div>`;
if(l.match(/Ko|chars|chunks|trouvés|Mo|Download/))return`<div class="log-dl">${l}</div>`;
return`<div class="log-info">${l}</div>`;
}
async function refreshPipeline(){
try{
const d=await fetch("/pipeline").then(r=>r.json());
const done=d.done||0; const pct=Math.round((done/77)*100);
setText("p-done",done); setText("p-failed",d.failed||0);
setText("p-speed",d.avg_chunk_s?d.avg_chunk_s+"s":"—");
setText("p-pct",pct+"%"); setBar("p-bar",pct,null);
const badge=document.getElementById("pipeline-badge");
if(d.running&&d.current_pdf){badge.className="badge badge-running";badge.textContent="● En cours"}
else if(d.running){badge.className="badge badge-running";badge.textContent="● Actif"}
else if(done>0){badge.className="badge badge-idle";badge.textContent="⏸ Cron 2h00"}
else{badge.className="badge badge-idle";badge.textContent="● En attente"}
if(d.running&&d.current_pdf){
setText("p-name",d.current_pdf);
setText("p-eta",(d.eta_current_pdf?"ETA: "+d.eta_current_pdf:"")+(d.last_chunk_s?" · "+d.last_chunk_s+"s/chunk":""));
if(d.current_chunk&&d.total_chunks){
const cp=Math.round(d.current_chunk/d.total_chunks*100);
setBar("p-chunk-bar",cp,null); setText("p-chunk-label","chunk "+d.current_chunk+"/"+d.total_chunks);
}
}else{setText("p-name",done>0?"En pause — cron 2h00":"Pipeline en attente...");setText("p-eta","")}
if(d.log_tail?.length){const b=document.getElementById("log-box");b.innerHTML=d.log_tail.map(logLine).join("");b.scrollTop=b.scrollHeight}
}catch(e){}
}
async function refreshFallout(){
try{
const d=await fetch("/fallout-ingest").then(r=>r.json());
const done=d.done||0; const pct=Math.round((done/17)*100);
setText("f-done",done); setText("f-failed",d.failed||0);
setText("f-speed",d.avg_embed_s?d.avg_embed_s+"s":"—");
setText("f-pct",pct+"%"); setBar("f-bar",pct,null);
const badge=document.getElementById("fallout-badge");
if(d.running&&d.current_pdf){badge.className="badge badge-running";badge.textContent="● En cours"}
else if(d.running){badge.className="badge badge-running";badge.textContent="● Actif"}
else if(done===17){badge.className="badge badge-done";badge.textContent="✓ 17/17 indexés"}
else if(done>0){badge.className="badge badge-done";badge.textContent="✓ "+done+"/17"}
else{badge.className="badge badge-idle";badge.textContent="● En attente"}
if(d.running&&d.current_pdf){
setText("f-name",d.current_pdf);
if(d.current_chunk&&d.total_chunks){
const cp=Math.round(d.current_chunk/d.total_chunks*100);
setBar("f-chunk-bar",cp,null); setText("f-chunk-label","chunk "+d.current_chunk+"/"+d.total_chunks);
}
}else{setText("f-name",done===17?"✓ Indexation terminée":(done>0?done+"/17 PDFs indexés":"En attente..."))}
if(d.log_tail?.length){const b=document.getElementById("f-log-box");b.innerHTML=d.log_tail.map(logLine).join("");b.scrollTop=b.scrollHeight}
}catch(e){}
}
// ── Summary ──────────────────────────────────────────────────────────────
async function refreshSummary(){
try{
const d=await fetch("/summary30").then(r=>r.json());
setText("sum-rx",d.rx_30d?fmt(d.rx_30d):"via Grafana");
setText("sum-tx",d.tx_30d?fmt(d.tx_30d):"via Grafana");
setText("sum-total",d.total_bw?fmt(d.total_bw):"via Grafana");
}catch(e){}
}
// ── Sites ─────────────────────────────────────────────────────────────────
const ICONS={globe:"🌐",book:"📚",git:"🐙",chart:"📊",music:"🎵",studio:"🎹",dash:"🖥️"};
async function refreshSites(){
try{
const sites=await fetch("/sites").then(r=>r.json());
document.getElementById("sites-grid").innerHTML=sites.map(s=>{
const dot=s.up?"sdot-up":"sdot-down";
const lbl=s.up?"En ligne":(s.status?"Erreur "+s.status:"Hors ligne");
return`<a class="site-btn" href="${s.url}" target="_blank">
<div class="site-icon">${ICONS[s.icon]||"🔗"}</div>
<div class="site-name">${s.name}</div>
<div class="site-status"><div class="sdot ${dot}"></div><span style="color:${s.up?"var(--green)":"var(--red)"}">${lbl}</span></div>
</a>`;
}).join("");
}catch(e){}
}
// ── AdminSys — Groups ─────────────────────────────────────────────────────
let allContainers = [];
let groups = [];
async function loadGroups(){
try{
[groups, allContainers] = await Promise.all([
fetch("/groups").then(r=>r.json()),
fetch("/docker-ps").then(r=>r.json())
]);
renderGroups();
}catch(e){}
}
function getContainerState(name){
const c=allContainers.find(x=>x.name===name);
return c?c.state:"unknown";
}
function renderGroups(){
const body=document.getElementById("groups-body");
body.innerHTML=groups.map(g=>{
const ctHtml=g.containers.map(name=>{
const state=getContainerState(name);
const up=state==="running"; const unknown=state==="unknown";
const dot=unknown?"":up?'<div class="ct-dot up"></div>':'<div class="ct-dot down"></div>';
const btns=g.protected?"":`
<button class="btn-ct ${up?"stop":"start"}" onclick="ctAction('ampere','${name}','${up?"stop":"start"}',this)">
${up?"■ Stop":"▶ Start"}
</button>
${up?`<button class="btn-ct restart" onclick="ctAction('ampere','${name}','restart',this)">↺</button>`:""}
`;
return`<div class="ct-control-row">${dot}<span class="ct-control-name" style="color:${unknown?"var(--muted)":up?"var(--text)":"var(--red)"}">${name}</span>${btns}</div>`;
}).join("");
const runCount=g.containers.filter(n=>getContainerState(n)==="running").length;
const allUp=runCount===g.containers.filter(n=>getContainerState(n)!=="unknown").length;
const groupBtn=g.protected?"":
`<button class="btn-group-toggle" onclick="groupAction('${g.id}','${allUp?"stop":"start"}',this)">${allUp?"■ Stop tout":"▶ Start tout"}</button>`;
return`<div class="group-card">
<div class="group-hdr" onclick="toggleGrp(this)">
<span class="group-icon">${g.icon}</span>
<span class="group-name">${g.name}</span>
<span class="group-badge">${runCount}/${g.containers.length} up</span>
${groupBtn}
<span style="color:var(--muted);font-size:.7rem;margin-left:.3rem">▼</span>
</div>
<div class="group-containers" style="display:none">${ctHtml}</div>
</div>`;
}).join("");
}
function toggleGrp(hdr){
const body=hdr.nextElementSibling;
body.style.display=body.style.display==="none"?"flex":"none";
body.style.flexDirection="column";
}
async function ctAction(vm, name, action, btn){
btn.disabled=true; btn.textContent="...";
try{
const r=await fetch("/admin/docker",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({vm,container:name,action})}).then(r=>r.json());
toast(r.ok?`${name} ${action}`:`✗ Erreur ${name}`,r.ok?"var(--green)":"var(--red)");
await new Promise(r=>setTimeout(r,800));
allContainers=await fetch("/docker-ps").then(r=>r.json());
renderGroups();
}catch(e){toast("Erreur réseau","var(--red)");btn.disabled=false}
}
async function groupAction(groupId, action, btn){
btn.disabled=true; btn.textContent="...";
const g=groups.find(x=>x.id===groupId);
if(!g){btn.disabled=false;return}
const targets=action==="stop"?g.containers.filter(n=>getContainerState(n)==="running")
:g.containers.filter(n=>getContainerState(n)!=="running");
for(const name of targets){
await fetch("/admin/docker",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({vm:"ampere",container:name,action})});
}
toast(`✓ Groupe ${g.name}${action}`,"var(--green)");
allContainers=await fetch("/docker-ps").then(r=>r.json());
renderGroups();
}
async function rebootVM(vm, btn){
btn.textContent="Rebooting..."; btn.disabled=true;
try{
const r=await fetch("/admin/reboot",{method:"POST",headers:{"Content-Type":"application/json"},
body:JSON.stringify({vm})}).then(r=>r.json());
toast(r.ok?`${vm} reboot lancé`:`✗ Erreur reboot ${vm}`,r.ok?"var(--orange)":"var(--red)");
}catch(e){toast("Erreur réseau","var(--red)")}
setTimeout(()=>{btn.disabled=false;btn.textContent=`${vm.charAt(0).toUpperCase()+vm.slice(1)}`;},5000);
}
// ── Init ─────────────────────────────────────────────────────────────────
refreshMetrics();
refreshVmMetrics("Guardian","g");
refreshVmMetrics("Vigile","v");
refreshOllama();
refreshDocker();
refreshPipeline();
refreshFallout();
refreshSummary();
refreshSites();
loadGroups();
setInterval(refreshMetrics, 5000);
setInterval(()=>refreshVmMetrics("Guardian","g"), 15000);
setInterval(()=>refreshVmMetrics("Vigile","v"), 15000);
setInterval(refreshOllama, 10000);
setInterval(refreshDocker, 15000);
setInterval(refreshPipeline, 8000);
setInterval(refreshFallout, 8000);
setInterval(refreshSummary, 60000);
setInterval(refreshSites, 30000);
setInterval(loadGroups, 20000);
</script>
</body>
</html>
+308
View File
@@ -0,0 +1,308 @@
"""
Ingestion Fallout PDFs → ChromaDB v2
Améliorations v2 :
- Chunking structurel (paragraphes/sections) au lieu de token-count fixe
- Métadonnées enrichies : systeme (2d20 | tardy | none), category, priority
- Évite la collision mécanique 2D20 vs Tardy en taggant chaque chunk
- Chunks 400-600 mots, overlap 80 mots, respecte les frontières de paragraphes
"""
import json, os, re, subprocess, tempfile, time, urllib.request, urllib.parse
from pathlib import Path
PDFFETCH_URL = "http://localhost:15001/files"
OLLAMA_EMBED = "http://localhost:11434/api/embeddings"
CHROMA_URL = "http://localhost:8800"
CHROMA_BASE = "/api/v2/tenants/default_tenant/databases/default_database"
COLLECTION = "fallout_lore"
EMBED_MODEL = "nomic-embed-text"
STATE_FILE = Path("/home/ubuntu/fallout/ingest_state.json")
FALLOUT_PREFIX = "fallout/"
# Chunking params (en mots, pas tokens)
CHUNK_WORDS = 450
OVERLAP_WORDS = 80
MIN_WORDS = 40
# ── État ──────────────────────────────────────────────────────────────────────
def load_state():
if STATE_FILE.exists():
return json.loads(STATE_FILE.read_text())
return {"ingested": [], "failed": [], "version": 2}
def save_state(s):
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
STATE_FILE.write_text(json.dumps(s, indent=2))
# ── ChromaDB ──────────────────────────────────────────────────────────────────
def chroma_request(method, path, data=None):
url = CHROMA_URL + path
body = json.dumps(data).encode() if data else None
req = urllib.request.Request(url, data=body, method=method,
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=30) as r:
return json.loads(r.read())
def get_or_create_collection():
try:
r = chroma_request("POST", f"{CHROMA_BASE}/collections", {
"name": COLLECTION,
"metadata": {"hnsw:space": "cosine"}
})
print(f"Collection créée : {COLLECTION} ({r['id']})")
return r["id"]
except urllib.error.HTTPError as e:
if e.code == 409:
cols = chroma_request("GET", f"{CHROMA_BASE}/collections")
col_id = next(c["id"] for c in cols if c["name"] == COLLECTION)
print(f"Collection existante : {COLLECTION} ({col_id})")
return col_id
raise
def add_documents(col_id, ids, embeddings, documents, metadatas):
chroma_request("POST", f"{CHROMA_BASE}/collections/{col_id}/add", {
"ids": ids, "embeddings": embeddings,
"documents": documents, "metadatas": metadatas
})
# ── Embeddings ────────────────────────────────────────────────────────────────
def embed(text):
payload = json.dumps({"model": EMBED_MODEL, "prompt": text}).encode()
req = urllib.request.Request(OLLAMA_EMBED, data=payload,
headers={"Content-Type": "application/json"}, method="POST")
with urllib.request.urlopen(req, timeout=60) as r:
return json.loads(r.read())["embedding"]
# ── PDF helpers ───────────────────────────────────────────────────────────────
def list_fallout_pdfs():
req = urllib.request.Request(PDFFETCH_URL,
headers={"User-Agent": "fallout-ingest/2.0"})
with urllib.request.urlopen(req, timeout=15) as r:
all_files = json.loads(r.read())
return [f for f in all_files if f.startswith(FALLOUT_PREFIX)]
def download_pdf(filename):
url = f"{PDFFETCH_URL}/{urllib.parse.quote(filename)}"
req = urllib.request.Request(url, headers={"User-Agent": "fallout-ingest/2.0"})
with urllib.request.urlopen(req, timeout=300) as r:
return r.read()
def extract_text(pdf_bytes):
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
f.write(pdf_bytes); tmp = f.name
text = ""
try:
r = subprocess.run(["pdftotext", "-layout", tmp, "-"],
capture_output=True, timeout=180)
if r.returncode == 0:
text = r.stdout.decode("utf-8", errors="replace")
finally:
os.unlink(tmp)
if len(text.strip()) < 200:
try:
import io
from pypdf import PdfReader
reader = PdfReader(io.BytesIO(pdf_bytes))
text = "\n".join(p.extract_text() or "" for p in reader.pages)
except Exception as e:
raise RuntimeError(f"Extraction impossible: {e}")
return text
# ── Chunking structurel ───────────────────────────────────────────────────────
def is_section_header(line):
"""Détecte les titres de section : ligne courte, capitalisée, pas de ponctuation finale."""
l = line.strip()
if not l or len(l) > 80:
return False
if l.isupper() and len(l.split()) <= 8:
return True
if re.match(r'^(CHAPITRE|CHAPTER|PARTIE|PART|SECTION|\d+[\.\)])\s+\S', l, re.I):
return True
return False
def chunk_by_structure(text, filename):
"""
Chunking basé sur les paragraphes et sections, pas sur le token count fixe.
Respecte les frontières naturelles du texte.
"""
# Nettoyage
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r'\n{3,}', '\n\n', text)
# Découpe en blocs (paragraphes ou sections)
raw_blocks = re.split(r'\n{2,}', text)
# Regroupe les headers avec le paragraphe suivant
blocks = []
i = 0
while i < len(raw_blocks):
b = raw_blocks[i].strip()
if not b:
i += 1
continue
if is_section_header(b) and i + 1 < len(raw_blocks):
merged = b + "\n\n" + raw_blocks[i+1].strip()
blocks.append(merged)
i += 2
else:
blocks.append(b)
i += 1
# Accumulation en chunks de ~CHUNK_WORDS mots avec overlap
chunks = []
current_blocks = []
current_words = 0
for block in blocks:
bwords = len(block.split())
if bwords < 5:
continue
if current_words + bwords > CHUNK_WORDS and current_blocks:
chunk = "\n\n".join(current_blocks).strip()
if len(chunk.split()) >= MIN_WORDS:
chunks.append(chunk)
# Overlap : garder les derniers blocs jusqu'à OVERLAP_WORDS mots
overlap_blocks = []
overlap_words = 0
for ob in reversed(current_blocks):
ow = len(ob.split())
if overlap_words + ow <= OVERLAP_WORDS:
overlap_blocks.insert(0, ob)
overlap_words += ow
else:
break
current_blocks = overlap_blocks + [block]
current_words = overlap_words + bwords
else:
current_blocks.append(block)
current_words += bwords
# Flush final
if current_blocks:
chunk = "\n\n".join(current_blocks).strip()
if len(chunk.split()) >= MIN_WORDS:
chunks.append(chunk)
return chunks
# ── Métadonnées ───────────────────────────────────────────────────────────────
def infer_metadata(filename):
f = filename.lower()
name = Path(filename).name
meta = {"source": name, "path": filename}
# Catégorie
if "lore_post" in f: meta["category"] = "lore_canon"
elif "lore_pre" in f: meta["category"] = "lore_inspiration"
elif "regles/core" in f: meta["category"] = "regles_core"
elif "regles/suppl" in f: meta["category"] = "regles_supplement"
elif "aventures" in f: meta["category"] = "aventure"
elif "ambiance" in f: meta["category"] = "ambiance"
else: meta["category"] = "divers"
# Système de règles — CRITIQUE pour éviter la collision 2D20 vs Tardy
if "tardy" in f:
meta["systeme"] = "tardy"
elif any(k in f for k in ["manuel_officiel", "quickstart", "rules_booklet",
"gm_toolkit", "bestiaire", "armes", "special",
"competences", "pizza", "frequence"]):
meta["systeme"] = "2d20"
elif "regles" in f:
meta["systeme"] = "2d20"
else:
meta["systeme"] = "none"
# Priorité lore : la bible canon prime
if "bible_v2" in f: meta["priority"] = "haute"
elif "bible_v1" in f: meta["priority"] = "normale"
elif "lore_canon" == meta.get("category"): meta["priority"] = "normale"
else: meta["priority"] = "basse"
return meta
# ── Main ──────────────────────────────────────────────────────────────────────
def main():
print("=== Ingestion Fallout v2 → ChromaDB ===")
print(f"Chunking structurel, ~{CHUNK_WORDS} mots/chunk, overlap {OVERLAP_WORDS} mots")
print(f"Modèle embedding : {EMBED_MODEL}\n")
state = load_state()
col_id = get_or_create_collection()
pdfs = list_fallout_pdfs()
print(f"{len(pdfs)} PDFs Fallout sur Guardian\n")
for filename in pdfs:
if filename in state["ingested"]:
print(f"[SKIP] {filename}")
continue
print(f"\n[{filename}]")
try:
pdf_bytes = download_pdf(filename)
print(f" {len(pdf_bytes)/1024:.0f} Ko téléchargés")
text = extract_text(pdf_bytes)
print(f" {len(text)} chars extraits")
chunks = chunk_by_structure(text, filename)
meta_base = infer_metadata(filename)
print(f" {len(chunks)} chunks | catégorie: {meta_base['category']} | système: {meta_base['systeme']}")
ids, embeddings, documents, metadatas = [], [], [], []
for i, chunk in enumerate(chunks):
chunk_id = f"{meta_base['source']}_{i:04d}"
t0 = time.time()
emb = embed(chunk)
word_count = len(chunk.split())
ids.append(chunk_id)
embeddings.append(emb)
documents.append(chunk)
metadatas.append({
**meta_base,
"chunk_index": i,
"total_chunks": len(chunks),
"word_count": word_count,
})
print(f" chunk {i+1}/{len(chunks)} ({word_count}w) OK ({time.time()-t0:.1f}s)")
if len(ids) >= 50:
add_documents(col_id, ids, embeddings, documents, metadatas)
ids, embeddings, documents, metadatas = [], [], [], []
if ids:
add_documents(col_id, ids, embeddings, documents, metadatas)
print(f"\n Indexé — {len(chunks)} chunks")
state["ingested"].append(filename)
# Retire des failed si présent
state["failed"] = [f for f in state["failed"] if f.get("file") != filename]
save_state(state)
except Exception as e:
print(f" ERREUR: {e}")
state["failed"].append({"file": filename, "error": str(e)})
save_state(state)
print(f"\n=== Terminé : {len(state['ingested'])} PDFs indexés, {len(state['failed'])} échecs ===")
for f in state.get("failed", []):
print(f" ECHEC: {f['file']}{f['error']}")
if __name__ == "__main__":
main()
+363
View File
@@ -0,0 +1,363 @@
"""
Ingestion MAO PDFs → ChromaDB
Collection : mao_docs
Embedding : nomic-embed-text (Ollama local)
Chunking structurel : paragraphes + sections, ~450 mots, overlap 80 mots
Double usage :
1. RAG assistant — répond aux questions sur les VSTs/plugins
2. Accélération wiki — fournit les chunks pertinents au LLM pour générer les pages wiki
(à la place de lire le PDF entier)
"""
import json, os, re, subprocess, tempfile, time, urllib.request, urllib.parse
from pathlib import Path
PDFFETCH_URL = "http://localhost:15001/files"
OLLAMA_EMBED = "http://localhost:11434/api/embeddings"
CHROMA_URL = "http://localhost:8800"
CHROMA_BASE = "/api/v2/tenants/default_tenant/databases/default_database"
COLLECTION = "mao_docs"
EMBED_MODEL = "nomic-embed-text"
STATE_FILE = Path("/home/ubuntu/mao/ingest_state.json")
CHUNK_WORDS = 450
OVERLAP_WORDS = 80
MIN_WORDS = 40
# Correspondances nom de fichier → métadonnées produit
# Enrichit la recherche : "comment fonctionne la reverb de Serum ?" → filtre type=synth, product=Serum
PRODUCT_HINTS = {
"serum": {"product": "Serum", "manufacturer": "Xfer", "type": "synth"},
"massive": {"product": "Massive", "manufacturer": "Native Instruments", "type": "synth"},
"vital": {"product": "Vital", "manufacturer": "Matt Tytel", "type": "synth"},
"surge": {"product": "Surge XT", "manufacturer": "Surge Synth", "type": "synth"},
"kontakt": {"product": "Kontakt", "manufacturer": "Native Instruments", "type": "sampler"},
"battery": {"product": "Battery", "manufacturer": "Native Instruments", "type": "sampler"},
"ableton": {"product": "Ableton Live", "manufacturer": "Ableton", "type": "daw"},
"live": {"product": "Ableton Live", "manufacturer": "Ableton", "type": "daw"},
"fl_studio": {"product": "FL Studio", "manufacturer": "Image-Line", "type": "daw"},
"fl studio": {"product": "FL Studio", "manufacturer": "Image-Line", "type": "daw"},
"logic": {"product": "Logic Pro", "manufacturer": "Apple", "type": "daw"},
"cubase": {"product": "Cubase", "manufacturer": "Steinberg", "type": "daw"},
"pro_tools": {"product": "Pro Tools", "manufacturer": "Avid", "type": "daw"},
"reaper": {"product": "Reaper", "manufacturer": "Cockos", "type": "daw"},
"izotope": {"product": "iZotope", "manufacturer": "iZotope", "type": "mastering"},
"ozone": {"product": "Ozone", "manufacturer": "iZotope", "type": "mastering"},
"neutron": {"product": "Neutron", "manufacturer": "iZotope", "type": "mixing"},
"fabfilter": {"product": "FabFilter", "manufacturer": "FabFilter", "type": "effects"},
"pro-q": {"product": "Pro-Q", "manufacturer": "FabFilter", "type": "eq"},
"pro-r": {"product": "Pro-R", "manufacturer": "FabFilter", "type": "reverb"},
"pro-c": {"product": "Pro-C", "manufacturer": "FabFilter", "type": "compressor"},
"valhalla": {"product": "Valhalla", "manufacturer": "Valhalla DSP", "type": "reverb"},
"waves": {"product": "Waves", "manufacturer": "Waves", "type": "effects"},
"u-he": {"product": "u-he", "manufacturer": "u-he", "type": "synth"},
"diva": {"product": "Diva", "manufacturer": "u-he", "type": "synth"},
"hive": {"product": "Hive", "manufacturer": "u-he", "type": "synth"},
"repro": {"product": "Repro", "manufacturer": "u-he", "type": "synth"},
"zebra": {"product": "Zebra", "manufacturer": "u-he", "type": "synth"},
"spire": {"product": "Spire", "manufacturer": "Reveal Sound", "type": "synth"},
"sylenth": {"product": "Sylenth1", "manufacturer": "LennarDigital","type": "synth"},
"omnisphere": {"product": "Omnisphere", "manufacturer": "Spectrasonics","type": "synth"},
"nexus": {"product": "Nexus", "manufacturer": "refx", "type": "rompler"},
"reaktor": {"product": "Reaktor", "manufacturer": "Native Instruments", "type": "modular"},
"max_msp": {"product": "Max/MSP", "manufacturer": "Cycling '74", "type": "modular"},
"soundtoys": {"product": "SoundToys", "manufacturer": "SoundToys", "type": "effects"},
"rc-20": {"product": "RC-20", "manufacturer": "XLN Audio", "type": "effects"},
"decimort": {"product": "Decimort", "manufacturer": "D16 Group", "type": "effects"},
"d16": {"product": "D16 Group", "manufacturer": "D16 Group", "type": "effects"},
}
# ── État ──────────────────────────────────────────────────────────────────────
def load_state():
if STATE_FILE.exists():
return json.loads(STATE_FILE.read_text())
return {"ingested": [], "failed": []}
def save_state(s):
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
STATE_FILE.write_text(json.dumps(s, indent=2))
# ── ChromaDB ──────────────────────────────────────────────────────────────────
def chroma_request(method, path, data=None):
url = CHROMA_URL + path
body = json.dumps(data).encode() if data else None
req = urllib.request.Request(url, data=body, method=method,
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=30) as r:
return json.loads(r.read())
def get_or_create_collection():
try:
r = chroma_request("POST", f"{CHROMA_BASE}/collections", {
"name": COLLECTION,
"metadata": {"hnsw:space": "cosine"}
})
print(f"Collection créée : {COLLECTION} ({r['id']})")
return r["id"]
except urllib.error.HTTPError as e:
if e.code == 409:
cols = chroma_request("GET", f"{CHROMA_BASE}/collections")
col_id = next(c["id"] for c in cols if c["name"] == COLLECTION)
print(f"Collection existante : {COLLECTION} ({col_id})")
return col_id
raise
def add_documents(col_id, ids, embeddings, documents, metadatas):
chroma_request("POST", f"{CHROMA_BASE}/collections/{col_id}/add", {
"ids": ids, "embeddings": embeddings,
"documents": documents, "metadatas": metadatas
})
# ── Embeddings ────────────────────────────────────────────────────────────────
def embed(text):
payload = json.dumps({"model": EMBED_MODEL, "prompt": text}).encode()
req = urllib.request.Request(OLLAMA_EMBED, data=payload,
headers={"Content-Type": "application/json"}, method="POST")
with urllib.request.urlopen(req, timeout=60) as r:
return json.loads(r.read())["embedding"]
# ── PDF helpers ───────────────────────────────────────────────────────────────
def list_mao_pdfs():
req = urllib.request.Request(PDFFETCH_URL,
headers={"User-Agent": "mao-ingest/1.0"})
with urllib.request.urlopen(req, timeout=15) as r:
all_files = json.loads(r.read())
# PDFs MAO = racine (pas de sous-dossier "fallout/")
return [f for f in all_files if not f.startswith("fallout/") and f.endswith(".pdf")]
def download_pdf(filename):
url = f"{PDFFETCH_URL}/{urllib.parse.quote(filename)}"
req = urllib.request.Request(url, headers={"User-Agent": "mao-ingest/1.0"})
with urllib.request.urlopen(req, timeout=300) as r:
return r.read()
def extract_text(pdf_bytes):
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
f.write(pdf_bytes); tmp = f.name
text = ""
try:
r = subprocess.run(["pdftotext", "-layout", tmp, "-"],
capture_output=True, timeout=180)
if r.returncode == 0:
text = r.stdout.decode("utf-8", errors="replace")
finally:
os.unlink(tmp)
if len(text.strip()) < 200:
try:
import io
from pypdf import PdfReader
reader = PdfReader(io.BytesIO(pdf_bytes))
text = "\n".join(p.extract_text() or "" for p in reader.pages)
except Exception as e:
raise RuntimeError(f"Extraction impossible: {e}")
return text
# ── Chunking structurel ───────────────────────────────────────────────────────
def is_section_header(line):
l = line.strip()
if not l or len(l) > 100:
return False
if l.isupper() and 2 <= len(l.split()) <= 10:
return True
if re.match(r'^(Chapter|Section|Part|Chapitre|Partie|\d+[\.\)])\s+\S', l, re.I):
return True
return False
def chunk_by_structure(text):
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r'\n{3,}', '\n\n', text)
raw_blocks = re.split(r'\n{2,}', text)
blocks = []
i = 0
while i < len(raw_blocks):
b = raw_blocks[i].strip()
if not b:
i += 1; continue
if is_section_header(b) and i + 1 < len(raw_blocks):
merged = b + "\n\n" + raw_blocks[i+1].strip()
blocks.append(merged)
i += 2
else:
blocks.append(b)
i += 1
chunks = []
current_blocks = []
current_words = 0
for block in blocks:
bwords = len(block.split())
if bwords < 5:
continue
if current_words + bwords > CHUNK_WORDS and current_blocks:
chunk = "\n\n".join(current_blocks).strip()
if len(chunk.split()) >= MIN_WORDS:
chunks.append(chunk)
overlap_blocks = []
overlap_words = 0
for ob in reversed(current_blocks):
ow = len(ob.split())
if overlap_words + ow <= OVERLAP_WORDS:
overlap_blocks.insert(0, ob)
overlap_words += ow
else:
break
current_blocks = overlap_blocks + [block]
current_words = overlap_words + bwords
else:
current_blocks.append(block)
current_words += bwords
if current_blocks:
chunk = "\n\n".join(current_blocks).strip()
if len(chunk.split()) >= MIN_WORDS:
chunks.append(chunk)
return chunks
# ── Métadonnées ───────────────────────────────────────────────────────────────
def infer_metadata(filename):
f = filename.lower()
name = Path(filename).name
stem = Path(filename).stem.lower().replace("-", "_").replace(" ", "_")
meta = {
"source": name,
"path": filename,
"product": "unknown",
"manufacturer": "unknown",
"type": "unknown",
"language": "fr" if any(k in f for k in ["_fr", "francais", "manuel"]) else "en",
}
for hint_key, hint_vals in PRODUCT_HINTS.items():
if hint_key in f:
meta.update(hint_vals)
break
# Fallback : nom de fichier sans extension comme product
if meta["product"] == "unknown":
meta["product"] = Path(filename).stem[:40]
return meta
# ── Query helper (pour wiki pipeline) ────────────────────────────────────────
def query_for_wiki(col_id, product_name, top_k=15):
"""
Retrouve les chunks les plus pertinents pour générer une page wiki.
Utiliser avec : query = "features workflow controls {product_name}"
"""
query = f"features workflow interface controls parameters {product_name}"
emb = embed(query)
result = chroma_request("POST", f"{CHROMA_BASE}/collections/{col_id}/query", {
"query_embeddings": [emb],
"n_results": top_k,
"where": {"product": product_name},
"include": ["documents", "metadatas", "distances"]
})
return result
# ── Main ──────────────────────────────────────────────────────────────────────
def main():
import sys
# Mode : ingest (défaut) ou query
mode = sys.argv[1] if len(sys.argv) > 1 else "ingest"
print(f"=== MAO ChromaDB — mode: {mode} ===")
print(f"Collection : {COLLECTION}")
print(f"Modèle embedding : {EMBED_MODEL}\n")
state = load_state()
col_id = get_or_create_collection()
if mode == "query":
product = sys.argv[2] if len(sys.argv) > 2 else "Serum"
print(f"Query wiki pour : {product}")
r = query_for_wiki(col_id, product)
for i, (doc, meta) in enumerate(zip(r["documents"][0], r["metadatas"][0])):
print(f"\n--- Chunk {i+1} [{meta['source']}] ---")
print(doc[:300] + "...")
return
pdfs = list_mao_pdfs()
print(f"{len(pdfs)} PDFs MAO sur Guardian\n")
for filename in pdfs:
if filename in state["ingested"]:
print(f"[SKIP] {filename}")
continue
print(f"\n[{filename}]")
try:
pdf_bytes = download_pdf(filename)
print(f" {len(pdf_bytes)/1024:.0f} Ko téléchargés")
text = extract_text(pdf_bytes)
print(f" {len(text)} chars extraits")
chunks = chunk_by_structure(text)
meta_base = infer_metadata(filename)
print(f" {len(chunks)} chunks | {meta_base['product']} ({meta_base['type']}) | lang:{meta_base['language']}")
ids, embeddings, documents, metadatas = [], [], [], []
for i, chunk in enumerate(chunks):
chunk_id = f"{Path(filename).stem}_{i:04d}"
t0 = time.time()
emb = embed(chunk)
word_count = len(chunk.split())
ids.append(chunk_id)
embeddings.append(emb)
documents.append(chunk)
metadatas.append({
**meta_base,
"chunk_index": i,
"total_chunks": len(chunks),
"word_count": word_count,
})
print(f" chunk {i+1}/{len(chunks)} ({word_count}w) OK ({time.time()-t0:.1f}s)")
if len(ids) >= 50:
add_documents(col_id, ids, embeddings, documents, metadatas)
ids, embeddings, documents, metadatas = [], [], [], []
if ids:
add_documents(col_id, ids, embeddings, documents, metadatas)
print(f"\n Indexé — {len(chunks)} chunks")
state["ingested"].append(filename)
state["failed"] = [f for f in state["failed"] if f.get("file") != filename]
save_state(state)
except Exception as e:
print(f" ERREUR: {e}")
state["failed"].append({"file": filename, "error": str(e)})
save_state(state)
done = len(state["ingested"])
failed = len(state["failed"])
print(f"\n=== Terminé : {done} PDFs indexés, {failed} échecs ===")
for f in state.get("failed", []):
print(f" ECHEC: {f['file']}{f['error']}")
if __name__ == "__main__":
main()