Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b9961fe220 | |||
| 150e003f18 | |||
| d4007aec00 | |||
| 1962bbb899 | |||
| 7f5531e3b9 | |||
| af0c0d5f33 | |||
| 5e7afaafae | |||
| 25b46ff146 | |||
| d9bf4b7d28 | |||
| 33c1429c52 |
@@ -108,6 +108,17 @@ def docker_reboot_host():
|
||||
docker_call("POST", "/containers/host-reboot/start", {})
|
||||
return status in (201, 409)
|
||||
|
||||
def docker_run_host_cmd(name, cmd_list):
|
||||
"""Run a command on the Ampère host via privileged nsenter container."""
|
||||
docker_call("DELETE", f"/containers/{name}?force=1")
|
||||
full_cmd = ["nsenter","-t","1","-m","-u","-i","-n","-p","--"] + cmd_list
|
||||
config = {"Image": "alpine", "Cmd": full_cmd,
|
||||
"HostConfig": {"Privileged": True, "PidMode": "host", "AutoRemove": True}}
|
||||
status, body = docker_call("POST", f"/containers/create?name={name}", config)
|
||||
if status in (201, 409):
|
||||
docker_call("POST", f"/containers/{name}/start", {})
|
||||
return status in (201, 409)
|
||||
|
||||
# ── SSH helpers (Guardian / Vigile) ───────────────────────────────────────
|
||||
SSH_KEYS = {
|
||||
"guardian": {"key": "/ssh/guardian.key", "host": "82.70.226.48"},
|
||||
@@ -190,11 +201,16 @@ def get_fallout_ingest():
|
||||
try:
|
||||
with open("/data/fallout_state.json") as f: state=json.load(f)
|
||||
except: pass
|
||||
running = _log_running("/data/fallout_ingest.log")
|
||||
# Préfère le log v2 s'il existe, sinon log v1
|
||||
import os as _os
|
||||
log_v2 = "/data/fallout_ingest_v2.log"
|
||||
log_v1 = "/data/fallout_ingest.log"
|
||||
log_path = log_v2 if _os.path.exists(log_v2) else log_v1
|
||||
running = _log_running(log_path)
|
||||
current_pdf=None; current_chunk=None; total_chunks=None
|
||||
chunks_done=[]; log_tail=[]
|
||||
chunks_done=[]; log_tail=[]; version="v2" if log_path==log_v2 else "v1"
|
||||
try:
|
||||
with open("/data/fallout_ingest.log") as f: lines=f.readlines()
|
||||
with open(log_path) as f: lines=f.readlines()
|
||||
log_tail=[l.rstrip() for l in lines[-30:] if l.strip()]
|
||||
for l in reversed(lines):
|
||||
l=l.rstrip()
|
||||
@@ -207,7 +223,7 @@ def get_fallout_ingest():
|
||||
if m2: chunks_done.append(float(m2.group(1)))
|
||||
except: pass
|
||||
avg_t=round(sum(chunks_done)/len(chunks_done),1) if chunks_done else None
|
||||
return {"running":running,"done":len(state.get("ingested",[])),"failed":len(state.get("failed",[])),
|
||||
return {"running":running,"version":version,"done":len(state.get("ingested",[])),"failed":len(state.get("failed",[])),
|
||||
"total":17,"current_pdf":current_pdf if running else None,
|
||||
"current_chunk":current_chunk,"total_chunks":total_chunks,"avg_embed_s":avg_t,"log_tail":log_tail}
|
||||
|
||||
@@ -358,6 +374,16 @@ class Handler(BaseHTTPRequestHandler):
|
||||
ok = ssh_reboot(vm)
|
||||
self.send_json({"ok":ok,"vm":vm})
|
||||
|
||||
elif p == "/admin/run-ingest":
|
||||
body = self.read_body()
|
||||
script = body.get("script","fallout")
|
||||
if script == "fallout":
|
||||
cmd = ["sh","-c","PYTHONUNBUFFERED=1 python3 -u /home/ubuntu/fallout/ingest.py >> /home/ubuntu/fallout/ingest.log 2>&1 &"]
|
||||
else:
|
||||
self.send_json({"ok":False,"error":"unknown script"}); return
|
||||
ok = docker_run_host_cmd("host-ingest", cmd)
|
||||
self.send_json({"ok":ok,"script":script})
|
||||
|
||||
else:
|
||||
self.send_response(404); self.end_headers()
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
services:
|
||||
dash:
|
||||
image: python:3.11-alpine
|
||||
container_name: dash
|
||||
restart: unless-stopped
|
||||
working_dir: /app
|
||||
volumes:
|
||||
- ./app.py:/app/app.py:ro
|
||||
- ./index.html:/app/index.html:ro
|
||||
- /home/ubuntu/wiki/pipeline_state.json:/data/pipeline_state.json:ro
|
||||
- /home/ubuntu/wiki/pipeline.log:/data/pipeline.log:ro
|
||||
- /home/ubuntu/fallout/ingest.log:/data/fallout_ingest.log:ro
|
||||
- /home/ubuntu/fallout/ingest_v2.log:/data/fallout_ingest_v2.log:ro
|
||||
- /home/ubuntu/fallout/ingest_state.json:/data/fallout_state.json:ro
|
||||
- /proc:/proc:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /home/ubuntu/.ssh/guardian.key:/ssh/guardian.key:ro
|
||||
- /home/ubuntu/.ssh/vigile_backup:/ssh/vigile.key:ro
|
||||
command: python app.py
|
||||
networks:
|
||||
- proxy-nw
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
networks:
|
||||
proxy-nw:
|
||||
external: true
|
||||
@@ -0,0 +1,687 @@
|
||||
"""
|
||||
fallout.coyoteos.ovh — Venice of Wasteland — Visualisation simulation
|
||||
Flask app sur Ampère, lit PostgreSQL Vigile via SSH tunnel
|
||||
"""
|
||||
import subprocess, time, threading, os, json
|
||||
from flask import Flask, render_template_string, jsonify, abort
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# ── SSH tunnel → PostgreSQL Vigile ────────────────────────────────────────────
|
||||
|
||||
TUNNEL_LOCAL_PORT = 15432
|
||||
_tunnel_proc = None
|
||||
|
||||
def start_tunnel():
|
||||
global _tunnel_proc
|
||||
cmd = [
|
||||
"ssh", "-N", "-L", f"{TUNNEL_LOCAL_PORT}:localhost:5432",
|
||||
"-i", "/ssh/vigile.key",
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "ServerAliveInterval=30",
|
||||
"-o", "ServerAliveCountMax=3",
|
||||
"ubuntu@79.72.30.231"
|
||||
]
|
||||
_tunnel_proc = subprocess.Popen(cmd)
|
||||
time.sleep(3)
|
||||
print(f"SSH tunnel → Vigile:5432 via local:{TUNNEL_LOCAL_PORT}")
|
||||
|
||||
def get_conn():
|
||||
import psycopg2
|
||||
return psycopg2.connect(
|
||||
host="localhost", port=TUNNEL_LOCAL_PORT,
|
||||
dbname="fallout", user="fallout",
|
||||
password="VeniceOfWasteland2026!",
|
||||
connect_timeout=5
|
||||
)
|
||||
|
||||
def query(sql, params=None):
|
||||
try:
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params or ())
|
||||
cols = [d[0] for d in cur.description]
|
||||
rows = [dict(zip(cols, r)) for r in cur.fetchall()]
|
||||
conn.close()
|
||||
return rows
|
||||
except Exception as e:
|
||||
print(f"DB error: {e}")
|
||||
return []
|
||||
|
||||
def execute(sql, params=None):
|
||||
try:
|
||||
conn = get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params or ())
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"DB error: {e}")
|
||||
return False
|
||||
|
||||
# ── Templates ─────────────────────────────────────────────────────────────────
|
||||
|
||||
BASE_CSS = """
|
||||
<style>
|
||||
@import url('https://fonts.googleapis.com/css2?family=VT323&family=Share+Tech+Mono&display=swap');
|
||||
:root {
|
||||
--green: #39ff14;
|
||||
--green-dim: #1a7a08;
|
||||
--green-dark: #0d3d04;
|
||||
--amber: #ffb000;
|
||||
--red: #ff3c3c;
|
||||
--bg: #050a05;
|
||||
--card-bg: #0a150a;
|
||||
--border: #1f4d1f;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
background: var(--bg);
|
||||
color: var(--green);
|
||||
font-family: 'Share Tech Mono', monospace;
|
||||
font-size: 14px;
|
||||
min-height: 100vh;
|
||||
position: relative;
|
||||
}
|
||||
body::before {
|
||||
content: '';
|
||||
position: fixed; inset: 0; pointer-events: none; z-index: 9999;
|
||||
background: repeating-linear-gradient(0deg, transparent, transparent 2px, rgba(0,0,0,0.08) 2px, rgba(0,0,0,0.08) 4px);
|
||||
}
|
||||
h1, h2, h3, .title { font-family: 'VT323', monospace; letter-spacing: 2px; }
|
||||
h1 { font-size: 2.4rem; color: var(--green); text-shadow: 0 0 10px var(--green); }
|
||||
h2 { font-size: 1.6rem; color: var(--green); margin-bottom: 12px; }
|
||||
h3 { font-size: 1.2rem; color: var(--amber); }
|
||||
a { color: var(--green-dim); text-decoration: none; }
|
||||
a:hover { color: var(--green); text-shadow: 0 0 6px var(--green); }
|
||||
|
||||
.layout { display: flex; min-height: 100vh; }
|
||||
.sidebar {
|
||||
width: 220px; flex-shrink: 0;
|
||||
background: var(--card-bg); border-right: 1px solid var(--border);
|
||||
padding: 20px 0;
|
||||
}
|
||||
.sidebar .logo { padding: 0 20px 20px; border-bottom: 1px solid var(--border); }
|
||||
.sidebar .logo .sub { font-size: 11px; color: var(--green-dim); }
|
||||
.sidebar nav a {
|
||||
display: block; padding: 10px 20px;
|
||||
color: var(--green-dim); font-size: 13px;
|
||||
border-left: 3px solid transparent;
|
||||
transition: all 0.15s;
|
||||
}
|
||||
.sidebar nav a:hover, .sidebar nav a.active {
|
||||
color: var(--green); border-left-color: var(--green);
|
||||
background: rgba(57,255,20,0.04);
|
||||
text-shadow: 0 0 6px var(--green);
|
||||
}
|
||||
.sidebar nav .section-label {
|
||||
padding: 16px 20px 4px; font-size: 10px;
|
||||
color: var(--green-dim); letter-spacing: 2px; text-transform: uppercase;
|
||||
}
|
||||
|
||||
.main { flex: 1; padding: 30px; overflow: auto; }
|
||||
.topbar {
|
||||
display: flex; justify-content: space-between; align-items: center;
|
||||
margin-bottom: 28px; padding-bottom: 16px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.badge {
|
||||
display: inline-block; padding: 3px 10px;
|
||||
border: 1px solid; border-radius: 2px;
|
||||
font-size: 11px; letter-spacing: 1px; text-transform: uppercase;
|
||||
}
|
||||
.badge-green { border-color: var(--green); color: var(--green); }
|
||||
.badge-amber { border-color: var(--amber); color: var(--amber); }
|
||||
.badge-red { border-color: var(--red); color: var(--red); }
|
||||
.badge-dim { border-color: var(--border); color: var(--green-dim); }
|
||||
|
||||
.card {
|
||||
background: var(--card-bg); border: 1px solid var(--border);
|
||||
border-radius: 2px; padding: 20px; margin-bottom: 16px;
|
||||
}
|
||||
.card-header { border-bottom: 1px solid var(--border); padding-bottom: 10px; margin-bottom: 14px; }
|
||||
|
||||
.grid { display: grid; gap: 16px; }
|
||||
.grid-2 { grid-template-columns: 1fr 1fr; }
|
||||
.grid-3 { grid-template-columns: 1fr 1fr 1fr; }
|
||||
.grid-5 { grid-template-columns: repeat(5, 1fr); }
|
||||
|
||||
.stat-bar { margin: 6px 0; }
|
||||
.stat-bar label { display: flex; justify-content: space-between; margin-bottom: 3px; font-size: 12px; }
|
||||
.bar-track { background: var(--green-dark); border-radius: 1px; height: 6px; }
|
||||
.bar-fill { height: 6px; background: var(--green); border-radius: 1px; transition: width 0.3s; }
|
||||
.bar-fill.danger { background: var(--red); }
|
||||
.bar-fill.warning { background: var(--amber); }
|
||||
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th { text-align: left; padding: 8px 12px; color: var(--green-dim); font-size: 11px; letter-spacing: 1px; text-transform: uppercase; border-bottom: 1px solid var(--border); }
|
||||
td { padding: 8px 12px; border-bottom: 1px solid rgba(31,77,31,0.4); font-size: 13px; }
|
||||
tr:hover td { background: rgba(57,255,20,0.03); }
|
||||
|
||||
.timeline { list-style: none; }
|
||||
.timeline li {
|
||||
position: relative; padding: 10px 10px 10px 24px;
|
||||
border-left: 1px solid var(--border); margin-left: 8px;
|
||||
}
|
||||
.timeline li::before {
|
||||
content: '▶'; position: absolute; left: -8px;
|
||||
background: var(--bg); color: var(--green-dim); font-size: 11px;
|
||||
}
|
||||
.timeline .meta { font-size: 11px; color: var(--green-dim); margin-bottom: 4px; }
|
||||
.timeline .action { color: var(--amber); }
|
||||
.timeline .result { color: var(--green); margin-top: 4px; font-size: 13px; }
|
||||
|
||||
.special-grid { display: grid; grid-template-columns: repeat(7, 1fr); gap: 8px; margin: 12px 0; }
|
||||
.special-box { text-align: center; padding: 10px 6px; border: 1px solid var(--border); }
|
||||
.special-box .label { font-size: 10px; color: var(--green-dim); }
|
||||
.special-box .val { font-size: 1.8rem; font-family: 'VT323', monospace; color: var(--green); }
|
||||
|
||||
.hp-display { display: flex; align-items: center; gap: 12px; margin: 12px 0; }
|
||||
.hp-num { font-family: 'VT323', monospace; font-size: 2rem; }
|
||||
.hp-num.danger { color: var(--red); }
|
||||
.hp-num.warning { color: var(--amber); }
|
||||
|
||||
.empty { text-align: center; padding: 40px; color: var(--green-dim); }
|
||||
.empty .ico { font-size: 2rem; margin-bottom: 8px; }
|
||||
|
||||
.status-alive { color: var(--green); }
|
||||
.status-injured { color: var(--amber); }
|
||||
.status-dead { color: var(--red); }
|
||||
.status-pending { color: var(--amber); }
|
||||
.status-resolving { color: #00bfff; }
|
||||
.status-resolved { color: var(--green-dim); }
|
||||
|
||||
.ticker {
|
||||
background: var(--card-bg); border: 1px solid var(--border);
|
||||
padding: 8px 14px; margin-bottom: 16px; font-size: 12px;
|
||||
color: var(--green-dim); white-space: nowrap; overflow: hidden;
|
||||
}
|
||||
.blink { animation: blink 1s step-end infinite; }
|
||||
@keyframes blink { 50% { opacity: 0; } }
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.grid-5 { grid-template-columns: 1fr 1fr; }
|
||||
.grid-3 { grid-template-columns: 1fr 1fr; }
|
||||
.sidebar { display: none; }
|
||||
}
|
||||
</style>
|
||||
"""
|
||||
|
||||
def nav(active=""):
|
||||
return f"""
|
||||
<div class="sidebar">
|
||||
<div class="logo">
|
||||
<h1 style="font-size:1.4rem">☢ VAULTCOM</h1>
|
||||
<div class="sub">Venice of Wasteland</div>
|
||||
<div class="sub">La Corboration</div>
|
||||
</div>
|
||||
<nav>
|
||||
<div class="section-label">Simulation</div>
|
||||
<a href="/" class="{'active' if active=='home' else ''}">⬡ Mission Control</a>
|
||||
<a href="/world" class="{'active' if active=='world' else ''}">◈ État du Monde</a>
|
||||
<div class="section-label">Agents</div>
|
||||
<a href="/characters" class="{'active' if active=='chars' else ''}">◉ Personnages</a>
|
||||
<a href="/actions" class="{'active' if active=='actions' else ''}">▶ File d'Actions</a>
|
||||
<div class="section-label">Archives</div>
|
||||
<a href="/lore" class="{'active' if active=='lore' else ''}">◎ Lore Avancée</a>
|
||||
<a href="/events" class="{'active' if active=='events' else ''}">⊟ Journal</a>
|
||||
</nav>
|
||||
</div>
|
||||
"""
|
||||
|
||||
SHELL = """<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>VaultCom — {{ title }}</title>
|
||||
""" + BASE_CSS + """
|
||||
</head>
|
||||
<body>
|
||||
<div class="layout">
|
||||
{{ nav | safe }}
|
||||
<div class="main">
|
||||
<div class="topbar">
|
||||
<h2>{{ title }}</h2>
|
||||
<div style="display:flex;gap:10px;align-items:center">
|
||||
{% if session %}
|
||||
<span class="badge badge-green">Jour {{ session.day_current }}/{{ session.day_total }}</span>
|
||||
<span class="badge badge-dim">{{ session.mode | upper }}</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">DB OFFLINE</span>
|
||||
{% endif %}
|
||||
<span style="font-size:11px;color:var(--green-dim)" id="clock"></span>
|
||||
</div>
|
||||
</div>
|
||||
{{ content | safe }}
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
function tick() { document.getElementById('clock').textContent = new Date().toLocaleTimeString('fr-FR'); }
|
||||
tick(); setInterval(tick, 1000);
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
def render(title, content, active="", session=None):
|
||||
return render_template_string(SHELL, title=title, content=content,
|
||||
nav=nav(active), session=session)
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_session():
|
||||
rows = query("SELECT * FROM sessions ORDER BY id DESC LIMIT 1")
|
||||
return rows[0] if rows else None
|
||||
|
||||
def hp_class(cur, mx):
|
||||
if mx == 0: return "warning"
|
||||
pct = cur / mx
|
||||
return "danger" if pct < 0.33 else ("warning" if pct < 0.66 else "")
|
||||
|
||||
def bar_class(cur, mx):
|
||||
if mx == 0: return "warning"
|
||||
pct = cur / mx
|
||||
return "danger" if pct < 0.33 else ("warning" if pct < 0.66 else "")
|
||||
|
||||
# ── Routes ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
sess = get_session()
|
||||
chars = query("SELECT * FROM characters WHERE session_id = %s ORDER BY faction, name",
|
||||
(sess["id"],)) if sess else []
|
||||
recent = query("""SELECT e.*, c.name as char_name FROM events e
|
||||
JOIN characters c ON c.id = e.character_id
|
||||
WHERE e.session_id = %s ORDER BY e.day_in_game DESC, e.hour_in_game DESC LIMIT 12""",
|
||||
(sess["id"],)) if sess else []
|
||||
factions = query("""SELECT DISTINCT faction, COUNT(*) as nb,
|
||||
SUM(CASE WHEN status='alive' THEN 1 ELSE 0 END) as alive
|
||||
FROM characters WHERE session_id = %s GROUP BY faction ORDER BY faction""",
|
||||
(sess["id"],)) if sess else []
|
||||
pending = query("SELECT COUNT(*) as n FROM action_queue WHERE session_id = %s AND status='pending'",
|
||||
(sess["id"],))[0]["n"] if sess else 0
|
||||
lore_last = query("""SELECT * FROM lore_avancee WHERE session_id = %s
|
||||
ORDER BY day_start DESC LIMIT 1""", (sess["id"],)) if sess else []
|
||||
|
||||
# Characters grid
|
||||
char_cards = ""
|
||||
if chars:
|
||||
for c in chars:
|
||||
hp_pct = int(c["hp_current"] / max(c["hp_max"], 1) * 100)
|
||||
bc = bar_class(c["hp_current"], c["hp_max"])
|
||||
nc = hp_class(c["hp_current"], c["hp_max"])
|
||||
st_cls = f"status-{c['status']}"
|
||||
char_cards += f"""
|
||||
<a href="/character/{c['id']}" style="text-decoration:none">
|
||||
<div class="card" style="cursor:pointer">
|
||||
<div class="card-header" style="display:flex;justify-content:space-between">
|
||||
<h3>{c['name']}</h3>
|
||||
<span class="badge badge-dim {st_cls}">{c['status'].upper()}</span>
|
||||
</div>
|
||||
<div style="font-size:11px;color:var(--green-dim);margin-bottom:8px">{c['faction'] or '—'} · {c['location'] or '?'}</div>
|
||||
<div class="stat-bar">
|
||||
<label><span>HP</span><span class="{nc}">{c['hp_current']}/{c['hp_max']}</span></label>
|
||||
<div class="bar-track"><div class="bar-fill {bc}" style="width:{hp_pct}%"></div></div>
|
||||
</div>
|
||||
<div style="font-size:11px;color:var(--green-dim);margin-top:6px">
|
||||
CAPS: {c['caps']} · XP: {c['xp']}
|
||||
</div>
|
||||
</div>
|
||||
</a>"""
|
||||
else:
|
||||
char_cards = '<div class="empty"><div class="ico">◉</div>Aucun agent initialisé</div>'
|
||||
|
||||
# Recent events
|
||||
ev_html = ""
|
||||
if recent:
|
||||
ev_html = '<ul class="timeline">'
|
||||
for e in recent:
|
||||
ev_html += f"""
|
||||
<li>
|
||||
<div class="meta">Jour {e['day_in_game']}h{e['hour_in_game']:02d} · {e['char_name']} · {e['location'] or '?'}</div>
|
||||
<div class="action">▶ {e['action'][:120]}{'…' if len(e['action'])>120 else ''}</div>
|
||||
{'<div class="result">⟹ ' + (e['result'][:150] if e['result'] else '') + '</div>' if e['result'] else ''}
|
||||
</li>"""
|
||||
ev_html += '</ul>'
|
||||
else:
|
||||
ev_html = '<div class="empty"><div class="ico">⊟</div>Journal vide — simulation non démarrée</div>'
|
||||
|
||||
# Faction summary
|
||||
fac_html = ""
|
||||
for f in factions:
|
||||
fac_html += f'<tr><td>{f["faction"] or "?"}</td><td>{f["alive"]}</td><td>{f["nb"]}</td></tr>'
|
||||
|
||||
# Lore entry
|
||||
lore_html = ""
|
||||
if lore_last:
|
||||
l = lore_last[0]
|
||||
lore_html = f"""
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<h3>{l['title'] or f"Résumé jours {l['day_start']}–{l['day_end']}"}</h3>
|
||||
<div style="font-size:11px;color:var(--green-dim)">Jours {l['day_start']}–{l['day_end']}</div>
|
||||
</div>
|
||||
<div style="line-height:1.7;color:var(--green)">{l['content'][:600]}{'…' if len(l['content'])>600 else ''}</div>
|
||||
</div>"""
|
||||
else:
|
||||
lore_html = '<div class="empty"><div class="ico">◎</div>Pas encore de lore synthétisé</div>'
|
||||
|
||||
content = f"""
|
||||
<div class="ticker">
|
||||
▌ VAULTCOM ACTIVE ▌ SESSION: {sess['name'] if sess else 'N/A'} ▌
|
||||
AGENTS: {len(chars)} ▌ ACTIONS EN ATTENTE: {pending} ▌
|
||||
<span class="blink">█</span>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-5" style="margin-bottom:20px">{char_cards}</div>
|
||||
|
||||
<div class="grid grid-2">
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>Journal récent</h3></div>
|
||||
{ev_html}
|
||||
</div>
|
||||
<div>
|
||||
<div class="card" style="margin-bottom:16px">
|
||||
<div class="card-header"><h3>Factions actives</h3></div>
|
||||
<table>
|
||||
<tr><th>Faction</th><th>Vivants</th><th>Total</th></tr>
|
||||
{fac_html if fac_html else '<tr><td colspan="3" style="color:var(--green-dim);text-align:center">—</td></tr>'}
|
||||
</table>
|
||||
</div>
|
||||
<div>
|
||||
<h3 style="margin-bottom:10px">Dernière synthèse narrative</h3>
|
||||
{lore_html}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
return render("Mission Control", content, "home", sess)
|
||||
|
||||
|
||||
@app.route("/characters")
|
||||
def characters():
|
||||
sess = get_session()
|
||||
chars = query("SELECT * FROM characters WHERE session_id = %s ORDER BY faction, name",
|
||||
(sess["id"],)) if sess else []
|
||||
rows = ""
|
||||
for c in chars:
|
||||
hp_pct = int(c["hp_current"] / max(c["hp_max"], 1) * 100)
|
||||
bc = bar_class(c["hp_current"], c["hp_max"])
|
||||
rows += f"""
|
||||
<tr>
|
||||
<td><a href="/character/{c['id']}">{c['name']}</a></td>
|
||||
<td style="color:var(--amber)">{c['faction'] or '—'}</td>
|
||||
<td>{c['location'] or '?'}</td>
|
||||
<td>
|
||||
<div class="bar-track" style="width:80px;display:inline-block">
|
||||
<div class="bar-fill {bc}" style="width:{hp_pct}%"></div>
|
||||
</div>
|
||||
<span style="font-size:11px;margin-left:6px">{c['hp_current']}/{c['hp_max']}</span>
|
||||
</td>
|
||||
<td>{c['caps']}</td>
|
||||
<td class="status-{c['status']}">{c['status'].upper()}</td>
|
||||
<td style="font-size:11px;color:var(--green-dim)">{c['xp']} XP</td>
|
||||
</tr>"""
|
||||
content = f"""
|
||||
<div class="card">
|
||||
<table>
|
||||
<tr><th>Nom</th><th>Faction</th><th>Lieu</th><th>HP</th><th>Caps</th><th>Status</th><th>XP</th></tr>
|
||||
{rows if rows else '<tr><td colspan="7" class="empty">Aucun personnage</td></tr>'}
|
||||
</table>
|
||||
</div>"""
|
||||
return render("Agents", content, "chars", sess)
|
||||
|
||||
|
||||
@app.route("/character/<int:cid>")
|
||||
def character(cid):
|
||||
sess = get_session()
|
||||
chars = query("SELECT * FROM characters WHERE id = %s", (cid,))
|
||||
if not chars: abort(404)
|
||||
c = chars[0]
|
||||
inv = query("SELECT * FROM inventory WHERE character_id = %s ORDER BY item_name", (cid,))
|
||||
events = query("""SELECT * FROM events WHERE character_id = %s
|
||||
ORDER BY day_in_game, hour_in_game""", (cid,))
|
||||
|
||||
SPECIALS = [("S","strength"),("P","perception"),("E","endurance"),
|
||||
("C","charisma"),("I","intelligence"),("A","agility"),("L","luck")]
|
||||
sp_html = "".join(f"""
|
||||
<div class="special-box">
|
||||
<div class="val">{c[key]}</div>
|
||||
<div class="label">{abbr}</div>
|
||||
</div>""" for abbr, key in SPECIALS)
|
||||
|
||||
hp_pct = int(c["hp_current"] / max(c["hp_max"], 1) * 100)
|
||||
bc = bar_class(c["hp_current"], c["hp_max"])
|
||||
nc = hp_class(c["hp_current"], c["hp_max"])
|
||||
|
||||
inv_rows = "".join(f"<tr><td>{i['item_name']}</td><td>{i.get('quantity',1)}</td><td style='color:var(--green-dim);font-size:12px'>{i.get('description','')}</td></tr>"
|
||||
for i in inv) if inv else "<tr><td colspan='3' style='color:var(--green-dim)'>Inventaire vide</td></tr>"
|
||||
|
||||
ev_html = ""
|
||||
if events:
|
||||
ev_html = '<ul class="timeline">'
|
||||
for e in events:
|
||||
ev_html += f"""
|
||||
<li>
|
||||
<div class="meta">Jour {e['day_in_game']}h{e['hour_in_game']:02d} · {e['location'] or '?'}</div>
|
||||
<div class="action">▶ {e['action']}</div>
|
||||
{'<div class="result">⟹ ' + (e['result'] or '') + '</div>' if e['result'] else ''}
|
||||
</li>"""
|
||||
ev_html += '</ul>'
|
||||
else:
|
||||
ev_html = '<div class="empty">Aucune action enregistrée</div>'
|
||||
|
||||
bio_html = f'<div style="color:var(--green-dim);line-height:1.7;margin-top:8px">{c["bio"] or "Dossier vide."}</div>'
|
||||
|
||||
content = f"""
|
||||
<div class="card" style="margin-bottom:20px">
|
||||
<div style="display:flex;justify-content:space-between;align-items:flex-start">
|
||||
<div>
|
||||
<h2 style="font-size:2rem">{c['name']}</h2>
|
||||
<div style="color:var(--amber);margin:4px 0">{c['faction'] or '—'}</div>
|
||||
<div style="color:var(--green-dim);font-size:12px">Localisation: {c['location'] or '?'} · Caps: {c['caps']} · XP: {c['xp']}</div>
|
||||
</div>
|
||||
<span class="badge badge-dim status-{c['status']}">{c['status'].upper()}</span>
|
||||
</div>
|
||||
<div class="hp-display">
|
||||
<span class="hp-num {nc}">{c['hp_current']}</span>
|
||||
<div style="flex:1">
|
||||
<div style="font-size:11px;color:var(--green-dim);margin-bottom:4px">/{c['hp_max']} HP</div>
|
||||
<div class="bar-track"><div class="bar-fill {bc}" style="width:{hp_pct}%"></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="special-grid">{sp_html}</div>
|
||||
{bio_html}
|
||||
</div>
|
||||
|
||||
<div class="grid grid-2">
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>Inventaire</h3></div>
|
||||
<table>
|
||||
<tr><th>Objet</th><th>Qté</th><th>Description</th></tr>
|
||||
{inv_rows}
|
||||
</table>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>Journal d'actions</h3></div>
|
||||
{ev_html}
|
||||
</div>
|
||||
</div>"""
|
||||
return render(c["name"], content, "chars", sess)
|
||||
|
||||
|
||||
@app.route("/actions")
|
||||
def actions():
|
||||
sess = get_session()
|
||||
rows = query("""
|
||||
SELECT aq.*, c.name as char_name, t.name as target_name
|
||||
FROM action_queue aq
|
||||
LEFT JOIN characters c ON c.id = aq.character_id
|
||||
LEFT JOIN characters t ON t.id = aq.target_char_id
|
||||
WHERE aq.session_id = %s
|
||||
ORDER BY aq.day_in_game DESC, aq.hour_in_game DESC, aq.id DESC
|
||||
LIMIT 100""", (sess["id"],)) if sess else []
|
||||
|
||||
def status_badge(s):
|
||||
cls = {"pending":"amber","resolving":"green","resolved":"dim","cancelled":"red"}.get(s,"dim")
|
||||
return f'<span class="badge badge-{cls}">{s.upper()}</span>'
|
||||
|
||||
trs = "".join(f"""
|
||||
<tr>
|
||||
<td>J{r['day_in_game']}h{r['hour_in_game']:02d}</td>
|
||||
<td><a href="/character/{r['character_id']}">{r['char_name'] or '?'}</a></td>
|
||||
<td>{r['location'] or '?'}</td>
|
||||
<td>{r['action_text'][:80]}{'…' if len(r['action_text'])>80 else ''}</td>
|
||||
<td>{r['target_name'] or '—'}</td>
|
||||
<td>{r['scene_id'] or '—'}</td>
|
||||
<td>{status_badge(r['status'])}</td>
|
||||
<td style="font-size:12px;color:var(--green-dim)">{(r['result_text'] or '')[:60]}</td>
|
||||
</tr>""" for r in rows) if rows else "<tr><td colspan='8' class='empty'>Aucune action</td></tr>"
|
||||
|
||||
content = f"""
|
||||
<div class="card">
|
||||
<table>
|
||||
<tr><th>Moment</th><th>Agent</th><th>Lieu</th><th>Action</th><th>Cible</th><th>Scène</th><th>Status</th><th>Résultat</th></tr>
|
||||
{trs}
|
||||
</table>
|
||||
</div>"""
|
||||
return render("File d'Actions", content, "actions", sess)
|
||||
|
||||
|
||||
@app.route("/world")
|
||||
def world():
|
||||
sess = get_session()
|
||||
if not sess:
|
||||
return render("État du Monde", '<div class="empty">DB offline</div>', "world")
|
||||
day = sess["day_current"]
|
||||
states = query("""SELECT * FROM world_state WHERE session_id = %s AND day_in_game = %s
|
||||
ORDER BY location""", (sess["id"], day))
|
||||
factions = query("""SELECT * FROM faction_relations WHERE session_id = %s
|
||||
ORDER BY faction_a, faction_b""", (sess["id"],))
|
||||
|
||||
def stab_color(s):
|
||||
if s >= 70: return "var(--green)"
|
||||
if s >= 40: return "var(--amber)"
|
||||
return "var(--red)"
|
||||
|
||||
world_rows = "".join(f"""
|
||||
<tr>
|
||||
<td style="color:var(--amber)">{s['location']}</td>
|
||||
<td>{s['controlling_faction'] or '—'}</td>
|
||||
<td>
|
||||
<div class="bar-track" style="width:80px;display:inline-block">
|
||||
<div class="bar-fill" style="width:{s['stability']}%;background:{stab_color(s['stability'])}"></div>
|
||||
</div>
|
||||
<span style="font-size:11px;margin-left:6px;color:{stab_color(s['stability'])}">{s['stability']}%</span>
|
||||
</td>
|
||||
<td>{s['population'] or '?'}</td>
|
||||
<td style="font-size:12px;color:var(--green-dim)">{s['notes'] or ''}</td>
|
||||
</tr>""" for s in states) if states else "<tr><td colspan='5' class='empty'>Aucune donnée</td></tr>"
|
||||
|
||||
fac_rows = "".join(f"""
|
||||
<tr>
|
||||
<td style="color:var(--amber)">{f['faction_a']}</td>
|
||||
<td>{f['faction_b']}</td>
|
||||
<td style="color:{'var(--green)' if f['relation_score']>0 else 'var(--red)' if f['relation_score']<0 else 'var(--green-dim)'}">{'+' if f['relation_score']>0 else ''}{f['relation_score']}</td>
|
||||
<td style="font-size:12px;color:var(--green-dim)">{f.get('notes','')}</td>
|
||||
</tr>""" for f in factions) if factions else "<tr><td colspan='4' class='empty'>Aucune donnée</td></tr>"
|
||||
|
||||
content = f"""
|
||||
<div class="grid grid-2">
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>Zones — Jour {day}</h3></div>
|
||||
<table>
|
||||
<tr><th>Lieu</th><th>Faction</th><th>Stabilité</th><th>Pop.</th><th>Notes</th></tr>
|
||||
{world_rows}
|
||||
</table>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-header"><h3>Relations entre factions</h3></div>
|
||||
<table>
|
||||
<tr><th>Faction A</th><th>Faction B</th><th>Score</th><th>Notes</th></tr>
|
||||
{fac_rows}
|
||||
</table>
|
||||
</div>
|
||||
</div>"""
|
||||
return render("État du Monde", content, "world", sess)
|
||||
|
||||
|
||||
@app.route("/lore")
|
||||
def lore():
|
||||
sess = get_session()
|
||||
entries = query("""SELECT * FROM lore_avancee WHERE session_id = %s
|
||||
ORDER BY day_start DESC""", (sess["id"],)) if sess else []
|
||||
cards = ""
|
||||
if entries:
|
||||
for e in entries:
|
||||
facs = ", ".join(e["factions_involved"] or []) or "—"
|
||||
locs = ", ".join(e["locations_involved"] or []) or "—"
|
||||
cards += f"""
|
||||
<div class="card">
|
||||
<div class="card-header" style="display:flex;justify-content:space-between">
|
||||
<h3>{e['title'] or f"Chronique jours {e['day_start']}–{e['day_end']}"}</h3>
|
||||
<span class="badge badge-dim">J{e['day_start']}–{e['day_end']}</span>
|
||||
</div>
|
||||
<div style="font-size:11px;color:var(--green-dim);margin-bottom:12px">
|
||||
Factions: {facs} · Lieux: {locs}
|
||||
</div>
|
||||
<div style="line-height:1.8;color:var(--green)">{e['content']}</div>
|
||||
</div>"""
|
||||
else:
|
||||
cards = '<div class="empty"><div class="ico">◎</div>Aucune synthèse narrative générée<br><span style="font-size:12px">Le MJ produira des entrées toutes les 24h de jeu simulé</span></div>'
|
||||
return render("Lore Avancée", cards, "lore", sess)
|
||||
|
||||
|
||||
@app.route("/events")
|
||||
def events():
|
||||
sess = get_session()
|
||||
evs = query("""SELECT e.*, c.name as char_name FROM events e
|
||||
JOIN characters c ON c.id = e.character_id
|
||||
WHERE e.session_id = %s
|
||||
ORDER BY e.day_in_game DESC, e.hour_in_game DESC
|
||||
LIMIT 200""", (sess["id"],)) if sess else []
|
||||
rows = "".join(f"""
|
||||
<tr>
|
||||
<td>J{e['day_in_game']}h{e['hour_in_game']:02d}</td>
|
||||
<td><a href="/character/{e['character_id']}">{e['char_name']}</a></td>
|
||||
<td style="color:var(--green-dim)">{e['location'] or '?'}</td>
|
||||
<td style="color:var(--amber)">{e['action'][:80]}{'…' if len(e['action'])>80 else ''}</td>
|
||||
<td style="font-size:12px">{(e['result'] or '')[:100]}{'…' if e['result'] and len(e['result'])>100 else ''}</td>
|
||||
</tr>""" for e in evs) if evs else "<tr><td colspan='5' class='empty'>Journal vide</td></tr>"
|
||||
|
||||
content = f"""
|
||||
<div class="card">
|
||||
<table>
|
||||
<tr><th>Moment</th><th>Agent</th><th>Lieu</th><th>Action</th><th>Résultat</th></tr>
|
||||
{rows}
|
||||
</table>
|
||||
</div>"""
|
||||
return render("Journal des Événements", content, "events", sess)
|
||||
|
||||
|
||||
@app.route("/api/summary")
|
||||
def api_summary():
|
||||
"""Pour le widget dashboard dash.coyoteos.ovh"""
|
||||
sess = get_session()
|
||||
if not sess:
|
||||
return jsonify({"ok": False})
|
||||
chars = query("SELECT status, COUNT(*) n FROM characters WHERE session_id=%s GROUP BY status", (sess["id"],))
|
||||
pending = query("SELECT COUNT(*) n FROM action_queue WHERE session_id=%s AND status='pending'", (sess["id"],))
|
||||
return jsonify({
|
||||
"ok": True,
|
||||
"session": sess["name"],
|
||||
"day": sess["day_current"],
|
||||
"day_total": sess["day_total"],
|
||||
"chars": {r["status"]: r["n"] for r in chars},
|
||||
"pending_actions": pending[0]["n"] if pending else 0,
|
||||
})
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
t = threading.Thread(target=start_tunnel, daemon=True)
|
||||
t.start()
|
||||
print("VaultCom :5000")
|
||||
app.run(host="0.0.0.0", port=5000, debug=False)
|
||||
@@ -0,0 +1,21 @@
|
||||
services:
|
||||
fallout-visu:
|
||||
image: python:3.11-alpine
|
||||
container_name: fallout-visu
|
||||
restart: unless-stopped
|
||||
working_dir: /app
|
||||
volumes:
|
||||
- ./app.py:/app/app.py:ro
|
||||
- /home/ubuntu/.ssh/vigile_backup:/ssh/vigile.key:ro
|
||||
command: >
|
||||
sh -c "apk add --no-cache openssh-client &&
|
||||
pip install flask psycopg2-binary --quiet &&
|
||||
python app.py"
|
||||
networks:
|
||||
- proxy-nw
|
||||
environment:
|
||||
- PYTHONUNBUFFERED=1
|
||||
|
||||
networks:
|
||||
proxy-nw:
|
||||
external: true
|
||||
@@ -0,0 +1,46 @@
|
||||
-- Migration fallout DB — tables manquantes
|
||||
-- À exécuter sur Vigile : docker exec -i fallout_db psql -U fallout -d fallout
|
||||
|
||||
-- File d'actions (coeur du moteur de simulation)
|
||||
CREATE TABLE IF NOT EXISTS action_queue (
|
||||
id SERIAL PRIMARY KEY,
|
||||
session_id INTEGER REFERENCES sessions(id),
|
||||
day_in_game INTEGER NOT NULL,
|
||||
hour_in_game INTEGER NOT NULL DEFAULT 0,
|
||||
character_id INTEGER REFERENCES characters(id),
|
||||
action_text TEXT NOT NULL,
|
||||
location VARCHAR(100),
|
||||
target_char_id INTEGER REFERENCES characters(id),
|
||||
target_location VARCHAR(100),
|
||||
scene_id VARCHAR(50), -- groupe les actions qui se croisent
|
||||
status VARCHAR(20) DEFAULT 'pending', -- pending|resolving|resolved|cancelled
|
||||
priority INTEGER DEFAULT 5,
|
||||
result_text TEXT, -- rempli après résolution par le MJ
|
||||
created_at TIMESTAMP DEFAULT now(),
|
||||
resolved_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_aq_session_day_hour ON action_queue(session_id, day_in_game, hour_in_game);
|
||||
CREATE INDEX IF NOT EXISTS idx_aq_status ON action_queue(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_aq_scene ON action_queue(scene_id);
|
||||
|
||||
-- Lore avancée — résumés narratifs générés par le LLM toutes les 24h de jeu
|
||||
CREATE TABLE IF NOT EXISTS lore_avancee (
|
||||
id SERIAL PRIMARY KEY,
|
||||
session_id INTEGER REFERENCES sessions(id),
|
||||
day_start INTEGER NOT NULL,
|
||||
day_end INTEGER NOT NULL,
|
||||
title VARCHAR(200),
|
||||
content TEXT NOT NULL, -- texte narratif généré par MJ 14b
|
||||
factions_involved TEXT[],
|
||||
locations_involved TEXT[],
|
||||
chroma_id VARCHAR(100), -- ref chunk fallout_canon dans ChromaDB
|
||||
created_at TIMESTAMP DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_la_session ON lore_avancee(session_id, day_start);
|
||||
|
||||
-- Vérification
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
ORDER BY table_name;
|
||||
+719
@@ -0,0 +1,719 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>CoyoteOS — Dashboard</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg:#0a0a0f; --surface:#111118; --surface2:#18181f; --border:#1f1f2e;
|
||||
--text:#e2e2f0; --muted:#6b6b88;
|
||||
--green:#22c55e; --orange:#f97316; --red:#ef4444;
|
||||
--blue:#3b82f6; --purple:#a855f7; --accent:#7c3aed;
|
||||
}
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{background:var(--bg);color:var(--text);font-family:'Segoe UI',system-ui,sans-serif;min-height:100vh}
|
||||
|
||||
/* Header */
|
||||
header{display:flex;align-items:center;justify-content:space-between;padding:1rem 2rem;
|
||||
border-bottom:1px solid var(--border);background:linear-gradient(90deg,#0d0d18,#110d1a);
|
||||
position:sticky;top:0;z-index:100}
|
||||
.logo{font-size:1.1rem;font-weight:700;letter-spacing:.05em;color:#fff}
|
||||
.logo span{color:var(--accent)}
|
||||
.live-dot{width:8px;height:8px;border-radius:50%;background:var(--green);
|
||||
box-shadow:0 0 8px var(--green);animation:pulse 2s infinite;display:inline-block}
|
||||
@keyframes pulse{0%,100%{opacity:1;transform:scale(1)}50%{opacity:.5;transform:scale(.8)}}
|
||||
.last-update{font-size:.72rem;color:var(--muted)}
|
||||
|
||||
/* Layout */
|
||||
main{padding:1.5rem 2rem;max-width:1600px;margin:0 auto;display:flex;flex-direction:column;gap:1.2rem}
|
||||
|
||||
/* Section title */
|
||||
.section-title{font-size:.68rem;font-weight:700;color:var(--muted);text-transform:uppercase;
|
||||
letter-spacing:.15em;margin-bottom:.8rem;display:flex;align-items:center;gap:.6rem}
|
||||
.section-title::after{content:"";flex:1;height:1px;background:var(--border)}
|
||||
|
||||
/* Cards */
|
||||
.card{background:var(--surface);border:1px solid var(--border);border-radius:12px;padding:1.2rem 1.4rem}
|
||||
|
||||
/* Grids */
|
||||
.grid-3{display:grid;grid-template-columns:repeat(3,1fr);gap:1rem}
|
||||
.grid-2{display:grid;grid-template-columns:repeat(2,1fr);gap:1rem}
|
||||
@media(max-width:1000px){.grid-3{grid-template-columns:repeat(2,1fr)}}
|
||||
@media(max-width:650px){.grid-3,.grid-2{grid-template-columns:1fr}}
|
||||
|
||||
/* Progress bars */
|
||||
.bar-bg{background:#1a1a2a;border-radius:6px;height:5px;overflow:hidden}
|
||||
.bar-fill{height:100%;border-radius:6px;transition:width .6s ease}
|
||||
|
||||
/* Colors */
|
||||
.c-green{color:var(--green)} .c-orange{color:var(--orange)} .c-red{color:var(--red)}
|
||||
.c-blue{color:var(--blue)} .c-purple{color:var(--purple)} .c-muted{color:var(--muted)}
|
||||
|
||||
/* Pipeline card */
|
||||
.pipeline-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;overflow:hidden}
|
||||
.pipeline-header{display:flex;align-items:center;justify-content:space-between;
|
||||
padding:1rem 1.4rem;border-bottom:1px solid var(--border);
|
||||
background:linear-gradient(135deg,#0d0d18,#160d24);cursor:pointer;user-select:none}
|
||||
.pipeline-header h2{font-size:.85rem;font-weight:600;color:#c4b5fd}
|
||||
.pipeline-body{padding:1.2rem 1.4rem;display:flex;flex-direction:column;gap:1rem}
|
||||
.pipeline-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:.8rem}
|
||||
.pipeline-stat{text-align:center}
|
||||
.pipeline-stat .n{font-size:2.2rem;font-weight:700}
|
||||
.pipeline-stat .l{font-size:.65rem;color:var(--muted);text-transform:uppercase;letter-spacing:.1em;margin-top:.2rem}
|
||||
.current-pdf{background:var(--surface2);border:1px solid var(--border);border-radius:8px;padding:.8rem 1rem}
|
||||
.pdf-name{font-size:.9rem;font-weight:600;color:#c4b5fd;margin-bottom:.4rem;word-break:break-all}
|
||||
.chunk-progress{display:flex;align-items:center;gap:.8rem;margin-top:.6rem}
|
||||
.chunk-bar-wrap{flex:1;background:#1a1a2a;border-radius:4px;height:8px;overflow:hidden}
|
||||
.chunk-bar{height:100%;background:linear-gradient(90deg,var(--accent),var(--purple));border-radius:4px;transition:width .4s}
|
||||
.chunk-label{font-size:.75rem;color:var(--muted);white-space:nowrap}
|
||||
.log-box{background:#080810;border:1px solid var(--border);border-radius:8px;padding:.8rem;
|
||||
font-family:'Cascadia Code','Consolas',monospace;font-size:.7rem;max-height:160px;overflow-y:auto;line-height:1.6}
|
||||
.log-ok{color:var(--green)} .log-err{color:var(--red)} .log-info{color:#6b6b88}
|
||||
.log-cur{color:#c4b5fd} .log-chunk{color:var(--blue)} .log-dl{color:var(--orange)}
|
||||
|
||||
/* VM cards */
|
||||
.vm-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;overflow:hidden}
|
||||
.vm-header{padding:.8rem 1.2rem;border-bottom:1px solid var(--border);display:flex;align-items:center;gap:.7rem}
|
||||
.vm-dot{width:9px;height:9px;border-radius:50%}
|
||||
.vm-dot.up{background:var(--green);box-shadow:0 0 6px var(--green)}
|
||||
.vm-dot.down{background:var(--red)}
|
||||
.vm-name{font-size:.85rem;font-weight:600}
|
||||
.vm-ip{font-size:.7rem;color:var(--muted);margin-left:auto;font-family:monospace}
|
||||
.vm-body{padding:1rem 1.2rem;display:flex;flex-direction:column;gap:.5rem}
|
||||
.vm-row{display:flex;align-items:center;justify-content:space-between}
|
||||
.vm-row .lbl{font-size:.72rem;color:var(--muted)}
|
||||
.vm-row .val{font-size:.85rem;font-weight:600}
|
||||
.mini-bar-wrap{flex:1;margin:0 .8rem;background:#1a1a2a;border-radius:3px;height:4px;overflow:hidden}
|
||||
.mini-bar{height:100%;border-radius:3px;transition:width .5s}
|
||||
|
||||
/* Summary */
|
||||
.summary-card{background:linear-gradient(135deg,#0f0a1e,#0a1020);border:1px solid #2a1a4a;
|
||||
border-radius:12px;padding:1.4rem;display:grid;grid-template-columns:repeat(3,1fr);gap:1rem;text-align:center}
|
||||
.sum-item .n{font-size:1.8rem;font-weight:700}
|
||||
.sum-item .l{font-size:.65rem;color:var(--muted);text-transform:uppercase;letter-spacing:.1em;margin-top:.3rem}
|
||||
.sum-divider{border-left:1px solid var(--border)}
|
||||
|
||||
/* Sites */
|
||||
.sites-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(150px,1fr));gap:.8rem}
|
||||
.site-btn{background:var(--surface);border:1px solid var(--border);border-radius:10px;
|
||||
padding:1rem;text-decoration:none;color:var(--text);
|
||||
display:flex;flex-direction:column;align-items:center;gap:.5rem;
|
||||
transition:border-color .2s,background .2s}
|
||||
.site-btn:hover{background:var(--surface2);border-color:var(--accent)}
|
||||
.site-btn .site-icon{font-size:1.5rem}
|
||||
.site-btn .site-name{font-size:.8rem;font-weight:600;text-align:center}
|
||||
.site-btn .site-status{display:flex;align-items:center;gap:.4rem;font-size:.68rem}
|
||||
.sdot{width:7px;height:7px;border-radius:50%;flex-shrink:0}
|
||||
.sdot-up{background:var(--green);box-shadow:0 0 5px var(--green)}
|
||||
.sdot-down{background:var(--red);box-shadow:0 0 5px var(--red)}
|
||||
|
||||
/* Badge */
|
||||
.badge{display:inline-flex;align-items:center;gap:.35rem;border-radius:20px;
|
||||
padding:.2rem .7rem;font-size:.68rem;font-weight:600}
|
||||
.badge-running{background:#0a2a0a;border:1px solid #22c55e44;color:var(--green)}
|
||||
.badge-idle{background:#1a1a1a;border:1px solid #33333366;color:var(--muted)}
|
||||
.badge-done{background:#0a1a2a;border:1px solid #3b82f644;color:var(--blue)}
|
||||
|
||||
/* Collapsible */
|
||||
.coll-hdr{display:flex;align-items:center;justify-content:space-between;
|
||||
cursor:pointer;user-select:none;margin-top:.6rem;padding-top:.6rem;border-top:1px solid var(--border)}
|
||||
.coll-hdr .coll-lbl{font-size:.65rem;color:var(--muted);text-transform:uppercase;letter-spacing:.1em}
|
||||
.coll-arrow{font-size:.7rem;color:var(--muted);transition:transform .2s}
|
||||
.coll-arrow.open{transform:rotate(180deg)}
|
||||
.coll-body{overflow:hidden;transition:max-height .3s ease;max-height:0}
|
||||
.coll-body.open{max-height:600px}
|
||||
.coll-inner{padding-top:.4rem}
|
||||
|
||||
/* Container list */
|
||||
.ct-row{display:flex;align-items:center;gap:.4rem;padding:.12rem 0;font-size:.7rem}
|
||||
.ct-dot{width:6px;height:6px;border-radius:50%;flex-shrink:0}
|
||||
.ct-dot.up{background:var(--green);box-shadow:0 0 4px var(--green)}
|
||||
.ct-dot.down{background:var(--red)}
|
||||
.ct-name{color:var(--text);flex:1}
|
||||
.ct-img{color:var(--muted);font-size:.63rem}
|
||||
|
||||
/* Ollama models */
|
||||
.ollama-model{display:flex;align-items:center;gap:.5rem;padding:.2rem 0;font-size:.72rem}
|
||||
.ollama-model .m-name{color:var(--green);font-weight:600;flex:1}
|
||||
.ollama-model .m-size{color:var(--muted)}
|
||||
|
||||
/* AdminSys */
|
||||
.admin-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:1rem}
|
||||
.admin-card{background:var(--surface);border:1px solid var(--border);border-radius:12px;overflow:hidden}
|
||||
.admin-card-header{padding:.8rem 1.2rem;background:linear-gradient(135deg,#120a1e,#0a1220);
|
||||
border-bottom:1px solid var(--border);display:flex;align-items:center;gap:.6rem}
|
||||
.admin-card-header .a-title{font-size:.85rem;font-weight:600;flex:1}
|
||||
.admin-card-body{padding:.8rem 1.2rem;display:flex;flex-direction:column;gap:.4rem}
|
||||
|
||||
/* Reboot buttons */
|
||||
.reboot-row{display:flex;gap:.6rem;flex-wrap:wrap;padding:.4rem 0}
|
||||
.btn-reboot{background:#1a0a0a;border:1px solid #ef444444;color:var(--red);
|
||||
border-radius:8px;padding:.4rem .9rem;font-size:.75rem;font-weight:600;cursor:pointer;
|
||||
transition:background .2s;white-space:nowrap}
|
||||
.btn-reboot:hover{background:#2a0a0a}
|
||||
|
||||
/* Group container controls */
|
||||
.group-card{background:var(--surface2);border:1px solid var(--border);border-radius:8px;margin-bottom:.4rem}
|
||||
.group-hdr{display:flex;align-items:center;gap:.5rem;padding:.5rem .8rem;cursor:pointer}
|
||||
.group-icon{font-size:.9rem}
|
||||
.group-name{font-size:.78rem;font-weight:600;flex:1}
|
||||
.group-badge{font-size:.62rem;color:var(--muted);background:#1a1a2a;border-radius:4px;padding:.1rem .4rem}
|
||||
.btn-group-toggle{font-size:.67rem;color:var(--muted);background:transparent;border:1px solid var(--border);
|
||||
border-radius:4px;padding:.15rem .5rem;cursor:pointer;transition:background .15s}
|
||||
.btn-group-toggle:hover{background:var(--border)}
|
||||
.group-containers{padding:0 .8rem .5rem;display:flex;flex-direction:column;gap:.25rem}
|
||||
.ct-control-row{display:flex;align-items:center;gap:.4rem}
|
||||
.ct-control-name{flex:1;font-size:.72rem;color:var(--text)}
|
||||
.btn-ct{font-size:.62rem;border:none;border-radius:4px;padding:.2rem .5rem;cursor:pointer;font-weight:600;transition:opacity .15s}
|
||||
.btn-ct:hover{opacity:.8}
|
||||
.btn-ct.stop{background:#2a0a0a;color:var(--red)}
|
||||
.btn-ct.start{background:#0a2a0a;color:var(--green)}
|
||||
.btn-ct.restart{background:#0a1a2a;color:var(--blue)}
|
||||
.btn-ct:disabled{opacity:.4;cursor:not-allowed}
|
||||
|
||||
/* Toast */
|
||||
.toast{position:fixed;bottom:1.5rem;right:1.5rem;background:#1a1a2a;border:1px solid var(--border);
|
||||
border-radius:8px;padding:.7rem 1.2rem;font-size:.8rem;z-index:999;
|
||||
animation:fadeIn .3s ease;pointer-events:none}
|
||||
@keyframes fadeIn{from{opacity:0;transform:translateY(8px)}to{opacity:1;transform:translateY(0)}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div class="logo"><span>Coyote</span>OS — Dashboard</div>
|
||||
<div style="display:flex;align-items:center;gap:1.2rem">
|
||||
<span><span class="live-dot"></span><span style="font-size:.8rem;margin-left:.4rem">Live</span></span>
|
||||
<span class="last-update" id="last-update">Chargement...</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
|
||||
<!-- ── PIPELINES ── -->
|
||||
<div>
|
||||
<div class="section-title">Pipelines actifs</div>
|
||||
<div class="grid-2">
|
||||
|
||||
<!-- MAO -->
|
||||
<div class="pipeline-card">
|
||||
<div class="pipeline-header" onclick="toggleColl(this.nextElementSibling)">
|
||||
<h2>qwen2.5 — Wiki MAO</h2>
|
||||
<div style="display:flex;align-items:center;gap:.6rem">
|
||||
<span id="pipeline-badge" class="badge badge-idle">● Vérification...</span>
|
||||
<span style="color:var(--muted);font-size:.8rem">▲</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="coll-body open" style="max-height:600px">
|
||||
<div class="pipeline-body">
|
||||
<div class="pipeline-grid">
|
||||
<div class="pipeline-stat"><div class="n c-green" id="p-done">—</div><div class="l">Terminés</div></div>
|
||||
<div class="pipeline-stat"><div class="n" style="color:#888">77</div><div class="l">Total FR/EN</div></div>
|
||||
<div class="pipeline-stat"><div class="n c-orange" id="p-failed">—</div><div class="l">Échecs</div></div>
|
||||
<div class="pipeline-stat"><div class="n c-blue" id="p-speed">—</div><div class="l">Moy. s/chunk</div></div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="display:flex;justify-content:space-between;margin-bottom:.4rem">
|
||||
<span style="font-size:.72rem;color:var(--muted)">Progression</span>
|
||||
<span style="font-size:.72rem;font-weight:600" id="p-pct">0%</span>
|
||||
</div>
|
||||
<div class="bar-bg" style="height:8px"><div class="bar-fill" id="p-bar" style="width:0%;background:var(--purple)"></div></div>
|
||||
</div>
|
||||
<div class="current-pdf">
|
||||
<div class="pdf-name" id="p-name">Attente...</div>
|
||||
<div style="font-size:.7rem;color:var(--muted)" id="p-eta"></div>
|
||||
<div class="chunk-progress">
|
||||
<div class="chunk-bar-wrap"><div class="chunk-bar" id="p-chunk-bar" style="width:0%"></div></div>
|
||||
<span class="chunk-label" id="p-chunk-label">chunk —/—</span>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="font-size:.67rem;color:var(--muted);margin-bottom:.3rem;text-transform:uppercase;letter-spacing:.1em">Log</div>
|
||||
<div class="log-box" id="log-box">Chargement...</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Fallout -->
|
||||
<div class="pipeline-card">
|
||||
<div class="pipeline-header" style="background:linear-gradient(135deg,#0d1010,#0d1a10)" onclick="toggleColl(this.nextElementSibling)">
|
||||
<h2 style="color:#4ade80">nomic-embed-text — Fallout ChromaDB</h2>
|
||||
<div style="display:flex;align-items:center;gap:.6rem">
|
||||
<span id="fallout-badge" class="badge badge-idle">● Vérification...</span>
|
||||
<span style="color:var(--muted);font-size:.8rem">▲</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="coll-body open" style="max-height:600px">
|
||||
<div class="pipeline-body">
|
||||
<div class="pipeline-grid">
|
||||
<div class="pipeline-stat"><div class="n c-green" id="f-done">—</div><div class="l">Indexés</div></div>
|
||||
<div class="pipeline-stat"><div class="n" style="color:#888">17</div><div class="l">Total</div></div>
|
||||
<div class="pipeline-stat"><div class="n c-orange" id="f-failed">—</div><div class="l">Échecs</div></div>
|
||||
<div class="pipeline-stat"><div class="n c-blue" id="f-speed">—</div><div class="l">Moy. s/embed</div></div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="display:flex;justify-content:space-between;margin-bottom:.4rem">
|
||||
<span style="font-size:.72rem;color:var(--muted)">Progression</span>
|
||||
<span style="font-size:.72rem;font-weight:600" id="f-pct">0%</span>
|
||||
</div>
|
||||
<div class="bar-bg" style="height:8px"><div class="bar-fill" id="f-bar" style="width:0%;background:var(--green)"></div></div>
|
||||
</div>
|
||||
<div class="current-pdf">
|
||||
<div class="pdf-name" id="f-name" style="color:#4ade80">Attente...</div>
|
||||
<div class="chunk-progress">
|
||||
<div class="chunk-bar-wrap"><div class="chunk-bar" id="f-chunk-bar" style="width:0%;background:linear-gradient(90deg,#22c55e,#4ade80)"></div></div>
|
||||
<span class="chunk-label" id="f-chunk-label">chunk —/—</span>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div style="font-size:.67rem;color:var(--muted);margin-bottom:.3rem;text-transform:uppercase;letter-spacing:.1em">Log</div>
|
||||
<div class="log-box" id="f-log-box">Chargement...</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── VMs ── -->
|
||||
<div>
|
||||
<div class="section-title">État des VMs</div>
|
||||
<div class="grid-3">
|
||||
|
||||
<!-- Ampère -->
|
||||
<div class="vm-card">
|
||||
<div class="vm-header">
|
||||
<div class="vm-dot up" id="vm-ampere-dot"></div>
|
||||
<div><div class="vm-name">Ampère</div><div style="font-size:.65rem;color:var(--muted)">ARM 4c / 24 Go — Cerveau</div></div>
|
||||
<div class="vm-ip">82.70.224.131</div>
|
||||
</div>
|
||||
<div class="vm-body">
|
||||
<div class="vm-row"><span class="lbl">CPU</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-a-cpu-bar" style="width:0%"></div></div>
|
||||
<span class="val" id="vm-a-cpu">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">RAM</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-a-mem-bar" style="width:0%;background:var(--blue)"></div></div>
|
||||
<span class="val c-blue" id="vm-a-mem">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Disque</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-a-disk-bar" style="width:0%;background:var(--purple)"></div></div>
|
||||
<span class="val c-purple" id="vm-a-disk">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Uptime</span><span class="val c-green" id="vm-a-uptime">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Réseau ↓/↑</span><span class="val" id="vm-a-net" style="font-size:.78rem">—</span></div>
|
||||
<!-- Ollama -->
|
||||
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
|
||||
<span class="coll-lbl">Ollama — modèles actifs</span><span class="coll-arrow open">▲</span>
|
||||
</div>
|
||||
<div class="coll-body open"><div class="coll-inner" id="vm-a-ollama">Chargement...</div></div>
|
||||
<!-- Containers -->
|
||||
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
|
||||
<span class="coll-lbl">Containers Docker (live)</span><span class="coll-arrow open">▲</span>
|
||||
</div>
|
||||
<div class="coll-body open"><div class="coll-inner" id="vm-a-containers">Chargement...</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Guardian -->
|
||||
<div class="vm-card">
|
||||
<div class="vm-header">
|
||||
<div class="vm-dot up" id="vm-guardian-dot"></div>
|
||||
<div><div class="vm-name">Guardian</div><div style="font-size:.65rem;color:var(--muted)">AMD 1c / 1 Go — Bouclier</div></div>
|
||||
<div class="vm-ip">82.70.226.48</div>
|
||||
</div>
|
||||
<div class="vm-body">
|
||||
<div class="vm-row"><span class="lbl">CPU</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-g-cpu-bar" style="width:0%"></div></div>
|
||||
<span class="val" id="vm-g-cpu">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">RAM</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-g-mem-bar" style="width:0%;background:var(--blue)"></div></div>
|
||||
<span class="val c-blue" id="vm-g-mem">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Disque</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-g-disk-bar" style="width:0%;background:var(--purple)"></div></div>
|
||||
<span class="val c-purple" id="vm-g-disk">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Uptime</span><span class="val c-green" id="vm-g-uptime">—</span></div>
|
||||
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
|
||||
<span class="coll-lbl">Containers Docker</span><span class="coll-arrow open">▲</span>
|
||||
</div>
|
||||
<div class="coll-body open"><div class="coll-inner">
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">pdffetch</span><span class="ct-img">python:alpine</span></div>
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">adguardhome</span><span class="ct-img">adguard/adguardhome</span></div>
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">wg-easy</span><span class="ct-img">ghcr.io/wg-easy</span></div>
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">node-exporter</span><span class="ct-img">prom/node-exporter</span></div>
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">portainer_agent</span><span class="ct-img">portainer/agent</span></div>
|
||||
</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Vigile -->
|
||||
<div class="vm-card">
|
||||
<div class="vm-header">
|
||||
<div class="vm-dot up" id="vm-vigile-dot"></div>
|
||||
<div><div class="vm-name">Vigile</div><div style="font-size:.65rem;color:var(--muted)">AMD 1c / 1 Go — Sentinelle</div></div>
|
||||
<div class="vm-ip">79.72.30.231</div>
|
||||
</div>
|
||||
<div class="vm-body">
|
||||
<div class="vm-row"><span class="lbl">CPU</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-v-cpu-bar" style="width:0%"></div></div>
|
||||
<span class="val" id="vm-v-cpu">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">RAM</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-v-mem-bar" style="width:0%;background:var(--blue)"></div></div>
|
||||
<span class="val c-blue" id="vm-v-mem">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Disque</span>
|
||||
<div class="mini-bar-wrap"><div class="mini-bar" id="vm-v-disk-bar" style="width:0%;background:var(--purple)"></div></div>
|
||||
<span class="val c-purple" id="vm-v-disk">—</span></div>
|
||||
<div class="vm-row"><span class="lbl">Uptime</span><span class="val c-green" id="vm-v-uptime">—</span></div>
|
||||
<div class="coll-hdr" onclick="toggleColl(this.nextElementSibling)">
|
||||
<span class="coll-lbl">Containers Docker</span><span class="coll-arrow open">▲</span>
|
||||
</div>
|
||||
<div class="coll-body open"><div class="coll-inner">
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">fallout_db</span><span class="ct-img">postgres:16-alpine</span></div>
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">node-exporter</span><span class="ct-img">prom/node-exporter</span></div>
|
||||
<div class="ct-row"><div class="ct-dot up"></div><span class="ct-name">portainer_agent</span><span class="ct-img">portainer/agent</span></div>
|
||||
</div></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── RÉSUMÉ 30j ── -->
|
||||
<div>
|
||||
<div class="section-title">Totaux infrastructure — 30 derniers jours</div>
|
||||
<div class="summary-card">
|
||||
<div class="sum-item"><div class="n c-blue" id="sum-rx">—</div><div class="l">Entrant ↓</div></div>
|
||||
<div class="sum-item sum-divider"><div class="n c-purple" id="sum-tx">—</div><div class="l">Sortant ↑</div></div>
|
||||
<div class="sum-item sum-divider"><div class="n" id="sum-total" style="color:#c4b5fd">—</div><div class="l">Total 3 VMs</div></div>
|
||||
</div>
|
||||
<div style="margin-top:.4rem;font-size:.68rem;color:var(--muted);text-align:right">Source : Prometheus PromQL</div>
|
||||
</div>
|
||||
|
||||
<!-- ── ADMINSYS ── -->
|
||||
<div>
|
||||
<div class="section-title">AdminSys</div>
|
||||
<div class="admin-grid">
|
||||
|
||||
<!-- Reboot VMs -->
|
||||
<div class="admin-card">
|
||||
<div class="admin-card-header">
|
||||
<span style="font-size:1.1rem">🔄</span>
|
||||
<span class="a-title">Redémarrage VMs</span>
|
||||
</div>
|
||||
<div class="admin-card-body" style="padding:1rem 1.2rem">
|
||||
<div style="font-size:.72rem;color:var(--muted);margin-bottom:.6rem">⚠️ Double-cliquez pour confirmer le reboot</div>
|
||||
<div class="reboot-row">
|
||||
<button class="btn-reboot" ondblclick="rebootVM('ampere',this)">⚡ Ampère</button>
|
||||
<button class="btn-reboot" ondblclick="rebootVM('guardian',this)">⚡ Guardian</button>
|
||||
<button class="btn-reboot" ondblclick="rebootVM('vigile',this)">⚡ Vigile</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Container groups -->
|
||||
<div class="admin-card" style="grid-column:span 2">
|
||||
<div class="admin-card-header">
|
||||
<span style="font-size:1.1rem">🐋</span>
|
||||
<span class="a-title">Containers Ampère — Groupes</span>
|
||||
<span style="font-size:.68rem;color:var(--muted)">Cliquer sur un groupe pour déployer</span>
|
||||
</div>
|
||||
<div class="admin-card-body" id="groups-body">Chargement...</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── SITES ── -->
|
||||
<div>
|
||||
<div class="section-title">Accès rapides — Services CoyoteOS</div>
|
||||
<div class="sites-grid" id="sites-grid"></div>
|
||||
</div>
|
||||
|
||||
</main>
|
||||
|
||||
<!-- Toast -->
|
||||
<div id="toast" class="toast" style="display:none"></div>
|
||||
|
||||
<script>
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
function fmt(b){if(!b)return"N/A";if(b>=1e12)return(b/1e12).toFixed(2)+" To";if(b>=1e9)return(b/1e9).toFixed(1)+" Go";if(b>=1e6)return(b/1e6).toFixed(1)+" Mo";if(b>=1e3)return(b/1e3).toFixed(0)+" Ko";return b+" o"}
|
||||
function fmtSpd(b){if(b>=1e6)return(b/1e6).toFixed(1)+" Mo/s";if(b>=1e3)return(b/1e3).toFixed(0)+" Ko/s";return b+" o/s"}
|
||||
function pctColor(p){if(p>=90)return"var(--red)";if(p>=70)return"var(--orange)";return"var(--green)"}
|
||||
function setBar(id,pct,color){const el=document.getElementById(id);if(el){el.style.width=pct+"%";if(color)el.style.background=color}}
|
||||
function setText(id,val){const el=document.getElementById(id);if(el)el.textContent=val}
|
||||
|
||||
function toast(msg, color="var(--green)"){
|
||||
const t=document.getElementById("toast");
|
||||
t.textContent=msg; t.style.color=color; t.style.display="block";
|
||||
setTimeout(()=>t.style.display="none",3000);
|
||||
}
|
||||
|
||||
function toggleColl(body){
|
||||
const open=body.classList.toggle("open");
|
||||
const arrow=body.previousElementSibling?.querySelector(".coll-arrow");
|
||||
if(arrow)arrow.classList.toggle("open",open);
|
||||
}
|
||||
|
||||
// ── Metrics Ampère ───────────────────────────────────────────────────────
|
||||
async function refreshMetrics(){
|
||||
try{
|
||||
const d=await fetch("/metrics").then(r=>r.json());
|
||||
const cpu=d.cpu.percent; const mem=d.memory.percent; const dsk=d.disk.percent;
|
||||
setText("vm-a-cpu",cpu+"%"); document.getElementById("vm-a-cpu").style.color=pctColor(cpu);
|
||||
setBar("vm-a-cpu-bar",cpu,pctColor(cpu));
|
||||
setText("vm-a-mem",mem+"%"); setBar("vm-a-mem-bar",mem,null);
|
||||
setText("vm-a-disk",dsk+"%"); document.getElementById("vm-a-disk").style.color=pctColor(dsk);
|
||||
setBar("vm-a-disk-bar",dsk,pctColor(dsk));
|
||||
const u=d.uptime; setText("vm-a-uptime",u.days+"j "+u.hours+"h "+u.minutes+"m");
|
||||
const net=d.network; setText("vm-a-net","↓"+fmtSpd(net.rx_bps)+" ↑"+fmtSpd(net.tx_bps));
|
||||
setText("last-update","Mis à jour "+new Date().toLocaleTimeString("fr-FR"));
|
||||
}catch(e){document.getElementById("vm-ampere-dot").className="vm-dot down"}
|
||||
}
|
||||
|
||||
// ── Métriques Guardian/Vigile via Prometheus ────────────────────────────
|
||||
async function refreshVmMetrics(instance, prefix){
|
||||
try{
|
||||
const d=await fetch(`/vm-metrics?instance=${instance}`).then(r=>r.json());
|
||||
if(!d.available) return;
|
||||
const cpu=d.cpu||0; const mem=d.mem_pct||0; const dsk=d.disk_pct||0;
|
||||
setText(`vm-${prefix}-cpu`,cpu+"%"); document.getElementById(`vm-${prefix}-cpu`).style.color=pctColor(cpu);
|
||||
setBar(`vm-${prefix}-cpu-bar`,cpu,pctColor(cpu));
|
||||
setText(`vm-${prefix}-mem`,mem+"%"); setBar(`vm-${prefix}-mem-bar`,mem,null);
|
||||
setText(`vm-${prefix}-disk`,dsk+"%"); document.getElementById(`vm-${prefix}-disk`).style.color=pctColor(dsk);
|
||||
setBar(`vm-${prefix}-disk-bar`,dsk,pctColor(dsk));
|
||||
if(d.uptime_human) setText(`vm-${prefix}-uptime`,d.uptime_human);
|
||||
document.getElementById(`vm-${prefix}-dot`).className="vm-dot up";
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// ── Ollama ───────────────────────────────────────────────────────────────
|
||||
async function refreshOllama(){
|
||||
try{
|
||||
const models=await fetch("/ollama-ps").then(r=>r.json());
|
||||
const el=document.getElementById("vm-a-ollama");
|
||||
if(!models.length){el.innerHTML='<div style="font-size:.7rem;color:var(--muted)">Aucun modèle chargé</div>';return}
|
||||
el.innerHTML=models.map(m=>`<div class="ollama-model">
|
||||
<span class="m-name">${m.name}</span>
|
||||
<span class="m-size">${m.size_gb}Go</span>
|
||||
</div>`).join("");
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// ── Docker Ampère ────────────────────────────────────────────────────────
|
||||
async function refreshDocker(){
|
||||
try{
|
||||
const cs=await fetch("/docker-ps").then(r=>r.json());
|
||||
const el=document.getElementById("vm-a-containers");
|
||||
if(!cs.length){el.innerHTML='<div class="ct-row" style="color:var(--muted)">Aucun container</div>';return}
|
||||
el.innerHTML=cs.map(c=>{
|
||||
const up=c.state==="running";
|
||||
return`<div class="ct-row"><div class="ct-dot ${up?"up":"down"}"></div><span class="ct-name" style="color:${up?"var(--text)":"var(--muted)"}">${c.name}</span><span class="ct-img">${c.image}</span></div>`;
|
||||
}).join("");
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// ── Pipelines ────────────────────────────────────────────────────────────
|
||||
function logLine(l){
|
||||
l=l.trim(); if(!l)return"";
|
||||
if(l.match(/OK \(\d/))return`<div class="log-ok">${l}</div>`;
|
||||
if(l.match(/ERREUR|Error/i))return`<div class="log-err">${l}</div>`;
|
||||
if(l.match(/^\[.+\.pdf\]/))return`<div class="log-cur">${l}</div>`;
|
||||
if(l.match(/chunk \d+\/\d+/))return`<div class="log-chunk">${l}</div>`;
|
||||
if(l.match(/Ko|chars|chunks|trouvés|Mo|Download/))return`<div class="log-dl">${l}</div>`;
|
||||
return`<div class="log-info">${l}</div>`;
|
||||
}
|
||||
|
||||
async function refreshPipeline(){
|
||||
try{
|
||||
const d=await fetch("/pipeline").then(r=>r.json());
|
||||
const done=d.done||0; const pct=Math.round((done/77)*100);
|
||||
setText("p-done",done); setText("p-failed",d.failed||0);
|
||||
setText("p-speed",d.avg_chunk_s?d.avg_chunk_s+"s":"—");
|
||||
setText("p-pct",pct+"%"); setBar("p-bar",pct,null);
|
||||
const badge=document.getElementById("pipeline-badge");
|
||||
if(d.running&&d.current_pdf){badge.className="badge badge-running";badge.textContent="● En cours"}
|
||||
else if(d.running){badge.className="badge badge-running";badge.textContent="● Actif"}
|
||||
else if(done>0){badge.className="badge badge-idle";badge.textContent="⏸ Cron 2h00"}
|
||||
else{badge.className="badge badge-idle";badge.textContent="● En attente"}
|
||||
if(d.running&&d.current_pdf){
|
||||
setText("p-name",d.current_pdf);
|
||||
setText("p-eta",(d.eta_current_pdf?"ETA: "+d.eta_current_pdf:"")+(d.last_chunk_s?" · "+d.last_chunk_s+"s/chunk":""));
|
||||
if(d.current_chunk&&d.total_chunks){
|
||||
const cp=Math.round(d.current_chunk/d.total_chunks*100);
|
||||
setBar("p-chunk-bar",cp,null); setText("p-chunk-label","chunk "+d.current_chunk+"/"+d.total_chunks);
|
||||
}
|
||||
}else{setText("p-name",done>0?"En pause — cron 2h00":"Pipeline en attente...");setText("p-eta","")}
|
||||
if(d.log_tail?.length){const b=document.getElementById("log-box");b.innerHTML=d.log_tail.map(logLine).join("");b.scrollTop=b.scrollHeight}
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
async function refreshFallout(){
|
||||
try{
|
||||
const d=await fetch("/fallout-ingest").then(r=>r.json());
|
||||
const done=d.done||0; const pct=Math.round((done/17)*100);
|
||||
setText("f-done",done); setText("f-failed",d.failed||0);
|
||||
setText("f-speed",d.avg_embed_s?d.avg_embed_s+"s":"—");
|
||||
setText("f-pct",pct+"%"); setBar("f-bar",pct,null);
|
||||
const badge=document.getElementById("fallout-badge");
|
||||
if(d.running&&d.current_pdf){badge.className="badge badge-running";badge.textContent="● En cours"}
|
||||
else if(d.running){badge.className="badge badge-running";badge.textContent="● Actif"}
|
||||
else if(done===17){badge.className="badge badge-done";badge.textContent="✓ 17/17 indexés"}
|
||||
else if(done>0){badge.className="badge badge-done";badge.textContent="✓ "+done+"/17"}
|
||||
else{badge.className="badge badge-idle";badge.textContent="● En attente"}
|
||||
if(d.running&&d.current_pdf){
|
||||
setText("f-name",d.current_pdf);
|
||||
if(d.current_chunk&&d.total_chunks){
|
||||
const cp=Math.round(d.current_chunk/d.total_chunks*100);
|
||||
setBar("f-chunk-bar",cp,null); setText("f-chunk-label","chunk "+d.current_chunk+"/"+d.total_chunks);
|
||||
}
|
||||
}else{setText("f-name",done===17?"✓ Indexation terminée":(done>0?done+"/17 PDFs indexés":"En attente..."))}
|
||||
if(d.log_tail?.length){const b=document.getElementById("f-log-box");b.innerHTML=d.log_tail.map(logLine).join("");b.scrollTop=b.scrollHeight}
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// ── Summary ──────────────────────────────────────────────────────────────
|
||||
async function refreshSummary(){
|
||||
try{
|
||||
const d=await fetch("/summary30").then(r=>r.json());
|
||||
setText("sum-rx",d.rx_30d?fmt(d.rx_30d):"via Grafana");
|
||||
setText("sum-tx",d.tx_30d?fmt(d.tx_30d):"via Grafana");
|
||||
setText("sum-total",d.total_bw?fmt(d.total_bw):"via Grafana");
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// ── Sites ─────────────────────────────────────────────────────────────────
|
||||
const ICONS={globe:"🌐",book:"📚",git:"🐙",chart:"📊",music:"🎵",studio:"🎹",dash:"🖥️"};
|
||||
async function refreshSites(){
|
||||
try{
|
||||
const sites=await fetch("/sites").then(r=>r.json());
|
||||
document.getElementById("sites-grid").innerHTML=sites.map(s=>{
|
||||
const dot=s.up?"sdot-up":"sdot-down";
|
||||
const lbl=s.up?"En ligne":(s.status?"Erreur "+s.status:"Hors ligne");
|
||||
return`<a class="site-btn" href="${s.url}" target="_blank">
|
||||
<div class="site-icon">${ICONS[s.icon]||"🔗"}</div>
|
||||
<div class="site-name">${s.name}</div>
|
||||
<div class="site-status"><div class="sdot ${dot}"></div><span style="color:${s.up?"var(--green)":"var(--red)"}">${lbl}</span></div>
|
||||
</a>`;
|
||||
}).join("");
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
// ── AdminSys — Groups ─────────────────────────────────────────────────────
|
||||
let allContainers = [];
|
||||
let groups = [];
|
||||
|
||||
async function loadGroups(){
|
||||
try{
|
||||
[groups, allContainers] = await Promise.all([
|
||||
fetch("/groups").then(r=>r.json()),
|
||||
fetch("/docker-ps").then(r=>r.json())
|
||||
]);
|
||||
renderGroups();
|
||||
}catch(e){}
|
||||
}
|
||||
|
||||
function getContainerState(name){
|
||||
const c=allContainers.find(x=>x.name===name);
|
||||
return c?c.state:"unknown";
|
||||
}
|
||||
|
||||
function renderGroups(){
|
||||
const body=document.getElementById("groups-body");
|
||||
body.innerHTML=groups.map(g=>{
|
||||
const ctHtml=g.containers.map(name=>{
|
||||
const state=getContainerState(name);
|
||||
const up=state==="running"; const unknown=state==="unknown";
|
||||
const dot=unknown?"":up?'<div class="ct-dot up"></div>':'<div class="ct-dot down"></div>';
|
||||
const btns=g.protected?"":`
|
||||
<button class="btn-ct ${up?"stop":"start"}" onclick="ctAction('ampere','${name}','${up?"stop":"start"}',this)">
|
||||
${up?"■ Stop":"▶ Start"}
|
||||
</button>
|
||||
${up?`<button class="btn-ct restart" onclick="ctAction('ampere','${name}','restart',this)">↺</button>`:""}
|
||||
`;
|
||||
return`<div class="ct-control-row">${dot}<span class="ct-control-name" style="color:${unknown?"var(--muted)":up?"var(--text)":"var(--red)"}">${name}</span>${btns}</div>`;
|
||||
}).join("");
|
||||
const runCount=g.containers.filter(n=>getContainerState(n)==="running").length;
|
||||
const allUp=runCount===g.containers.filter(n=>getContainerState(n)!=="unknown").length;
|
||||
const groupBtn=g.protected?"":
|
||||
`<button class="btn-group-toggle" onclick="groupAction('${g.id}','${allUp?"stop":"start"}',this)">${allUp?"■ Stop tout":"▶ Start tout"}</button>`;
|
||||
return`<div class="group-card">
|
||||
<div class="group-hdr" onclick="toggleGrp(this)">
|
||||
<span class="group-icon">${g.icon}</span>
|
||||
<span class="group-name">${g.name}</span>
|
||||
<span class="group-badge">${runCount}/${g.containers.length} up</span>
|
||||
${groupBtn}
|
||||
<span style="color:var(--muted);font-size:.7rem;margin-left:.3rem">▼</span>
|
||||
</div>
|
||||
<div class="group-containers" style="display:none">${ctHtml}</div>
|
||||
</div>`;
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function toggleGrp(hdr){
|
||||
const body=hdr.nextElementSibling;
|
||||
body.style.display=body.style.display==="none"?"flex":"none";
|
||||
body.style.flexDirection="column";
|
||||
}
|
||||
|
||||
async function ctAction(vm, name, action, btn){
|
||||
btn.disabled=true; btn.textContent="...";
|
||||
try{
|
||||
const r=await fetch("/admin/docker",{method:"POST",headers:{"Content-Type":"application/json"},
|
||||
body:JSON.stringify({vm,container:name,action})}).then(r=>r.json());
|
||||
toast(r.ok?`✓ ${name} ${action}`:`✗ Erreur ${name}`,r.ok?"var(--green)":"var(--red)");
|
||||
await new Promise(r=>setTimeout(r,800));
|
||||
allContainers=await fetch("/docker-ps").then(r=>r.json());
|
||||
renderGroups();
|
||||
}catch(e){toast("Erreur réseau","var(--red)");btn.disabled=false}
|
||||
}
|
||||
|
||||
async function groupAction(groupId, action, btn){
|
||||
btn.disabled=true; btn.textContent="...";
|
||||
const g=groups.find(x=>x.id===groupId);
|
||||
if(!g){btn.disabled=false;return}
|
||||
const targets=action==="stop"?g.containers.filter(n=>getContainerState(n)==="running")
|
||||
:g.containers.filter(n=>getContainerState(n)!=="running");
|
||||
for(const name of targets){
|
||||
await fetch("/admin/docker",{method:"POST",headers:{"Content-Type":"application/json"},
|
||||
body:JSON.stringify({vm:"ampere",container:name,action})});
|
||||
}
|
||||
toast(`✓ Groupe ${g.name} — ${action}`,"var(--green)");
|
||||
allContainers=await fetch("/docker-ps").then(r=>r.json());
|
||||
renderGroups();
|
||||
}
|
||||
|
||||
async function rebootVM(vm, btn){
|
||||
btn.textContent="Rebooting..."; btn.disabled=true;
|
||||
try{
|
||||
const r=await fetch("/admin/reboot",{method:"POST",headers:{"Content-Type":"application/json"},
|
||||
body:JSON.stringify({vm})}).then(r=>r.json());
|
||||
toast(r.ok?`⚡ ${vm} reboot lancé`:`✗ Erreur reboot ${vm}`,r.ok?"var(--orange)":"var(--red)");
|
||||
}catch(e){toast("Erreur réseau","var(--red)")}
|
||||
setTimeout(()=>{btn.disabled=false;btn.textContent=`⚡ ${vm.charAt(0).toUpperCase()+vm.slice(1)}`;},5000);
|
||||
}
|
||||
|
||||
// ── Init ─────────────────────────────────────────────────────────────────
|
||||
refreshMetrics();
|
||||
refreshVmMetrics("Guardian","g");
|
||||
refreshVmMetrics("Vigile","v");
|
||||
refreshOllama();
|
||||
refreshDocker();
|
||||
refreshPipeline();
|
||||
refreshFallout();
|
||||
refreshSummary();
|
||||
refreshSites();
|
||||
loadGroups();
|
||||
|
||||
setInterval(refreshMetrics, 5000);
|
||||
setInterval(()=>refreshVmMetrics("Guardian","g"), 15000);
|
||||
setInterval(()=>refreshVmMetrics("Vigile","v"), 15000);
|
||||
setInterval(refreshOllama, 10000);
|
||||
setInterval(refreshDocker, 15000);
|
||||
setInterval(refreshPipeline, 8000);
|
||||
setInterval(refreshFallout, 8000);
|
||||
setInterval(refreshSummary, 60000);
|
||||
setInterval(refreshSites, 30000);
|
||||
setInterval(loadGroups, 20000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
Ingestion Fallout PDFs → ChromaDB v2
|
||||
Améliorations v2 :
|
||||
- Chunking structurel (paragraphes/sections) au lieu de token-count fixe
|
||||
- Métadonnées enrichies : systeme (2d20 | tardy | none), category, priority
|
||||
- Évite la collision mécanique 2D20 vs Tardy en taggant chaque chunk
|
||||
- Chunks 400-600 mots, overlap 80 mots, respecte les frontières de paragraphes
|
||||
"""
|
||||
import json, os, re, subprocess, tempfile, time, urllib.request, urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
PDFFETCH_URL = "http://localhost:15001/files"
|
||||
OLLAMA_EMBED = "http://localhost:11434/api/embeddings"
|
||||
CHROMA_URL = "http://localhost:8800"
|
||||
CHROMA_BASE = "/api/v2/tenants/default_tenant/databases/default_database"
|
||||
COLLECTION = "fallout_lore"
|
||||
EMBED_MODEL = "nomic-embed-text"
|
||||
STATE_FILE = Path("/home/ubuntu/fallout/ingest_state.json")
|
||||
FALLOUT_PREFIX = "fallout/"
|
||||
|
||||
# Chunking params (en mots, pas tokens)
|
||||
CHUNK_WORDS = 450
|
||||
OVERLAP_WORDS = 80
|
||||
MIN_WORDS = 40
|
||||
|
||||
|
||||
# ── État ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def load_state():
|
||||
if STATE_FILE.exists():
|
||||
return json.loads(STATE_FILE.read_text())
|
||||
return {"ingested": [], "failed": [], "version": 2}
|
||||
|
||||
def save_state(s):
|
||||
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
STATE_FILE.write_text(json.dumps(s, indent=2))
|
||||
|
||||
|
||||
# ── ChromaDB ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def chroma_request(method, path, data=None):
|
||||
url = CHROMA_URL + path
|
||||
body = json.dumps(data).encode() if data else None
|
||||
req = urllib.request.Request(url, data=body, method=method,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
def get_or_create_collection():
|
||||
try:
|
||||
r = chroma_request("POST", f"{CHROMA_BASE}/collections", {
|
||||
"name": COLLECTION,
|
||||
"metadata": {"hnsw:space": "cosine"}
|
||||
})
|
||||
print(f"Collection créée : {COLLECTION} ({r['id']})")
|
||||
return r["id"]
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 409:
|
||||
cols = chroma_request("GET", f"{CHROMA_BASE}/collections")
|
||||
col_id = next(c["id"] for c in cols if c["name"] == COLLECTION)
|
||||
print(f"Collection existante : {COLLECTION} ({col_id})")
|
||||
return col_id
|
||||
raise
|
||||
|
||||
def add_documents(col_id, ids, embeddings, documents, metadatas):
|
||||
chroma_request("POST", f"{CHROMA_BASE}/collections/{col_id}/add", {
|
||||
"ids": ids, "embeddings": embeddings,
|
||||
"documents": documents, "metadatas": metadatas
|
||||
})
|
||||
|
||||
|
||||
# ── Embeddings ────────────────────────────────────────────────────────────────
|
||||
|
||||
def embed(text):
|
||||
payload = json.dumps({"model": EMBED_MODEL, "prompt": text}).encode()
|
||||
req = urllib.request.Request(OLLAMA_EMBED, data=payload,
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.loads(r.read())["embedding"]
|
||||
|
||||
|
||||
# ── PDF helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
def list_fallout_pdfs():
|
||||
req = urllib.request.Request(PDFFETCH_URL,
|
||||
headers={"User-Agent": "fallout-ingest/2.0"})
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
all_files = json.loads(r.read())
|
||||
return [f for f in all_files if f.startswith(FALLOUT_PREFIX)]
|
||||
|
||||
def download_pdf(filename):
|
||||
url = f"{PDFFETCH_URL}/{urllib.parse.quote(filename)}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "fallout-ingest/2.0"})
|
||||
with urllib.request.urlopen(req, timeout=300) as r:
|
||||
return r.read()
|
||||
|
||||
def extract_text(pdf_bytes):
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
f.write(pdf_bytes); tmp = f.name
|
||||
text = ""
|
||||
try:
|
||||
r = subprocess.run(["pdftotext", "-layout", tmp, "-"],
|
||||
capture_output=True, timeout=180)
|
||||
if r.returncode == 0:
|
||||
text = r.stdout.decode("utf-8", errors="replace")
|
||||
finally:
|
||||
os.unlink(tmp)
|
||||
if len(text.strip()) < 200:
|
||||
try:
|
||||
import io
|
||||
from pypdf import PdfReader
|
||||
reader = PdfReader(io.BytesIO(pdf_bytes))
|
||||
text = "\n".join(p.extract_text() or "" for p in reader.pages)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Extraction impossible: {e}")
|
||||
return text
|
||||
|
||||
|
||||
# ── Chunking structurel ───────────────────────────────────────────────────────
|
||||
|
||||
def is_section_header(line):
|
||||
"""Détecte les titres de section : ligne courte, capitalisée, pas de ponctuation finale."""
|
||||
l = line.strip()
|
||||
if not l or len(l) > 80:
|
||||
return False
|
||||
if l.isupper() and len(l.split()) <= 8:
|
||||
return True
|
||||
if re.match(r'^(CHAPITRE|CHAPTER|PARTIE|PART|SECTION|\d+[\.\)])\s+\S', l, re.I):
|
||||
return True
|
||||
return False
|
||||
|
||||
def chunk_by_structure(text, filename):
|
||||
"""
|
||||
Chunking basé sur les paragraphes et sections, pas sur le token count fixe.
|
||||
Respecte les frontières naturelles du texte.
|
||||
"""
|
||||
# Nettoyage
|
||||
text = re.sub(r'[ \t]+', ' ', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
|
||||
# Découpe en blocs (paragraphes ou sections)
|
||||
raw_blocks = re.split(r'\n{2,}', text)
|
||||
|
||||
# Regroupe les headers avec le paragraphe suivant
|
||||
blocks = []
|
||||
i = 0
|
||||
while i < len(raw_blocks):
|
||||
b = raw_blocks[i].strip()
|
||||
if not b:
|
||||
i += 1
|
||||
continue
|
||||
if is_section_header(b) and i + 1 < len(raw_blocks):
|
||||
merged = b + "\n\n" + raw_blocks[i+1].strip()
|
||||
blocks.append(merged)
|
||||
i += 2
|
||||
else:
|
||||
blocks.append(b)
|
||||
i += 1
|
||||
|
||||
# Accumulation en chunks de ~CHUNK_WORDS mots avec overlap
|
||||
chunks = []
|
||||
current_blocks = []
|
||||
current_words = 0
|
||||
|
||||
for block in blocks:
|
||||
bwords = len(block.split())
|
||||
if bwords < 5:
|
||||
continue
|
||||
|
||||
if current_words + bwords > CHUNK_WORDS and current_blocks:
|
||||
chunk = "\n\n".join(current_blocks).strip()
|
||||
if len(chunk.split()) >= MIN_WORDS:
|
||||
chunks.append(chunk)
|
||||
|
||||
# Overlap : garder les derniers blocs jusqu'à OVERLAP_WORDS mots
|
||||
overlap_blocks = []
|
||||
overlap_words = 0
|
||||
for ob in reversed(current_blocks):
|
||||
ow = len(ob.split())
|
||||
if overlap_words + ow <= OVERLAP_WORDS:
|
||||
overlap_blocks.insert(0, ob)
|
||||
overlap_words += ow
|
||||
else:
|
||||
break
|
||||
current_blocks = overlap_blocks + [block]
|
||||
current_words = overlap_words + bwords
|
||||
else:
|
||||
current_blocks.append(block)
|
||||
current_words += bwords
|
||||
|
||||
# Flush final
|
||||
if current_blocks:
|
||||
chunk = "\n\n".join(current_blocks).strip()
|
||||
if len(chunk.split()) >= MIN_WORDS:
|
||||
chunks.append(chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
# ── Métadonnées ───────────────────────────────────────────────────────────────
|
||||
|
||||
def infer_metadata(filename):
|
||||
f = filename.lower()
|
||||
name = Path(filename).name
|
||||
meta = {"source": name, "path": filename}
|
||||
|
||||
# Catégorie
|
||||
if "lore_post" in f: meta["category"] = "lore_canon"
|
||||
elif "lore_pre" in f: meta["category"] = "lore_inspiration"
|
||||
elif "regles/core" in f: meta["category"] = "regles_core"
|
||||
elif "regles/suppl" in f: meta["category"] = "regles_supplement"
|
||||
elif "aventures" in f: meta["category"] = "aventure"
|
||||
elif "ambiance" in f: meta["category"] = "ambiance"
|
||||
else: meta["category"] = "divers"
|
||||
|
||||
# Système de règles — CRITIQUE pour éviter la collision 2D20 vs Tardy
|
||||
if "tardy" in f:
|
||||
meta["systeme"] = "tardy"
|
||||
elif any(k in f for k in ["manuel_officiel", "quickstart", "rules_booklet",
|
||||
"gm_toolkit", "bestiaire", "armes", "special",
|
||||
"competences", "pizza", "frequence"]):
|
||||
meta["systeme"] = "2d20"
|
||||
elif "regles" in f:
|
||||
meta["systeme"] = "2d20"
|
||||
else:
|
||||
meta["systeme"] = "none"
|
||||
|
||||
# Priorité lore : la bible canon prime
|
||||
if "bible_v2" in f: meta["priority"] = "haute"
|
||||
elif "bible_v1" in f: meta["priority"] = "normale"
|
||||
elif "lore_canon" == meta.get("category"): meta["priority"] = "normale"
|
||||
else: meta["priority"] = "basse"
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
print("=== Ingestion Fallout v2 → ChromaDB ===")
|
||||
print(f"Chunking structurel, ~{CHUNK_WORDS} mots/chunk, overlap {OVERLAP_WORDS} mots")
|
||||
print(f"Modèle embedding : {EMBED_MODEL}\n")
|
||||
|
||||
state = load_state()
|
||||
col_id = get_or_create_collection()
|
||||
|
||||
pdfs = list_fallout_pdfs()
|
||||
print(f"{len(pdfs)} PDFs Fallout sur Guardian\n")
|
||||
|
||||
for filename in pdfs:
|
||||
if filename in state["ingested"]:
|
||||
print(f"[SKIP] {filename}")
|
||||
continue
|
||||
|
||||
print(f"\n[{filename}]")
|
||||
try:
|
||||
pdf_bytes = download_pdf(filename)
|
||||
print(f" {len(pdf_bytes)/1024:.0f} Ko téléchargés")
|
||||
|
||||
text = extract_text(pdf_bytes)
|
||||
print(f" {len(text)} chars extraits")
|
||||
|
||||
chunks = chunk_by_structure(text, filename)
|
||||
meta_base = infer_metadata(filename)
|
||||
print(f" {len(chunks)} chunks | catégorie: {meta_base['category']} | système: {meta_base['systeme']}")
|
||||
|
||||
ids, embeddings, documents, metadatas = [], [], [], []
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
chunk_id = f"{meta_base['source']}_{i:04d}"
|
||||
t0 = time.time()
|
||||
emb = embed(chunk)
|
||||
word_count = len(chunk.split())
|
||||
ids.append(chunk_id)
|
||||
embeddings.append(emb)
|
||||
documents.append(chunk)
|
||||
metadatas.append({
|
||||
**meta_base,
|
||||
"chunk_index": i,
|
||||
"total_chunks": len(chunks),
|
||||
"word_count": word_count,
|
||||
})
|
||||
print(f" chunk {i+1}/{len(chunks)} ({word_count}w) OK ({time.time()-t0:.1f}s)")
|
||||
|
||||
if len(ids) >= 50:
|
||||
add_documents(col_id, ids, embeddings, documents, metadatas)
|
||||
ids, embeddings, documents, metadatas = [], [], [], []
|
||||
|
||||
if ids:
|
||||
add_documents(col_id, ids, embeddings, documents, metadatas)
|
||||
|
||||
print(f"\n Indexé — {len(chunks)} chunks")
|
||||
state["ingested"].append(filename)
|
||||
# Retire des failed si présent
|
||||
state["failed"] = [f for f in state["failed"] if f.get("file") != filename]
|
||||
save_state(state)
|
||||
|
||||
except Exception as e:
|
||||
print(f" ERREUR: {e}")
|
||||
state["failed"].append({"file": filename, "error": str(e)})
|
||||
save_state(state)
|
||||
|
||||
print(f"\n=== Terminé : {len(state['ingested'])} PDFs indexés, {len(state['failed'])} échecs ===")
|
||||
for f in state.get("failed", []):
|
||||
print(f" ECHEC: {f['file']} — {f['error']}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,363 @@
|
||||
"""
|
||||
Ingestion MAO PDFs → ChromaDB
|
||||
Collection : mao_docs
|
||||
Embedding : nomic-embed-text (Ollama local)
|
||||
Chunking structurel : paragraphes + sections, ~450 mots, overlap 80 mots
|
||||
|
||||
Double usage :
|
||||
1. RAG assistant — répond aux questions sur les VSTs/plugins
|
||||
2. Accélération wiki — fournit les chunks pertinents au LLM pour générer les pages wiki
|
||||
(à la place de lire le PDF entier)
|
||||
"""
|
||||
import json, os, re, subprocess, tempfile, time, urllib.request, urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
PDFFETCH_URL = "http://localhost:15001/files"
|
||||
OLLAMA_EMBED = "http://localhost:11434/api/embeddings"
|
||||
CHROMA_URL = "http://localhost:8800"
|
||||
CHROMA_BASE = "/api/v2/tenants/default_tenant/databases/default_database"
|
||||
COLLECTION = "mao_docs"
|
||||
EMBED_MODEL = "nomic-embed-text"
|
||||
STATE_FILE = Path("/home/ubuntu/mao/ingest_state.json")
|
||||
|
||||
CHUNK_WORDS = 450
|
||||
OVERLAP_WORDS = 80
|
||||
MIN_WORDS = 40
|
||||
|
||||
# Correspondances nom de fichier → métadonnées produit
|
||||
# Enrichit la recherche : "comment fonctionne la reverb de Serum ?" → filtre type=synth, product=Serum
|
||||
PRODUCT_HINTS = {
|
||||
"serum": {"product": "Serum", "manufacturer": "Xfer", "type": "synth"},
|
||||
"massive": {"product": "Massive", "manufacturer": "Native Instruments", "type": "synth"},
|
||||
"vital": {"product": "Vital", "manufacturer": "Matt Tytel", "type": "synth"},
|
||||
"surge": {"product": "Surge XT", "manufacturer": "Surge Synth", "type": "synth"},
|
||||
"kontakt": {"product": "Kontakt", "manufacturer": "Native Instruments", "type": "sampler"},
|
||||
"battery": {"product": "Battery", "manufacturer": "Native Instruments", "type": "sampler"},
|
||||
"ableton": {"product": "Ableton Live", "manufacturer": "Ableton", "type": "daw"},
|
||||
"live": {"product": "Ableton Live", "manufacturer": "Ableton", "type": "daw"},
|
||||
"fl_studio": {"product": "FL Studio", "manufacturer": "Image-Line", "type": "daw"},
|
||||
"fl studio": {"product": "FL Studio", "manufacturer": "Image-Line", "type": "daw"},
|
||||
"logic": {"product": "Logic Pro", "manufacturer": "Apple", "type": "daw"},
|
||||
"cubase": {"product": "Cubase", "manufacturer": "Steinberg", "type": "daw"},
|
||||
"pro_tools": {"product": "Pro Tools", "manufacturer": "Avid", "type": "daw"},
|
||||
"reaper": {"product": "Reaper", "manufacturer": "Cockos", "type": "daw"},
|
||||
"izotope": {"product": "iZotope", "manufacturer": "iZotope", "type": "mastering"},
|
||||
"ozone": {"product": "Ozone", "manufacturer": "iZotope", "type": "mastering"},
|
||||
"neutron": {"product": "Neutron", "manufacturer": "iZotope", "type": "mixing"},
|
||||
"fabfilter": {"product": "FabFilter", "manufacturer": "FabFilter", "type": "effects"},
|
||||
"pro-q": {"product": "Pro-Q", "manufacturer": "FabFilter", "type": "eq"},
|
||||
"pro-r": {"product": "Pro-R", "manufacturer": "FabFilter", "type": "reverb"},
|
||||
"pro-c": {"product": "Pro-C", "manufacturer": "FabFilter", "type": "compressor"},
|
||||
"valhalla": {"product": "Valhalla", "manufacturer": "Valhalla DSP", "type": "reverb"},
|
||||
"waves": {"product": "Waves", "manufacturer": "Waves", "type": "effects"},
|
||||
"u-he": {"product": "u-he", "manufacturer": "u-he", "type": "synth"},
|
||||
"diva": {"product": "Diva", "manufacturer": "u-he", "type": "synth"},
|
||||
"hive": {"product": "Hive", "manufacturer": "u-he", "type": "synth"},
|
||||
"repro": {"product": "Repro", "manufacturer": "u-he", "type": "synth"},
|
||||
"zebra": {"product": "Zebra", "manufacturer": "u-he", "type": "synth"},
|
||||
"spire": {"product": "Spire", "manufacturer": "Reveal Sound", "type": "synth"},
|
||||
"sylenth": {"product": "Sylenth1", "manufacturer": "LennarDigital","type": "synth"},
|
||||
"omnisphere": {"product": "Omnisphere", "manufacturer": "Spectrasonics","type": "synth"},
|
||||
"nexus": {"product": "Nexus", "manufacturer": "refx", "type": "rompler"},
|
||||
"reaktor": {"product": "Reaktor", "manufacturer": "Native Instruments", "type": "modular"},
|
||||
"max_msp": {"product": "Max/MSP", "manufacturer": "Cycling '74", "type": "modular"},
|
||||
"soundtoys": {"product": "SoundToys", "manufacturer": "SoundToys", "type": "effects"},
|
||||
"rc-20": {"product": "RC-20", "manufacturer": "XLN Audio", "type": "effects"},
|
||||
"decimort": {"product": "Decimort", "manufacturer": "D16 Group", "type": "effects"},
|
||||
"d16": {"product": "D16 Group", "manufacturer": "D16 Group", "type": "effects"},
|
||||
}
|
||||
|
||||
|
||||
# ── État ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def load_state():
|
||||
if STATE_FILE.exists():
|
||||
return json.loads(STATE_FILE.read_text())
|
||||
return {"ingested": [], "failed": []}
|
||||
|
||||
def save_state(s):
|
||||
STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
STATE_FILE.write_text(json.dumps(s, indent=2))
|
||||
|
||||
|
||||
# ── ChromaDB ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def chroma_request(method, path, data=None):
|
||||
url = CHROMA_URL + path
|
||||
body = json.dumps(data).encode() if data else None
|
||||
req = urllib.request.Request(url, data=body, method=method,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
def get_or_create_collection():
|
||||
try:
|
||||
r = chroma_request("POST", f"{CHROMA_BASE}/collections", {
|
||||
"name": COLLECTION,
|
||||
"metadata": {"hnsw:space": "cosine"}
|
||||
})
|
||||
print(f"Collection créée : {COLLECTION} ({r['id']})")
|
||||
return r["id"]
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 409:
|
||||
cols = chroma_request("GET", f"{CHROMA_BASE}/collections")
|
||||
col_id = next(c["id"] for c in cols if c["name"] == COLLECTION)
|
||||
print(f"Collection existante : {COLLECTION} ({col_id})")
|
||||
return col_id
|
||||
raise
|
||||
|
||||
|
||||
def add_documents(col_id, ids, embeddings, documents, metadatas):
|
||||
chroma_request("POST", f"{CHROMA_BASE}/collections/{col_id}/add", {
|
||||
"ids": ids, "embeddings": embeddings,
|
||||
"documents": documents, "metadatas": metadatas
|
||||
})
|
||||
|
||||
|
||||
# ── Embeddings ────────────────────────────────────────────────────────────────
|
||||
|
||||
def embed(text):
|
||||
payload = json.dumps({"model": EMBED_MODEL, "prompt": text}).encode()
|
||||
req = urllib.request.Request(OLLAMA_EMBED, data=payload,
|
||||
headers={"Content-Type": "application/json"}, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.loads(r.read())["embedding"]
|
||||
|
||||
|
||||
# ── PDF helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
def list_mao_pdfs():
|
||||
req = urllib.request.Request(PDFFETCH_URL,
|
||||
headers={"User-Agent": "mao-ingest/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
all_files = json.loads(r.read())
|
||||
# PDFs MAO = racine (pas de sous-dossier "fallout/")
|
||||
return [f for f in all_files if not f.startswith("fallout/") and f.endswith(".pdf")]
|
||||
|
||||
def download_pdf(filename):
|
||||
url = f"{PDFFETCH_URL}/{urllib.parse.quote(filename)}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "mao-ingest/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=300) as r:
|
||||
return r.read()
|
||||
|
||||
def extract_text(pdf_bytes):
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
|
||||
f.write(pdf_bytes); tmp = f.name
|
||||
text = ""
|
||||
try:
|
||||
r = subprocess.run(["pdftotext", "-layout", tmp, "-"],
|
||||
capture_output=True, timeout=180)
|
||||
if r.returncode == 0:
|
||||
text = r.stdout.decode("utf-8", errors="replace")
|
||||
finally:
|
||||
os.unlink(tmp)
|
||||
if len(text.strip()) < 200:
|
||||
try:
|
||||
import io
|
||||
from pypdf import PdfReader
|
||||
reader = PdfReader(io.BytesIO(pdf_bytes))
|
||||
text = "\n".join(p.extract_text() or "" for p in reader.pages)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Extraction impossible: {e}")
|
||||
return text
|
||||
|
||||
|
||||
# ── Chunking structurel ───────────────────────────────────────────────────────
|
||||
|
||||
def is_section_header(line):
|
||||
l = line.strip()
|
||||
if not l or len(l) > 100:
|
||||
return False
|
||||
if l.isupper() and 2 <= len(l.split()) <= 10:
|
||||
return True
|
||||
if re.match(r'^(Chapter|Section|Part|Chapitre|Partie|\d+[\.\)])\s+\S', l, re.I):
|
||||
return True
|
||||
return False
|
||||
|
||||
def chunk_by_structure(text):
|
||||
text = re.sub(r'[ \t]+', ' ', text)
|
||||
text = re.sub(r'\n{3,}', '\n\n', text)
|
||||
|
||||
raw_blocks = re.split(r'\n{2,}', text)
|
||||
|
||||
blocks = []
|
||||
i = 0
|
||||
while i < len(raw_blocks):
|
||||
b = raw_blocks[i].strip()
|
||||
if not b:
|
||||
i += 1; continue
|
||||
if is_section_header(b) and i + 1 < len(raw_blocks):
|
||||
merged = b + "\n\n" + raw_blocks[i+1].strip()
|
||||
blocks.append(merged)
|
||||
i += 2
|
||||
else:
|
||||
blocks.append(b)
|
||||
i += 1
|
||||
|
||||
chunks = []
|
||||
current_blocks = []
|
||||
current_words = 0
|
||||
|
||||
for block in blocks:
|
||||
bwords = len(block.split())
|
||||
if bwords < 5:
|
||||
continue
|
||||
if current_words + bwords > CHUNK_WORDS and current_blocks:
|
||||
chunk = "\n\n".join(current_blocks).strip()
|
||||
if len(chunk.split()) >= MIN_WORDS:
|
||||
chunks.append(chunk)
|
||||
overlap_blocks = []
|
||||
overlap_words = 0
|
||||
for ob in reversed(current_blocks):
|
||||
ow = len(ob.split())
|
||||
if overlap_words + ow <= OVERLAP_WORDS:
|
||||
overlap_blocks.insert(0, ob)
|
||||
overlap_words += ow
|
||||
else:
|
||||
break
|
||||
current_blocks = overlap_blocks + [block]
|
||||
current_words = overlap_words + bwords
|
||||
else:
|
||||
current_blocks.append(block)
|
||||
current_words += bwords
|
||||
|
||||
if current_blocks:
|
||||
chunk = "\n\n".join(current_blocks).strip()
|
||||
if len(chunk.split()) >= MIN_WORDS:
|
||||
chunks.append(chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
# ── Métadonnées ───────────────────────────────────────────────────────────────
|
||||
|
||||
def infer_metadata(filename):
|
||||
f = filename.lower()
|
||||
name = Path(filename).name
|
||||
stem = Path(filename).stem.lower().replace("-", "_").replace(" ", "_")
|
||||
|
||||
meta = {
|
||||
"source": name,
|
||||
"path": filename,
|
||||
"product": "unknown",
|
||||
"manufacturer": "unknown",
|
||||
"type": "unknown",
|
||||
"language": "fr" if any(k in f for k in ["_fr", "francais", "manuel"]) else "en",
|
||||
}
|
||||
|
||||
for hint_key, hint_vals in PRODUCT_HINTS.items():
|
||||
if hint_key in f:
|
||||
meta.update(hint_vals)
|
||||
break
|
||||
|
||||
# Fallback : nom de fichier sans extension comme product
|
||||
if meta["product"] == "unknown":
|
||||
meta["product"] = Path(filename).stem[:40]
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
# ── Query helper (pour wiki pipeline) ────────────────────────────────────────
|
||||
|
||||
def query_for_wiki(col_id, product_name, top_k=15):
|
||||
"""
|
||||
Retrouve les chunks les plus pertinents pour générer une page wiki.
|
||||
Utiliser avec : query = "features workflow controls {product_name}"
|
||||
"""
|
||||
query = f"features workflow interface controls parameters {product_name}"
|
||||
emb = embed(query)
|
||||
result = chroma_request("POST", f"{CHROMA_BASE}/collections/{col_id}/query", {
|
||||
"query_embeddings": [emb],
|
||||
"n_results": top_k,
|
||||
"where": {"product": product_name},
|
||||
"include": ["documents", "metadatas", "distances"]
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
import sys
|
||||
# Mode : ingest (défaut) ou query
|
||||
mode = sys.argv[1] if len(sys.argv) > 1 else "ingest"
|
||||
|
||||
print(f"=== MAO ChromaDB — mode: {mode} ===")
|
||||
print(f"Collection : {COLLECTION}")
|
||||
print(f"Modèle embedding : {EMBED_MODEL}\n")
|
||||
|
||||
state = load_state()
|
||||
col_id = get_or_create_collection()
|
||||
|
||||
if mode == "query":
|
||||
product = sys.argv[2] if len(sys.argv) > 2 else "Serum"
|
||||
print(f"Query wiki pour : {product}")
|
||||
r = query_for_wiki(col_id, product)
|
||||
for i, (doc, meta) in enumerate(zip(r["documents"][0], r["metadatas"][0])):
|
||||
print(f"\n--- Chunk {i+1} [{meta['source']}] ---")
|
||||
print(doc[:300] + "...")
|
||||
return
|
||||
|
||||
pdfs = list_mao_pdfs()
|
||||
print(f"{len(pdfs)} PDFs MAO sur Guardian\n")
|
||||
|
||||
for filename in pdfs:
|
||||
if filename in state["ingested"]:
|
||||
print(f"[SKIP] {filename}")
|
||||
continue
|
||||
|
||||
print(f"\n[{filename}]")
|
||||
try:
|
||||
pdf_bytes = download_pdf(filename)
|
||||
print(f" {len(pdf_bytes)/1024:.0f} Ko téléchargés")
|
||||
|
||||
text = extract_text(pdf_bytes)
|
||||
print(f" {len(text)} chars extraits")
|
||||
|
||||
chunks = chunk_by_structure(text)
|
||||
meta_base = infer_metadata(filename)
|
||||
print(f" {len(chunks)} chunks | {meta_base['product']} ({meta_base['type']}) | lang:{meta_base['language']}")
|
||||
|
||||
ids, embeddings, documents, metadatas = [], [], [], []
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
chunk_id = f"{Path(filename).stem}_{i:04d}"
|
||||
t0 = time.time()
|
||||
emb = embed(chunk)
|
||||
word_count = len(chunk.split())
|
||||
ids.append(chunk_id)
|
||||
embeddings.append(emb)
|
||||
documents.append(chunk)
|
||||
metadatas.append({
|
||||
**meta_base,
|
||||
"chunk_index": i,
|
||||
"total_chunks": len(chunks),
|
||||
"word_count": word_count,
|
||||
})
|
||||
print(f" chunk {i+1}/{len(chunks)} ({word_count}w) OK ({time.time()-t0:.1f}s)")
|
||||
|
||||
if len(ids) >= 50:
|
||||
add_documents(col_id, ids, embeddings, documents, metadatas)
|
||||
ids, embeddings, documents, metadatas = [], [], [], []
|
||||
|
||||
if ids:
|
||||
add_documents(col_id, ids, embeddings, documents, metadatas)
|
||||
|
||||
print(f"\n Indexé — {len(chunks)} chunks")
|
||||
state["ingested"].append(filename)
|
||||
state["failed"] = [f for f in state["failed"] if f.get("file") != filename]
|
||||
save_state(state)
|
||||
|
||||
except Exception as e:
|
||||
print(f" ERREUR: {e}")
|
||||
state["failed"].append({"file": filename, "error": str(e)})
|
||||
save_state(state)
|
||||
|
||||
done = len(state["ingested"])
|
||||
failed = len(state["failed"])
|
||||
print(f"\n=== Terminé : {done} PDFs indexés, {failed} échecs ===")
|
||||
for f in state.get("failed", []):
|
||||
print(f" ECHEC: {f['file']} — {f['error']}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user