Corback
393 lines
18 KiB
Python
393 lines
18 KiB
Python
"""CoyoteOS Dashboard — API backend"""
|
|
import json, os, time, re, ssl, socket, subprocess, urllib.request, urllib.error
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
|
|
_ssl_ctx = ssl.create_default_context()
|
|
_ssl_ctx.check_hostname = False
|
|
_ssl_ctx.verify_mode = ssl.CERT_NONE
|
|
|
|
PROM = "http://prometheus:9090"
|
|
|
|
# Container groups for AdminSys
|
|
GROUPS = [
|
|
{"id": "infra", "name": "Infrastructure", "icon": "🔧", "protected": True,
|
|
"containers": ["nginx-proxy-manager","gitea","wiki-serve","prometheus","grafana","dash","node-exporter","chromadb"]},
|
|
{"id": "multimedia", "name": "Multimédia", "icon": "🎵", "protected": False,
|
|
"containers": ["navidrome","metube","beets"]},
|
|
{"id": "famille", "name": "Sites famille", "icon": "🏠", "protected": False,
|
|
"containers": ["voyage-famille","voyage-famille-test","CoyoteOS_db_container"]},
|
|
{"id": "corback", "name": "Corback Studio", "icon": "🎹", "protected": False,
|
|
"containers": ["corback-studio","php-fpm-corback"]},
|
|
{"id": "sites", "name": "Sites perso / PHP", "icon": "🌐", "protected": False,
|
|
"containers": ["coyote-site","php-fpm","php-fpm-test"]},
|
|
{"id": "fallout", "name": "Fallout JDR", "icon": "☢️", "protected": False,
|
|
"containers": ["chromadb"]},
|
|
]
|
|
|
|
# ── /proc helpers (Ampere) ─────────────────────────────────────────────────
|
|
def get_cpu():
|
|
def read():
|
|
with open("/proc/stat") as f: p = f.readline().split()
|
|
t = sum(int(x) for x in p[1:]); i = int(p[4]); return t, i
|
|
t1,i1 = read(); time.sleep(0.5); t2,i2 = read()
|
|
dt = t2-t1
|
|
return round((1-((i2-i1)/dt))*100,1) if dt>0 else 0.0
|
|
|
|
def get_mem():
|
|
d={}
|
|
with open("/proc/meminfo") as f:
|
|
for l in f:
|
|
k,v = l.split(":")[0], l.split(":")[1].strip().split()[0]
|
|
d[k]=int(v)
|
|
total=d["MemTotal"]*1024; avail=d["MemAvailable"]*1024; used=total-avail
|
|
return {"total":total,"used":used,"free":avail,"percent":round(used/total*100,1)}
|
|
|
|
def get_disk():
|
|
s=os.statvfs("/"); total=s.f_blocks*s.f_frsize; free=s.f_bavail*s.f_frsize; used=total-free
|
|
return {"total":total,"used":used,"free":free,"percent":round(used/total*100,1)}
|
|
|
|
def get_uptime():
|
|
with open("/proc/uptime") as f: secs=float(f.read().split()[0])
|
|
d=int(secs//86400); h=int((secs%86400)//3600); m=int((secs%3600)//60)
|
|
return {"seconds":int(secs),"days":d,"hours":h,"minutes":m,"human":f"{d}j {h}h {m}m"}
|
|
|
|
def get_network():
|
|
def read():
|
|
r={}
|
|
with open("/proc/net/dev") as f:
|
|
for l in f.readlines()[2:]:
|
|
p=l.split(); iface=p[0].rstrip(":")
|
|
if iface in ("lo",) or iface.startswith(("docker","br-","veth")): continue
|
|
r[iface]={"rx":int(p[1]),"tx":int(p[9])}
|
|
return r
|
|
n1=read(); time.sleep(1); n2=read()
|
|
rx=tx=0
|
|
for i in n1:
|
|
if i in n2: rx+=n2[i]["rx"]-n1[i]["rx"]; tx+=n2[i]["tx"]-n1[i]["tx"]
|
|
return {"rx_bps":max(0,rx),"tx_bps":max(0,tx)}
|
|
|
|
# ── Docker socket ──────────────────────────────────────────────────────────
|
|
def docker_call(method, path, data=None):
|
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
sock.settimeout(10)
|
|
sock.connect("/var/run/docker.sock")
|
|
body = json.dumps(data).encode() if data is not None else b""
|
|
hdr = (f"{method} {path} HTTP/1.0\r\nHost: localhost\r\n"
|
|
f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\n\r\n")
|
|
sock.sendall(hdr.encode() + body)
|
|
resp = b""
|
|
while True:
|
|
chunk = sock.recv(65536)
|
|
if not chunk: break
|
|
resp += chunk
|
|
sock.close()
|
|
status = int(resp.split(b" ")[1]) if resp else 0
|
|
rbody = resp.split(b"\r\n\r\n", 1)[1] if b"\r\n\r\n" in resp else b""
|
|
return status, rbody
|
|
|
|
def get_docker_containers():
|
|
try:
|
|
_, body = docker_call("GET", "/containers/json")
|
|
containers = json.loads(body)
|
|
return [{"name": c["Names"][0].lstrip("/"), "image": c["Image"].split(":")[0],
|
|
"status": c["Status"], "state": c["State"]} for c in containers]
|
|
except: return []
|
|
|
|
def docker_action_local(name, action):
|
|
"""action: start | stop | restart"""
|
|
status, _ = docker_call("POST", f"/containers/{name}/{action}", {})
|
|
return status in (200, 204, 304)
|
|
|
|
def docker_reboot_host():
|
|
"""Reboot Ampère via privileged container."""
|
|
docker_call("DELETE", "/containers/host-reboot?force=1")
|
|
config = {"Image": "alpine", "Cmd": ["nsenter","-t","1","-m","-u","-i","-n","-p","--","reboot"],
|
|
"HostConfig": {"Privileged": True, "PidMode": "host", "AutoRemove": True}}
|
|
status, body = docker_call("POST", "/containers/create?name=host-reboot", config)
|
|
if status in (201, 409):
|
|
docker_call("POST", "/containers/host-reboot/start", {})
|
|
return status in (201, 409)
|
|
|
|
def docker_run_host_cmd(name, cmd_list):
|
|
"""Run a command on the Ampère host via privileged nsenter container."""
|
|
docker_call("DELETE", f"/containers/{name}?force=1")
|
|
full_cmd = ["nsenter","-t","1","-m","-u","-i","-n","-p","--"] + cmd_list
|
|
config = {"Image": "alpine", "Cmd": full_cmd,
|
|
"HostConfig": {"Privileged": True, "PidMode": "host", "AutoRemove": True}}
|
|
status, body = docker_call("POST", f"/containers/create?name={name}", config)
|
|
if status in (201, 409):
|
|
docker_call("POST", f"/containers/{name}/start", {})
|
|
return status in (201, 409)
|
|
|
|
# ── SSH helpers (Guardian / Vigile) ───────────────────────────────────────
|
|
SSH_KEYS = {
|
|
"guardian": {"key": "/ssh/guardian.key", "host": "82.70.226.48"},
|
|
"vigile": {"key": "/ssh/vigile.key", "host": "79.72.30.231"},
|
|
}
|
|
|
|
def ssh_run(vm, cmd, timeout=10):
|
|
cfg = SSH_KEYS.get(vm)
|
|
if not cfg: return False, "VM inconnue"
|
|
try:
|
|
r = subprocess.run(
|
|
["ssh", "-i", cfg["key"], "-o", "StrictHostKeyChecking=no",
|
|
"-o", "BatchMode=yes", "-o", f"ConnectTimeout={timeout}",
|
|
f"ubuntu@{cfg['host']}", cmd],
|
|
capture_output=True, timeout=timeout+2)
|
|
return r.returncode == 0, r.stdout.decode().strip() or r.stderr.decode().strip()
|
|
except Exception as e:
|
|
return False, str(e)
|
|
|
|
def docker_action_remote(vm, name, action):
|
|
ok, out = ssh_run(vm, f"sudo docker {action} {name}")
|
|
return ok
|
|
|
|
def ssh_reboot(vm):
|
|
ok, _ = ssh_run(vm, "sudo reboot", timeout=5)
|
|
return ok
|
|
|
|
# ── Ollama ────────────────────────────────────────────────────────────────
|
|
OLLAMA_URL = "http://172.20.0.1:11434"
|
|
|
|
def get_ollama_ps():
|
|
try:
|
|
with urllib.request.urlopen(OLLAMA_URL+"/api/ps", timeout=3) as r:
|
|
d = json.loads(r.read())
|
|
return [{"name": m["name"], "size_gb": round(m.get("size",0)/1e9,1),
|
|
"expires": m.get("expires_at","")} for m in d.get("models",[])]
|
|
except: return []
|
|
|
|
# ── Pipeline helpers ───────────────────────────────────────────────────────
|
|
def _log_running(path, max_age=120):
|
|
try: return (time.time() - os.path.getmtime(path)) < max_age
|
|
except: return False
|
|
|
|
def get_pipeline():
|
|
state={"processed":[],"failed":[]}
|
|
try:
|
|
with open("/data/pipeline_state.json") as f: state=json.load(f)
|
|
except: pass
|
|
running = _log_running("/data/pipeline.log")
|
|
current_pdf=None; current_chunk=None; total_chunks=None
|
|
chunks_done=[]; log_tail=[]
|
|
try:
|
|
with open("/data/pipeline.log") as f: lines=f.readlines()
|
|
log_tail=[l.rstrip() for l in lines[-40:] if l.strip()]
|
|
if running:
|
|
for l in reversed(lines):
|
|
l=l.rstrip()
|
|
if not current_pdf and re.search(r"^\[.+\.pdf\]",l):
|
|
current_pdf=l.strip()[1:-1]
|
|
m=re.search(r"chunk (\d+)/(\d+)",l)
|
|
if m and current_chunk is None:
|
|
current_chunk=int(m.group(1)); total_chunks=int(m.group(2))
|
|
m2=re.search(r"chunk \d+/\d+.*OK \((\d+)s",l)
|
|
if m2: chunks_done.append(int(m2.group(1)))
|
|
if current_pdf and current_chunk and len(chunks_done)>=1: break
|
|
except: pass
|
|
last_t=chunks_done[-1] if chunks_done else None
|
|
avg_t=round(sum(chunks_done)/len(chunks_done)) if chunks_done else None
|
|
eta=None
|
|
if current_chunk and total_chunks and last_t:
|
|
rem=(total_chunks-current_chunk)*last_t
|
|
eta=f"{rem//3600}h{(rem%3600)//60}m" if rem>=3600 else f"{rem//60}m"
|
|
return {"running":running,"done":len(state.get("processed",[])),"failed":len(state.get("failed",[])),
|
|
"current_pdf":current_pdf if running else None,"current_chunk":current_chunk,
|
|
"total_chunks":total_chunks,"avg_chunk_s":avg_t,"last_chunk_s":last_t,
|
|
"eta_current_pdf":eta,"log_tail":log_tail}
|
|
|
|
def get_fallout_ingest():
|
|
state={"ingested":[],"failed":[]}
|
|
try:
|
|
with open("/data/fallout_state.json") as f: state=json.load(f)
|
|
except: pass
|
|
# Préfère le log v2 s'il existe, sinon log v1
|
|
import os as _os
|
|
log_v2 = "/data/fallout_ingest_v2.log"
|
|
log_v1 = "/data/fallout_ingest.log"
|
|
log_path = log_v2 if _os.path.exists(log_v2) else log_v1
|
|
running = _log_running(log_path)
|
|
current_pdf=None; current_chunk=None; total_chunks=None
|
|
chunks_done=[]; log_tail=[]; version="v2" if log_path==log_v2 else "v1"
|
|
try:
|
|
with open(log_path) as f: lines=f.readlines()
|
|
log_tail=[l.rstrip() for l in lines[-30:] if l.strip()]
|
|
for l in reversed(lines):
|
|
l=l.rstrip()
|
|
if not current_pdf and re.search(r"^\[fallout/.+\.pdf\]",l):
|
|
current_pdf=l.strip()[1:-1]
|
|
m=re.search(r"chunk (\d+)/(\d+)",l)
|
|
if m and current_chunk is None:
|
|
current_chunk=int(m.group(1)); total_chunks=int(m.group(2))
|
|
m2=re.search(r"chunk \d+/\d+.*OK \((\d+\.?\d*)s",l)
|
|
if m2: chunks_done.append(float(m2.group(1)))
|
|
except: pass
|
|
avg_t=round(sum(chunks_done)/len(chunks_done),1) if chunks_done else None
|
|
return {"running":running,"version":version,"done":len(state.get("ingested",[])),"failed":len(state.get("failed",[])),
|
|
"total":17,"current_pdf":current_pdf if running else None,
|
|
"current_chunk":current_chunk,"total_chunks":total_chunks,"avg_embed_s":avg_t,"log_tail":log_tail}
|
|
|
|
# ── Prometheus ─────────────────────────────────────────────────────────────
|
|
def prom_query(q):
|
|
try:
|
|
url = PROM+"/api/v1/query?query="+urllib.request.quote(q)
|
|
with urllib.request.urlopen(url, timeout=4) as r:
|
|
d=json.loads(r.read()); res=d.get("data",{}).get("result",[])
|
|
if res: return float(res[0]["value"][1])
|
|
except: pass
|
|
return None
|
|
|
|
def prom_query_all(q):
|
|
try:
|
|
url = PROM+"/api/v1/query?query="+urllib.request.quote(q)
|
|
with urllib.request.urlopen(url, timeout=4) as r:
|
|
d=json.loads(r.read()); return d.get("data",{}).get("result",[])
|
|
except: return []
|
|
|
|
def get_vm_metrics(instance):
|
|
cpu = prom_query(f'100-(avg by(instance)(rate(node_cpu_seconds_total{{mode="idle",instance="{instance}"}}[2m]))*100)')
|
|
mem_total = prom_query(f'node_memory_MemTotal_bytes{{instance="{instance}"}}')
|
|
mem_avail = prom_query(f'node_memory_MemAvailable_bytes{{instance="{instance}"}}')
|
|
disk_size = prom_query(f'node_filesystem_size_bytes{{instance="{instance}",mountpoint="/",fstype!="tmpfs"}}')
|
|
disk_avail = prom_query(f'node_filesystem_avail_bytes{{instance="{instance}",mountpoint="/",fstype!="tmpfs"}}')
|
|
uptime_s = prom_query(f'time()-node_boot_time_seconds{{instance="{instance}"}}')
|
|
mem_pct = round((1 - mem_avail/mem_total)*100, 1) if mem_total and mem_avail else None
|
|
disk_pct = round((1 - disk_avail/disk_size)*100, 1) if disk_size and disk_avail else None
|
|
up_h = int(uptime_s//3600) if uptime_s else None
|
|
up_d = int(uptime_s//86400) if uptime_s else None
|
|
return {
|
|
"cpu": round(cpu,1) if cpu else None,
|
|
"mem_pct": mem_pct,
|
|
"mem_total_gb": round(mem_total/1e9,1) if mem_total else None,
|
|
"disk_pct": disk_pct,
|
|
"uptime_human": f"{up_d}j {(up_h or 0)%24}h" if uptime_s else None,
|
|
"available": cpu is not None
|
|
}
|
|
|
|
def get_summary30():
|
|
rx=prom_query('sum(increase(node_network_receive_bytes_total{device!="lo"}[30d]))')
|
|
tx=prom_query('sum(increase(node_network_transmit_bytes_total{device!="lo"}[30d]))')
|
|
cpu=prom_query('avg(100-(avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[30d]))*100))')
|
|
total=(rx or 0)+(tx or 0)
|
|
return {"rx_30d":rx,"tx_30d":tx,"total_bw":total if total>0 else None,
|
|
"avg_cpu_30d":round(cpu,1) if cpu else None}
|
|
|
|
# ── Sites ─────────────────────────────────────────────────────────────────
|
|
SITES = [
|
|
{"name":"CoyoteOS", "url":"http://coyoteos.ovh", "icon":"globe"},
|
|
{"name":"Wiki MAO", "url":"https://wiki.coyoteos.ovh", "icon":"book"},
|
|
{"name":"Gitea", "url":"https://git.coyoteos.ovh", "icon":"git"},
|
|
{"name":"Grafana", "url":"http://grafana.coyoteos.ovh", "icon":"chart"},
|
|
{"name":"Navidrome", "url":"http://music.coyoteos.ovh", "icon":"music"},
|
|
{"name":"Studio", "url":"http://studio.coyoteos.ovh", "icon":"studio"},
|
|
{"name":"Dashboard", "url":"http://dash.coyoteos.ovh", "icon":"dash"},
|
|
]
|
|
_site_cache={"ts":0,"data":[]}
|
|
|
|
def check_sites():
|
|
now=time.time()
|
|
if now-_site_cache["ts"]<30: return _site_cache["data"]
|
|
results=[]
|
|
for s in SITES:
|
|
if "dash.coyoteos.ovh" in s["url"]:
|
|
results.append({**s,"status":200,"up":True}); continue
|
|
try:
|
|
req=urllib.request.Request(s["url"],method="HEAD")
|
|
req.add_header("User-Agent","CoyoteOS-Dash/1.0")
|
|
with urllib.request.urlopen(req, timeout=6, context=_ssl_ctx) as r:
|
|
results.append({**s,"status":r.status,"up":r.status<400})
|
|
except urllib.error.HTTPError as e:
|
|
results.append({**s,"status":e.code,"up":e.code<400})
|
|
except Exception as e:
|
|
results.append({**s,"status":0,"up":False,"error":str(e)[:60]})
|
|
_site_cache["ts"]=now; _site_cache["data"]=results
|
|
return results
|
|
|
|
# ── HTTP Server ────────────────────────────────────────────────────────────
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self,*a): pass
|
|
|
|
def send_json(self, data, code=200):
|
|
body=json.dumps(data,default=str).encode()
|
|
self.send_response(code)
|
|
self.send_header("Content-Type","application/json")
|
|
self.send_header("Access-Control-Allow-Origin","*")
|
|
self.end_headers(); self.wfile.write(body)
|
|
|
|
def read_body(self):
|
|
length = int(self.headers.get("Content-Length",0))
|
|
return json.loads(self.rfile.read(length)) if length else {}
|
|
|
|
def do_OPTIONS(self):
|
|
self.send_response(204)
|
|
self.send_header("Access-Control-Allow-Origin","*")
|
|
self.send_header("Access-Control-Allow-Methods","GET,POST")
|
|
self.send_header("Access-Control-Allow-Headers","Content-Type")
|
|
self.end_headers()
|
|
|
|
def do_GET(self):
|
|
p = self.path.split("?")[0]
|
|
if p == "/metrics":
|
|
self.send_json({"cpu":{"percent":get_cpu()},"memory":get_mem(),
|
|
"disk":get_disk(),"uptime":get_uptime(),"network":get_network(),
|
|
"timestamp":int(time.time())})
|
|
elif p == "/pipeline": self.send_json(get_pipeline())
|
|
elif p == "/fallout-ingest": self.send_json(get_fallout_ingest())
|
|
elif p == "/docker-ps": self.send_json(get_docker_containers())
|
|
elif p == "/groups": self.send_json(GROUPS)
|
|
elif p == "/ollama-ps": self.send_json(get_ollama_ps())
|
|
elif p == "/sites": self.send_json(check_sites())
|
|
elif p == "/summary30": self.send_json(get_summary30())
|
|
elif p == "/vm-metrics":
|
|
qs = self.path[self.path.find("?")+1:] if "?" in self.path else ""
|
|
params = dict(kv.split("=") for kv in qs.split("&") if "=" in kv)
|
|
inst = params.get("instance","Guardian")
|
|
self.send_json(get_vm_metrics(inst))
|
|
elif p in ("/", "/index.html"):
|
|
with open("/app/index.html","rb") as f: body=f.read()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type","text/html; charset=utf-8")
|
|
self.end_headers(); self.wfile.write(body)
|
|
else:
|
|
self.send_response(404); self.end_headers()
|
|
|
|
def do_POST(self):
|
|
p = self.path
|
|
if p == "/admin/docker":
|
|
body = self.read_body()
|
|
vm = body.get("vm","ampere")
|
|
name = body.get("container","")
|
|
action = body.get("action","stop") # start | stop | restart
|
|
if not name: self.send_json({"ok":False,"error":"no container"}, 400); return
|
|
if vm == "ampere":
|
|
ok = docker_action_local(name, action)
|
|
else:
|
|
ok = docker_action_remote(vm, name, action)
|
|
self.send_json({"ok":ok,"vm":vm,"container":name,"action":action})
|
|
|
|
elif p == "/admin/reboot":
|
|
body = self.read_body()
|
|
vm = body.get("vm","ampere")
|
|
if vm == "ampere":
|
|
ok = docker_reboot_host()
|
|
else:
|
|
ok = ssh_reboot(vm)
|
|
self.send_json({"ok":ok,"vm":vm})
|
|
|
|
elif p == "/admin/run-ingest":
|
|
body = self.read_body()
|
|
script = body.get("script","fallout")
|
|
if script == "fallout":
|
|
cmd = ["sh","-c","PYTHONUNBUFFERED=1 python3 -u /home/ubuntu/fallout/ingest.py >> /home/ubuntu/fallout/ingest.log 2>&1 &"]
|
|
else:
|
|
self.send_json({"ok":False,"error":"unknown script"}); return
|
|
ok = docker_run_host_cmd("host-ingest", cmd)
|
|
self.send_json({"ok":ok,"script":script})
|
|
|
|
else:
|
|
self.send_response(404); self.end_headers()
|
|
|
|
if __name__=="__main__":
|
|
print("CoyoteOS Dashboard :5000")
|
|
HTTPServer(("0.0.0.0",5000),Handler).serve_forever()
|