# AEON Bench — POD appliance (the controlled A→B benchmark you run on YOUR GPU host). # # TWO WAYS TO USE THIS FILE: # # 1. INFRASTRUCTURE UP (default — no model needed): # docker compose -f deploy/pod/docker-compose.yml up -d --build # Brings up the POD DASHBOARD on http://localhost:8091 and builds the three agentic-harness # images. Benchmarks are then launched FROM THE GUI (Run tab: validate a model by hash, pick # an engine container, tune the recipe, launch) or via the pod API. This is the intended # day-to-day mode. (No clone at all? The prebuilt image one-liner in README.md does the same.) # # 2. HEADLESS ONE-SHOT PIPELINE (optional, explicit): # AEON_HF_LINK=org/Your-Model docker compose --profile pipeline -f deploy/pod/docker-compose.yml up --build # The classic A→B batch flow: pull → hash-verify → serve → bench → submit, then exit. # Only THIS profile requires AEON_HF_LINK. # # The moving parts (see AGENTS.md for detail): # pod-dashboard the GUI + API + control plane [default] # harness-build-* build the 3 vanilla agent harnesses (Hermes/OpenClaw/OpenCode) the agentic # suite is driven THROUGH — NOT long-running services: the pod spawns a FRESH # one-shot `docker run --rm` harness container per task via the mounted # Docker socket (exact image names the adapters run) [default] # pull / model-under-test / aeon-pod the headless batch pipeline [--profile pipeline] # # Build context is the repo root (`../..`) so the pod image can COPY mvp/. name: aeon-pod services: # ── The served model under test ("B") ────────────────────────────────────────────────────── # vLLM serves the local weights on the FIXED alias `model-under-test` (modelhost.DEFAULT_ALIAS) # — the pod + every harness talk to this alias, never to a raw model name. On a DGX Spark the # pod's recipe defaults the ENGINE to `aeon-vllm-ultimate`; this compose ships vanilla vLLM as # the portable default and documents the swap below. # # network_mode: host so the port is on 127.0.0.1:8000 of the HOST — that is the single address # the pod AND the per-task harness containers (which the pod launches with `--network host`, see # the adapters) all reach the model on. (With a bridge network the host-net harness containers # could not resolve a compose service name, so everything that talks to the model uses host net.) # # The weights are pulled+verified by the `pull` step (below) into the shared volume; vLLM then # serves them from disk. AEON_HF_LINK selects the model. model-under-test: profiles: ["pipeline"] # headless one-shot only — GUI runs serve their own engine # vllm/vllm-openai:latest is the upstream OpenAI-compatible server image; pin a concrete tag # for strict reproducibility. On a DGX Spark (GB10/arm64) use the aeon-vllm-ultimate image # instead (see the commented block below) — that is the pod's DGX-Spark default engine. image: ${AEON_VLLM_IMAGE:-vllm/vllm-openai:latest} depends_on: pull: condition: service_completed_successfully network_mode: host # serve on 127.0.0.1:${AEON_TARGET_PORT:-8000} command: - "--model" - "/weights" # the verified local snapshot (see `pull`) - "--served-model-name" - "model-under-test" # FIXED alias — must match modelhost.DEFAULT_ALIAS - "--host" - "0.0.0.0" - "--port" - "${AEON_TARGET_PORT:-8000}" # --max-model-len / --quantization are model-dependent; the pod's derive_recipe() computes # them from config.json. Set AEON_MAX_MODEL_LEN / AEON_QUANT in .env to override here. # NOTE: the agentic suite needs a served context >=64K (the Hermes harness refuses smaller); # serve with --max-model-len 65536 or higher. TODO verify model-specific serve flags # (tensor-parallel, dtype, trust-remote-code). environment: HF_HOME: /hf-cache volumes: - weights:/weights:ro # verified weights (read-only to the engine) - hf-cache:/hf-cache # GPU access via the NVIDIA Container Toolkit. deploy: resources: reservations: devices: - driver: nvidia count: all capabilities: ["gpu"] healthcheck: test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:${AEON_TARGET_PORT:-8000}/v1/models || exit 1"] interval: 15s timeout: 5s retries: 20 start_period: 120s # large-weight load can be slow # ── DGX Spark default engine (aeon-vllm-ultimate) ─────────────────────────────────────────── # On a DGX Spark, replace the `image:` above with the first-party signed engine image and set # AEON_SYSTEM=dgx-spark in .env (modelhost.is_dgx_spark / derive_recipe default it). It serves # the SAME `model-under-test` alias on :8000, so nothing else changes. Example: # # model-under-test: # image: ${AEON_ULTIMATE_IMAGE:-ghcr.io/aeon-7/aeon-vllm-ultimate:latest} # command: ["serve", "/weights", "--served-model-name", "model-under-test", # "--host", "0.0.0.0", "--port", "8000", "--max-model-len", "${AEON_MAX_MODEL_LEN:-65536}"] # (cosign-verify the digest first — see trust-architecture §2.2 first-party exception.) # ── Weight pull + hash-verify (modelhost) ─────────────────────────────────────────────────── # One-shot: resolve the HF link → pull the snapshot (hub-verified per file) → hash-verify into a # content-addressed weights_hash and compare to HF's published LFS sha256 (the SIGNATURE that the # bytes on disk ARE exactly repo@sha). Lands the verified snapshot in the shared `weights` volume. pull: profiles: ["pipeline"] # headless one-shot only build: context: ../.. # repo root (Dockerfile COPYs mvp/) dockerfile: deploy/pod/Dockerfile image: aeon-pod:latest # built once, reused by pull / aeon-pod / pod-dashboard entrypoint: ["python", "-c"] command: - | import os, json, sys sys.path.insert(0, "/app/mvp") from pod import modelhost as mh link = os.environ.get("AEON_HF_LINK") or "" if not link: print("[pull] AEON_HF_LINK is required for the pipeline profile: " "AEON_HF_LINK=org/Your-Model docker compose --profile pipeline up", file=sys.stderr) sys.exit(2) repo, rev = mh.resolve(link) print(f"[pull] {repo}@{rev}", flush=True) ref = mh.fetch_ref(repo, rev) mh.pull(repo, ref.get("revision") or rev, "/weights") v = mh.verify("/weights", ref) print("[verify]", json.dumps({k: v[k] for k in ("verified","method","weights_hash","revision","n_weight_files","lfs_checked","mismatches")}), flush=True) if not v["verified"]: print("[verify] WEIGHTS DID NOT VERIFY — refusing to serve", file=sys.stderr); sys.exit(1) img = os.environ.get("AEON_ENGINE_IMAGE") or os.environ.get("AEON_VLLM_IMAGE") or None json.dump({"repo": repo, "revision": ref.get("revision") or rev, "verification": v, "recipe": mh.derive_recipe("/weights", ref, engine=os.environ.get("AEON_ENGINE") or None, image=img)}, open("/weights/.aeon-modelref.json", "w")) environment: AEON_HF_LINK: ${AEON_HF_LINK:-} # required for THIS profile; the pull step errors clearly if empty AEON_SYSTEM: ${AEON_SYSTEM:-} # set to dgx-spark to force the aeon-vllm-ultimate recipe AEON_ENGINE: ${AEON_ENGINE:-} # pin engine: vllm | aeon-vllm-ultimate | llama.cpp AEON_ENGINE_IMAGE: ${AEON_ENGINE_IMAGE:-} # explicit any-engine image override (recorded AND served) # NO default here: an always-set image would override the engine catalog's image in the # derived recipe (build_serve: img = image or catalog), recording — and SIGNING — # vllm/vllm-openai:latest as engine provenance even when the real engine is # aeon-vllm-ultimate or llama.cpp. Empty collapses to None in the sidecar (`or` chain), # so the catalog image is recorded unless the operator explicitly overrides. AEON_VLLM_IMAGE: ${AEON_VLLM_IMAGE:-} HF_HOME: /hf-cache HF_TOKEN: ${HF_TOKEN:-} # needed for gated/private repos volumes: - weights:/weights - hf-cache:/hf-cache # ── Harness images (built once; the pod launches them per-task) ───────────────────────────── # The pod is the CONTROL PLANE for the agentic suite: for each task it does a fresh, one-shot # `docker run --rm` of the relevant harness image (mvp/pod/adapters/{hermes,openclaw,opencode}.py # → run_harness2.py), `docker cp`s the outputs back, and tears the container down. So there are # NO persistent harness services — but the images must exist locally under the EXACT names the # adapters run. The three `harness-build-*` services below build those images (via `image:` + # `build:`) then exit immediately (`command: ["true"]`); `aeon-pod` depends on their completion. # To rebuild only the harnesses: docker compose build harness-build-hermes harness-build-openclaw harness-build-opencode harness-build-hermes: image: aeon-harness-hermes # == HermesAdapter.IMAGE (AEON_HERMES_IMAGE default) build: context: . dockerfile: harness-hermes.Dockerfile args: HERMES_REF: ${AEON_HERMES_REF:-main} # git ref of NousResearch/hermes-agent to pin entrypoint: ["true"] # build-only: never actually runs as a service restart: "no" harness-build-openclaw: image: aeon-harness-openclaw # == OpenClawAdapter.IMAGE (AEON_OPENCLAW_IMAGE default) build: context: . dockerfile: harness-openclaw.Dockerfile args: OPENCLAW_VERSION: ${AEON_OPENCLAW_VERSION:-0.17.0} # TODO verify pinned version exists entrypoint: ["true"] restart: "no" harness-build-opencode: image: aeon-harness-opencode # == OpenCodeAdapter.IMAGE (AEON_OPENCODE_IMAGE default) build: context: . dockerfile: harness-opencode.Dockerfile args: OPENCODE_VERSION: ${AEON_OPENCODE_VERSION:-0.3.0} # TODO verify pinned version exists entrypoint: ["true"] restart: "no" # ── The pod control plane ("A") ───────────────────────────────────────────────────────────── # Benchmarks the served alias and submits the signed bundle. Runs once and exits. It drives the # agentic suite by launching the harness images above as one-shot containers through the mounted # Docker socket — so it needs the docker CLI + /var/run/docker.sock, and it runs on the HOST # network so it (and the host-net harness containers it spawns) reach the model on 127.0.0.1. # The flags map 1:1 to aeon_pod.py. The device key + local pod.db persist in `pod-state` (~/.aeon). aeon-pod: profiles: ["pipeline"] # headless one-shot only image: aeon-pod:latest build: context: ../.. dockerfile: deploy/pod/Dockerfile depends_on: pull: # the verified weights + .aeon-modelref.json must exist first condition: service_completed_successfully model-under-test: condition: service_healthy # the harness IMAGES must be built before the pod launches per-task containers from them harness-build-hermes: {condition: service_completed_successfully} harness-build-openclaw: {condition: service_completed_successfully} harness-build-opencode: {condition: service_completed_successfully} network_mode: host # reach the model + spawn host-net harness containers on 127.0.0.1 entrypoint: ["/bin/sh", "-c"] command: # --modelref makes this an ATTESTED-eligible run: it carries the pull sidecar's weights_hash # + per-file hashes + recipe so the mothership re-verifies against HF before ranking. (Swap # to --target/--model alone for a self_reported LOCAL run.) - | exec python -m pod.aeon_pod \ --target "http://127.0.0.1:$${AEON_TARGET_PORT:-8000}/v1" \ --modelref "/weights/.aeon-modelref.json" \ --mothership "$${AEON_MOTHERSHIP}" \ --harness all \ $${AEON_HARDWARE:+--hardware "$${AEON_HARDWARE}"} \ $${AEON_JUDGE:+--judge "$${AEON_JUDGE}"} \ $${AEON_JUDGE_URL:+--judge-url "$${AEON_JUDGE_URL}"} \ $${AEON_JUDGE_KEY:+--judge-key "$${AEON_JUDGE_KEY}"} \ $${AEON_MAX_TOKENS:+--max-tokens "$${AEON_MAX_TOKENS}"} \ $${AEON_LIMIT:+--limit "$${AEON_LIMIT}"} environment: AEON_MOTHERSHIP: ${AEON_MOTHERSHIP:-https://aeon-bench.com} # prod default; override in .env only to test elsewhere AEON_HF_LINK: ${AEON_HF_LINK:-} # the pipeline's one required input (org/Model or a URL) AEON_TARGET_PORT: ${AEON_TARGET_PORT:-8000} AEON_ENGINE: ${AEON_ENGINE:-} # vllm | aeon-vllm-ultimate (DGX default) | llama.cpp AEON_HARDWARE: ${AEON_HARDWARE:-} # label, e.g. "NVIDIA DGX Spark GB10 128GB" AEON_SYSTEM: ${AEON_SYSTEM:-} # dgx-spark to force the ultimate recipe AEON_JUDGE: ${AEON_JUDGE:-} # FRONTIER judge id, or empty = deterministic-only (never self-judge) AEON_JUDGE_URL: ${AEON_JUDGE_URL:-} AEON_JUDGE_KEY: ${AEON_JUDGE_KEY:-} AEON_MAX_TOKENS: ${AEON_MAX_TOKENS:-32768} # generation cap incl. hidden reasoning tokens AEON_LIMIT: ${AEON_LIMIT:-} # first N cases only (quick smoke); empty = full suite AEON_KEEP_WEIGHTS: ${AEON_KEEP_WEIGHTS:-1} # keep verified weights after the run (volume persists regardless) AEON_TARGET_API_KEY: ${AEON_TARGET_API_KEY:-sk-local} # Harness image names the adapters `docker run` — kept in sync with the images built above. AEON_HERMES_IMAGE: aeon-harness-hermes AEON_OPENCLAW_IMAGE: aeon-harness-openclaw AEON_OPENCODE_IMAGE: aeon-harness-opencode volumes: - pod-state:/root/.aeon # ed25519 device key (enrolled once) + local pod.db - weights:/weights # read the pull sidecar's .aeon-modelref.json (verification) - /var/run/docker.sock:/var/run/docker.sock # spawn per-task harness containers # Optional: mount a generated suite corpus so it overrides the built-ins: # - ./suites:/app/mvp/suites:ro restart: "no" # ── The POD DASHBOARD (default service — this IS the pod) ────────────────────────────────── # GUI + API + control plane on http://localhost:8091. Benchmarks are launched from its Run tab # (or the pod API): validate a model by hash (paste an HF link / scan disk / browse), pick the # engine container, tune the recipe, launch — the dashboard spawns the engine + per-task # harness containers as SIBLINGS through the mounted Docker socket. Local only — nothing here # is public; the mothership shows a run only once an attested submission is accepted. pod-dashboard: image: aeon-pod:latest build: context: ../.. dockerfile: deploy/pod/Dockerfile environment: AEON_ROLE: pod # your lab: Run tab + Live view + your own runs AEON_DB: /root/.aeon/pod.db # SAME SQLite the pipeline profile writes (WAL-safe) AEON_PORT: "8091" # Submissions go to prod by default; override in .env only to test elsewhere. AEON_MOTHERSHIP: ${AEON_MOTHERSHIP:-https://aeon-bench.com} # Validated weights live here; sibling engine containers need the HOST path for -v mounts. AEON_MODELS_DIR: /models AEON_MODELS_HOST_DIR: ${AEON_MODELS_HOST:-${HOME}/aeon-models} # Read-only host-home scan: lets the Run tab find HF cache, LM Studio, ~/models, etc. AEON_HOST_HOME_DIR: ${AEON_HOST_HOME:-${HOME}} # Harness image names the Run tab's launches spawn (built by harness-build-* below). AEON_HERMES_IMAGE: aeon-harness-hermes AEON_OPENCLAW_IMAGE: aeon-harness-openclaw AEON_OPENCODE_IMAGE: aeon-harness-opencode volumes: - pod-state:/root/.aeon # device key + pod.db (persists across updates) - /var/run/docker.sock:/var/run/docker.sock # spawn engine + per-task harness containers - ${AEON_MODELS_HOST:-${HOME}/aeon-models}:/models # validated weights (host-visible) - ${AEON_HOST_HOME:-${HOME}}:/host-home:ro # full-host model scan, read-only ports: - "${AEON_DASH_PORT:-8091}:8091" # the pod dashboard # NVIDIA hosts: give the dashboard GPU visibility (platform detection drives the engine # catalog — without it CUDA engines disable themselves). Uncomment on GPU boxes: # deploy: # resources: # reservations: # devices: # - driver: nvidia # count: all # capabilities: ["gpu"] restart: unless-stopped volumes: weights: # verified model snapshot, shared pull → engine hf-cache: # huggingface_hub download cache pod-state: # ~/.aeon — device key + local SQLite dashboard (PERSIST to keep your enrolled key)