"""Turn-based E2E inference: one utterance (wav + emotion) -> face(52) + body(vrma). This is Phase A — the offline core the OpenAI Realtime bridge (Phase C) will call on every `response.done`. No mic, no corpusify: the audio is treated as the real assistant voice it will be in production. face : V3 face model (face_v3_infer.render_face, in-process, warm) body : kemix velocity codec+flow (persistent warm worker, separate process) Writes outputs/turns//{face.json, body.vrma, audio.wav, meta.json} and the combined viewer (Phase B) reads that folder. python infer_turn.py --wav <16k mono wav> --emotion excitement --id demo1 """ import argparse import base64 import json import os import shutil import subprocess import sys import threading import time import wave import numpy as np import torch HERE = os.path.dirname(os.path.abspath(__file__)) KEMIX = os.path.abspath(os.path.join(HERE, "..")) # package/ WH = os.path.join(KEMIX, "motion-blender", "experiments", "wave-hands") GVRM = os.path.join(KEMIX, "motion", "GestureVRM") PY = sys.executable FPS = 30 # Body model wiring. Keep the existing Tag3 model as the default; opt into the # isolated scratch-trained solo model with AVATAR_BODY_PROFILE=solo. BODY_PROFILE = os.environ.get("AVATAR_BODY_PROFILE", "v4").strip().lower() if BODY_PROFILE == "solo": # AVATAR_BODY_RUN으로 런을 고른다. r01은 44본 + 오염 데이터(폐기, 증거 보존). SOLO_RUN = os.path.join( WH, "outputs", "gen_train", "runs", os.environ.get("AVATAR_BODY_RUN", "solo_body_v1__50b_vel__scratch__r03")) BODY_CKPT = os.path.join(SOLO_RUN, "generator", "best.pth") VQVAE_DIR = SOLO_RUN MEANSTD_DIR = os.path.join(SOLO_RUN, "mean_std") BODY_NPZ = "/data/mocap-convertor/datasets/solo_20260722_body_v1/processed/npz" PROTO_SEED = None BODY_MODEL_KIND = "base" CODEC_SELECTION = os.path.join(SOLO_RUN, "codec_selection.json") CONTINUOUS_DECODE = True # 이 NPZ를 만든 바로 그 베이크 템플릿 — 발행 구조·rest의 단일 소스. # 종전엔 구 Blender 코퍼스용 kemix_rest_table.npy로 되입혀 어깨 120°/ # hips 90°가 틀어진 채 발행됐다 (2026-08-05 실측). BODY_TEMPLATE = ("/data/mocap-convertor/datasets/solo_20260722_body_v1" "/artifacts/templates/kemix.vrma") elif BODY_PROFILE == "v4": BODY_CKPT = os.path.join(WH, "outputs", "gen_train", "weights_v4_vel", "best.pth") VQVAE_DIR = os.path.join(WH, "outputs", "gen_train", "vqvae_kemix_v4_vel") BODY_NPZ = os.path.join(WH, "outputs", "kemix_npz_v4") MEANSTD_DIR = os.path.join(BODY_NPZ, "mean_std") PROTO_SEED = os.path.join( WH, "outputs", "gen_train", "seed_bank", "seed_avg.npz") BODY_MODEL_KIND = "tag3" CODEC_SELECTION = None CONTINUOUS_DECODE = False BODY_TEMPLATE = None # 구 코퍼스는 종전 제네릭 리그 경로 유지 else: raise ValueError( f"unknown AVATAR_BODY_PROFILE={BODY_PROFILE!r}; expected 'v4' or 'solo'") sys.path.insert(0, HERE) from face_v3_infer import ( # noqa: E402 render_face, load_face_model, EMOTION_LABELS, FaceStream, ARKIT_52_NAMES, ) _FACE_MODEL = None # warm in-process (V3 face owns `models`/`scripts` here) _BODY_PROC = None # warm SEPARATE process (GVRM owns `models`/`scripts` there) _BODY_LOCK = threading.Lock() _STREAMING_WARM = False def _start_body_worker(): """Spawn the persistent body worker; block until it reports @@READY@@ (model warm). Separate process because V3-face and GVRM-body both use top-level `models`/`scripts` packages that collide in one interpreter.""" cmd = [PY, os.path.join(HERE, "body_worker.py"), "--ckpt", BODY_CKPT, "--vqvae_dir", VQVAE_DIR, "--meanstd_dir", MEANSTD_DIR, "--npz", BODY_NPZ, "--model_kind", BODY_MODEL_KIND, "--steps", "8"] if PROTO_SEED: cmd.extend(["--proto_seed", PROTO_SEED]) if CODEC_SELECTION: cmd.extend(["--codec_selection", CODEC_SELECTION]) if CONTINUOUS_DECODE: cmd.append("--continuous_decode") if BODY_TEMPLATE: cmd.extend(["--template_vrma", BODY_TEMPLATE]) p = subprocess.Popen(cmd, cwd=HERE, text=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) # stderr inherits (logs) for line in p.stdout: # skip load logs until ready if line.strip() == "@@READY@@": return p raise RuntimeError("body worker exited before READY") def _body_request(job): """Send one command to the persistent body worker and return its sentinel result.""" global _BODY_PROC with _BODY_LOCK: # one worker, one job at a time if _BODY_PROC is None or _BODY_PROC.poll() is not None: _BODY_PROC = _start_body_worker() _BODY_PROC.stdin.write(json.dumps(job, separators=(",", ":")) + "\n") _BODY_PROC.stdin.flush() for line in _BODY_PROC.stdout: # skip stray log lines until the sentinel if line.startswith("@@RESULT@@"): res = json.loads(line[len("@@RESULT@@"):].strip()) if not res.get("ok"): raise RuntimeError("body worker: " + res.get("error", "unknown")) return res raise RuntimeError("body worker closed unexpectedly") def get_face_model(): """Return the warm V3 face model, loading it once on first use.""" global _FACE_MODEL if _FACE_MODEL is None: _FACE_MODEL = load_face_model() return _FACE_MODEL def warm_models(): """Warm both independent GPU model processes before realtime audio starts.""" global _STREAMING_WARM t0 = time.time() face_model = get_face_model() t_face = time.time() - t0 t1 = time.time() _body_request({"op": "ping"}) t_body = time.time() - t1 # The first librosa onset pass and first streaming-sized CUDA forwards have # one-time setup costs (~2s on this machine). Pay those before remote audio # starts, using the exact production geometry (24 frames / 0.8s) so the # first real chunk does not introduce a new face temporal shape. t2 = time.time() if not _STREAMING_WARM: sid = f"__warmup_{os.getpid()}" emit_frames = 24 n_samples = emit_frames * 16000 // FPS silence_b64 = base64.b64encode(bytes(n_samples * 2)).decode() _body_request({"op": "stream_start", "stream_id": sid, "emotion": "neutral", "emit_frames": emit_frames}) _body_request({"op": "stream_chunk", "stream_id": sid, "emotion": "neutral", "pcm_b64": silence_b64}) _body_request({"op": "stream_cancel", "stream_id": sid}) FaceStream(face_model, sid).push_pcm( np.zeros(n_samples, dtype=np.float32), "neutral", emit_frames, frame_start=0) _STREAMING_WARM = True t_stream = time.time() - t2 return {"face_s": round(t_face, 3), "body_s": round(t_body, 3), "stream_s": round(t_stream, 3), "total_s": round(time.time() - t0, 3)} def body_generate(wav_path, emotion, out_vrma, guidance=3.0): """Send one batch job to the warm body worker; writes body.vrma.""" return _body_request({"op": "generate", "wav": wav_path, "emotion": emotion, "out_vrma": out_vrma, "guidance": guidance}) def body_stream_start(stream_id, emotion="neutral", guidance=3.0, emit_frames=24): return _body_request({"op": "stream_start", "stream_id": stream_id, "emotion": emotion, "guidance": guidance, "emit_frames": emit_frames}) def body_stream_chunk(stream_id, pcm_b64, emotion=None, flush=False): job = {"op": "stream_chunk", "stream_id": stream_id, "pcm_b64": pcm_b64, "flush": bool(flush)} if emotion: job["emotion"] = emotion return _body_request(job) def body_stream_end(stream_id, pcm_b64="", emotion=None): job = {"op": "stream_end", "stream_id": stream_id, "pcm_b64": pcm_b64} if emotion: job["emotion"] = emotion return _body_request(job) def body_stream_cancel(stream_id): return _body_request({"op": "stream_cancel", "stream_id": stream_id}) def render_turn(wav_path, emotion, out_id, level=4, steps=8, guidance=3.0): global _FACE_MODEL if emotion not in EMOTION_LABELS: raise ValueError(f"unknown emotion {emotion!r} — must be one of {EMOTION_LABELS}") get_face_model() turn_dir = os.path.join(HERE, "outputs", "turns", out_id) os.makedirs(turn_dir, exist_ok=True) t0 = time.time() # face (in-process, warm) bs, Tf = render_face(wav_path, emotion, level=level, model_dev=_FACE_MODEL, scenario_id=out_id) face_json = os.path.join(turn_dir, "face.json") from face_v3_infer import ARKIT_52_NAMES json.dump({"scenario_id": out_id, "fps": FPS, "num_frames": int(Tf), "names": ARKIT_52_NAMES, "blendshapes": [[round(float(v), 4) for v in bs[t]] for t in range(Tf)]}, open(face_json, "w")) t_face = time.time() - t0 # body (persistent warm worker — separate process, own module namespace) t1 = time.time() body_vrma = os.path.join(turn_dir, "body.vrma") body_generate(wav_path, emotion, body_vrma, guidance=guidance) t_body = time.time() - t1 # audio copy audio_out = os.path.join(turn_dir, "audio.wav") shutil.copyfile(wav_path, audio_out) meta = {"id": out_id, "emotion": emotion, "fps": FPS, "face_frames": int(Tf), "level": level, "steps": steps, "guidance": guidance, "face": "face.json", "body": "body.vrma", "audio": "audio.wav", "timing": {"face_s": round(t_face, 2), "body_s": round(t_body, 2), "total_s": round(time.time() - t0, 2)}} json.dump(meta, open(os.path.join(turn_dir, "meta.json"), "w"), ensure_ascii=False, indent=1) return meta def main(): ap = argparse.ArgumentParser() ap.add_argument("--wav", required=True) ap.add_argument("--emotion", default="neutral", help=f"one of: {', '.join(EMOTION_LABELS)}") ap.add_argument("--id", default=None, help="turn id (folder name)") ap.add_argument("--level", type=int, default=4) ap.add_argument("--steps", type=int, default=8) ap.add_argument("--guidance", type=float, default=3.0) a = ap.parse_args() out_id = a.id or f"turn_{int(time.time())}" meta = render_turn(a.wav, a.emotion, out_id, level=a.level, steps=a.steps, guidance=a.guidance) print(json.dumps(meta, ensure_ascii=False, indent=1)) print(f"\n-> outputs/turns/{out_id}/ (face.json + body.vrma + audio.wav)") if __name__ == "__main__": main()