From f26eb4dda60753c7a7250192233b03376b61c2d2 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 01/21] LoRA compatibility pre-check, drag-to-reorder, and failure report export --- freevideo_engine/comfy_nodes.py | 11 +- freevideo_engine/comfy_upload.py | 14 ++ freevideo_engine/lora_cache.py | 24 +--- freevideo_engine/lora_check.py | 221 +++++++++++++++++++++++++++++++ freevideo_engine/lora_names.py | 25 ++++ scripts/test_freevideo_entry.mjs | 3 +- web/error_panel.js | 70 ++++++++-- web/failure_issue.js | 37 ++++++ web/feedback.js | 8 +- web/freevideo.js | 2 + web/health.js | 2 +- web/lora_check.js | 63 +++++++++ web/native_errors.js | 65 +++++++++ web/studio.css | 11 +- web/studio.js | 104 +++++++++++++-- 15 files changed, 598 insertions(+), 62 deletions(-) create mode 100644 freevideo_engine/lora_check.py create mode 100644 freevideo_engine/lora_names.py create mode 100644 web/failure_issue.js create mode 100644 web/lora_check.js create mode 100644 web/native_errors.js diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index d3be9e32..87678ee2 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -458,6 +458,9 @@ def execute(cls, lora, strength, previous=None): return io.NodeOutput(list(previous or [])) import folder_paths path = folder_paths.get_full_path_or_raise('loras', lora) + if strength != 0: + from .lora_check import require_usable + require_usable(path) return io.NodeOutput(list(previous or []) + [{'path': str(Path(path).resolve()), 'strength': strength}]) @@ -473,8 +476,12 @@ def define_schema(cls): def execute(cls, adapters, previous=None): from .comfy_assets import resolve_loras import folder_paths - return io.NodeOutput(resolve_loras(adapters, folder_paths.get_filename_list('loras'), - lambda name: folder_paths.get_full_path_or_raise('loras', name), previous)) + from .lora_check import require_usable + result = resolve_loras(adapters, folder_paths.get_filename_list('loras'), + lambda name: folder_paths.get_full_path_or_raise('loras', name), previous) + for row in result[len(previous or []):]: + require_usable(row['path']) + return io.NodeOutput(result) @classmethod def fingerprint_inputs(cls, **kwargs): diff --git a/freevideo_engine/comfy_upload.py b/freevideo_engine/comfy_upload.py index 108cfa7e..81389b16 100644 --- a/freevideo_engine/comfy_upload.py +++ b/freevideo_engine/comfy_upload.py @@ -1,5 +1,6 @@ """Stream browser media to ComfyUI input without buffering a clip in RAM.""" import asyncio +import json from pathlib import Path, PureWindowsPath import shutil import uuid @@ -95,4 +96,17 @@ async def save_lora_triggers(request): return web.json_response({'error': str(error)}, status=400) return web.json_response(result, headers={'Cache-Control': 'no-store'}) + @server.routes.get('/freevideo/lora/check') + async def lora_check(request): + from .comfy_assets import resolve_loras + from .lora_check import check_file + try: + name = request.query.get('name') + path = resolve_loras(json.dumps([dict(name=name)]), folder_paths.get_filename_list('loras'), + lambda n: folder_paths.get_full_path_or_raise('loras', n))[0]['path'] + result = await asyncio.to_thread(check_file, path) + except (OSError, ValueError, TypeError) as error: + return web.json_response({'error': str(error)}, status=400) + return web.json_response(result, headers={'Cache-Control': 'no-store'}) + server._freevideo_upload_registered = True diff --git a/freevideo_engine/lora_cache.py b/freevideo_engine/lora_cache.py index c523783d..93d7e528 100644 --- a/freevideo_engine/lora_cache.py +++ b/freevideo_engine/lora_cache.py @@ -20,6 +20,7 @@ from safetensors.torch import save_file import torch +from .lora_names import _targets from .tensor_io import open_tensors from .media_request import digest, verify_files from .monitoring import save @@ -31,29 +32,6 @@ MODULATION = re.compile(r'transformer_blocks\.\d+\.adaln_proj\.') -def _targets(name): - prefixes = ('base_model.model.', 'diffusion_model.', 'model.diffusion_model.', 'transformer.') - while any(name.startswith(prefix) for prefix in prefixes): - prefix = next(prefix for prefix in prefixes if name.startswith(prefix)) - name = name[len(prefix):] - # Community H3 trainers also export Kohya-style flattened module names. - # Only the known H3 block grammar is expanded; underscores inside qkv_proj - # and out_proj are meaningful and must not become dots. - name = re.sub(r'^lora_unet_blocks_(\d+)_(attn|mlp)_(out_proj|qkv_proj|fc1|fc2)(?=\.|$)', - r'blocks.\1.\2.\3', name) - name = re.sub(r'^blocks\.', 'transformer_blocks.', name) - name = name.replace('token_refiner.blocks.', 'token_refiner.refiner_blocks.') - name = name.replace('final_layer.adaln_proj.linear', 'norm_out.linear') - # Original H3 fc1 uses [gate; value], including token-refiner MLPs. - # Diffusers adapters already name ff.net.0.proj and need no swap. - swap = '.mlp.fc1' in name - for before, after in (('.mlp.fc1', '.ff.net.0.proj'), ('.mlp.fc2', '.ff.net.2'), ('.attn.out_proj', '.attn.to_out.0')): - name = name.replace(before, after) - if '.attn.qkv_proj' in name: - return [(name.replace('.attn.qkv_proj', '.attn.' + suffix), i, False) for i, suffix in enumerate(('to_q', 'to_k', 'to_v'))] - return [(name, None, swap)] - - def index_adapter(row, targets, skipped=None): pairs = {} full = [] diff --git a/freevideo_engine/lora_check.py b/freevideo_engine/lora_check.py new file mode 100644 index 00000000..f2dd6ff1 --- /dev/null +++ b/freevideo_engine/lora_check.py @@ -0,0 +1,221 @@ +"""Tell from a LoRA file's header alone whether it can be used with the FreeVideo model. + +Only the key names and tensor shapes are read; no weights are loaded. The module +list below is the model's fixed layout, so the verdict needs no prepared model. +The engine still makes the exact check when it merges the adapter. +""" +import json +import os +import re +import struct +import threading + +from .lora_names import _targets + +MAX_HEADER_BYTES = 64 * 1024 * 1024 +BLOCKS, REFINER_BLOCKS = 50, 2 +UNCHECKED = None # known module, shape not asserted here + + +def _layout(): + shapes = {} + for scope, count in (('transformer_blocks', BLOCKS), ('token_refiner.refiner_blocks', REFINER_BLOCKS)): + for i in range(count): + p = f'{scope}.{i}.' + attn = p + ('attn.orig.' if scope == 'transformer_blocks' else 'attn.') + for name in ('to_q', 'to_k', 'to_v'): + shapes[attn + name] = (7168, 5376) + shapes[attn + 'to_out.0'] = (5376, 7168) + shapes[p + 'ff.net.0.proj'] = (28672, 5376) + shapes[p + 'ff.net.2'] = (5376, 14336) + for i in range(BLOCKS): + p = f'transformer_blocks.{i}.' + shapes[p + 'attn.to_out_linear'] = (5376, 7168) + shapes[p + 'adaln_proj.linear'] = UNCHECKED + shapes[p + 'attn.linear_attention.alpha.down'] = (128, 5376) + shapes[p + 'attn.linear_attention.alpha.up'] = (7168, 128) + shapes[p + 'attn.linear_attention.beta_proj'] = (56, 5376) + shapes[p + 'attn.linear_attention.output_gate.down'] = (128, 5376) + shapes[p + 'attn.linear_attention.output_gate.up'] = (7168, 128) + shapes[p + 'attn.softmax_gate.up'] = (56, 5376) + shapes[p + 'attn.linear_attention.norm'] = (128,) + for name, shape in (('k_sp', (7168, 1, 5, 5)), ('v_sp', (7168, 1, 5, 5)), ('k_tm', (7168, 1, 5)), ('v_tm', (7168, 1, 5))): + shapes[p + 'attn.linear_attention.short_conv.' + name] = shape + shapes[p + 'attn.orig.norm_k'] = shapes[p + 'attn.orig.norm_q'] = (128,) + shapes[p + 'norm1'] = shapes[p + 'norm2'] = (5376,) + for i in range(REFINER_BLOCKS): + p = f'token_refiner.refiner_blocks.{i}.' + shapes[p + 'attn.norm_k'] = shapes[p + 'attn.norm_q'] = (128,) + shapes[p + 'norm1'] = shapes[p + 'norm2'] = (5376,) + shapes.update({ + 'context_embedder': (5376, 5120), 'audio_proj_in': (5376, 32), 'audio_proj_out': (32, 5376), + 'proj_in': (5376, 96), 'proj_out': (96, 5376), 'time_embedder.linear_1': (5376, 256), + 'time_embedder.linear_2': (2688, 5376), 'norm_out.linear': (10752, 2688), + 'norm_out.norm': (5376,), 'token_refiner.final_norm': (5376,)}) + return shapes + + +LAYOUT = _layout() +UNUSABLE = 'This LoRA was not trained for the FreeVideo model and cannot be used.' +_CACHE, _LOCK = {}, threading.Lock() + + +def read_header(path): + """The safetensors header as {name: shape} plus its metadata; ValueError when it is not one.""" + with open(path, 'rb') as stream: + raw = stream.read(8) + if len(raw) != 8: + raise ValueError('Not a safetensors file') + size = struct.unpack('= 0 for n in shape): + raise ValueError('Not a safetensors file') + shapes[key] = tuple(shape) + return shapes, metadata if isinstance(metadata, dict) else {} + + +def _find(target): + """The model module a target name refers to (without any #bias mark), or None.""" + target = target.removesuffix('#bias') + if target not in LAYOUT and '.attn.' in target: + target = target.replace('.attn.', '.attn.orig.', 1) + return target if target in LAYOUT else None + + +FAMILIES = (('wan', re.compile(r'\.(?:self_attn|cross_attn)\.|\.ffn\.\d|^head\.head')), + ('hunyuan', re.compile(r'double_blocks|single_blocks')), + ('ltx', re.compile(r'\.attn[12]\.')), + ('sd', re.compile(r'down_blocks|up_blocks|input_blocks|lora_te|text_model'))) + + +def family(keys): + """Which other model a file's layer names belong to, or None when they match none we know.""" + for name, pattern in FAMILIES: + if any(pattern.search(key) for key in keys): + return name + return None + + +def _bad(reason, keys, **detail): + return dict(reason=reason, family=family(keys) if reason == 'other-model' else None, **detail) + + +def _block(target): + found = re.search(r'blocks\.(\d+)\.', target) + return int(found[1]) + 1 if found else None + + +def assess(shapes): + """None when the adapter fits, else why not: reason 'other-model' (names of another model), 'shape' (our names, + a different size) or 'format' (not a LoRA we can read). 'layer' and the sizes say where. A layer name the table + does not list, in a file that matches no other known model, is let through.""" + keys = list(shapes) + pairs, full = {}, [] + for key in keys: + canonical = key.replace('.default.', '.').replace('.lora_down.weight', '.lora_A.weight').replace('.lora_up.weight', '.lora_B.weight') + match = re.fullmatch(r'(.+)\.lora_([AB])\.weight', canonical) + if match: + pairs.setdefault(match[1], {})[match[2]] = shapes[key] + elif canonical.endswith('.alpha'): + pairs.setdefault(canonical[:-6], {}) + elif canonical.endswith(('.diff', '.diff_b')): + bias = canonical.endswith('.diff_b') + full.append((canonical[:-7] if bias else canonical[:-5], shapes[key], bias)) + else: + return _bad('other-model' if family(keys) else 'format', keys, layer=key) + if not any('A' in pair for pair in pairs.values()) and not full: + return _bad('format', keys, layer=None) + wrong = [] + for name, pair in pairs.items(): + if not {'A', 'B'} <= pair.keys(): + return _bad('format', keys, layer=name) + a, b = pair['A'], pair['B'] + if len(a) != 2 or len(b) != 2 or a[0] != b[1] or a[0] <= 0: + return _bad('format', keys, layer=name) + for target, split, swap in _targets(name): + found = _find(target) + if found is None: + if family(keys): + return _bad('other-model', keys, layer=name) + continue # a name this table does not list: the engine makes the exact check + expected = LAYOUT[found] + rows = b[0] if split is None else b[0] // 3 + if expected is UNCHECKED: + continue + if (split is not None and b[0] % 3) or (swap and rows % 2) or list(expected) != [rows, a[1]]: + wrong.append((found, list(expected), [rows, a[1]])) + for name, shape, bias in full: + for target, split, swap in _targets(name): + found = _find(target) + if found is None: + if family(keys): + return _bad('other-model', keys, layer=name) + continue + expected = LAYOUT[found] + if bias or expected is UNCHECKED: + continue + actual = list(shape) + if split is not None: + if not actual or actual[0] % 3: + wrong.append((found, list(expected), actual)) + continue + actual[0] //= 3 + if actual != list(expected) or (swap and (not actual or actual[0] % 2)): + wrong.append((found, list(expected), actual)) + if wrong: + layer, expected, actual = wrong[0] + return dict(reason='shape', family=None, layer=layer, block=_block(layer), expected=expected, actual=actual, count=len(wrong)) + return None + + +BASE_FIELDS = ('ss_base_model_version', 'modelspec.architecture', 'base_model', 'modelspec.implementation', 'ss_network_module') + + +def summarize(shapes, metadata): + """Counts and a few names from the header; no path, no weights.""" + first = [[key, list(shapes[key])] for key in list(shapes)[:5]] + base = {name: str(metadata[name])[:80] for name in BASE_FIELDS if name in metadata} + return dict(keys=len(shapes), first=first, base=base) + + +def check_file(path): + """{'usable', 'reason', ...} for a LoRA file, remembered until the file changes.""" + try: + stat = os.stat(path) + stamp = (str(path), stat.st_size, stat.st_mtime_ns) + except OSError: + return {'usable': False, 'reason': 'unreadable'} + with _LOCK: + if stamp in _CACHE: + return dict(_CACHE[stamp]) + try: + shapes, metadata = read_header(path) + problem = assess(shapes) + result = {'usable': problem is None, 'reason': None, **(problem or {}), 'summary': summarize(shapes, metadata)} + except (OSError, ValueError): + result = {'usable': False, 'reason': 'unreadable'} + with _LOCK: + if len(_CACHE) > 256: + _CACHE.clear() + _CACHE[stamp] = dict(result) + return dict(result) + + +def require_usable(path): + """Stop a request before generation starts when a LoRA cannot work with the model.""" + if not check_file(path)['usable']: + raise ValueError(UNUSABLE) diff --git a/freevideo_engine/lora_names.py b/freevideo_engine/lora_names.py new file mode 100644 index 00000000..9694bb89 --- /dev/null +++ b/freevideo_engine/lora_names.py @@ -0,0 +1,25 @@ +"""Names a LoRA file uses for the FreeVideo model, shared by the engine and the pre-check.""" +import re + + +def _targets(name): + prefixes = ('base_model.model.', 'diffusion_model.', 'model.diffusion_model.', 'transformer.') + while any(name.startswith(prefix) for prefix in prefixes): + prefix = next(prefix for prefix in prefixes if name.startswith(prefix)) + name = name[len(prefix):] + # Community H3 trainers also export Kohya-style flattened module names. + # Only the known H3 block grammar is expanded; underscores inside qkv_proj + # and out_proj are meaningful and must not become dots. + name = re.sub(r'^lora_unet_blocks_(\d+)_(attn|mlp)_(out_proj|qkv_proj|fc1|fc2)(?=\.|$)', + r'blocks.\1.\2.\3', name) + name = re.sub(r'^blocks\.', 'transformer_blocks.', name) + name = name.replace('token_refiner.blocks.', 'token_refiner.refiner_blocks.') + name = name.replace('final_layer.adaln_proj.linear', 'norm_out.linear') + # Original H3 fc1 uses [gate; value], including token-refiner MLPs. + # Diffusers adapters already name ff.net.0.proj and need no swap. + swap = '.mlp.fc1' in name + for before, after in (('.mlp.fc1', '.ff.net.0.proj'), ('.mlp.fc2', '.ff.net.2'), ('.attn.out_proj', '.attn.to_out.0')): + name = name.replace(before, after) + if '.attn.qkv_proj' in name: + return [(name.replace('.attn.qkv_proj', '.attn.' + suffix), i, False) for i, suffix in enumerate(('to_q', 'to_k', 'to_v'))] + return [(name, None, swap)] diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index bf0ca415..b3ef8a6c 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -116,7 +116,8 @@ test('main browser entry registers both views and preserves node hooks', async t .replace("import { imageGenerationSupported } from './image_generation.js';", 'const imageGenerationSupported = () => true;') .replace("import { uploadMedia } from './media_upload.js';", 'const {uploadMedia} = globalThis.__freevideoEntryTest;') .replace("import { createCastLibrary } from './cast_library.js';", 'const {createCastLibrary} = globalThis.__freevideoEntryTest;') - .replace("import { agentInterfaceEnabled } from './agent_connect.js';", 'const agentInterfaceEnabled = async () => false;'); + .replace("import { agentInterfaceEnabled } from './agent_connect.js';", 'const agentInterfaceEnabled = async () => false;') + .replace("import { installNativeErrorGuard } from './native_errors.js';", 'const installNativeErrorGuard = () => {};'); await import('data:text/javascript;base64,' + Buffer.from(source).toString('base64')); const extension = extensions.find(row => row.name === 'FreeVideo.UnifiedMedia'); assert.ok(extension, 'the main extension must register'); diff --git a/web/error_panel.js b/web/error_panel.js index a3a4c775..5326c4b6 100644 --- a/web/error_panel.js +++ b/web/error_panel.js @@ -1,6 +1,7 @@ // Shared by the Studio and graph view. Text only; never render exception HTML. import {detectBrowserExtension, browserExtensionAdvice, extensionSteps} from './browser_extension.js'; import {SAMPLING_EFFORTS, STEP_CACHES, tableLevel} from './sampling_effort.js'; +import {failureIssue, failureStage} from './failure_issue.js'; const reports = new WeakSet(); const captured = new WeakMap(); const diagnosticReports = new WeakSet(); @@ -48,9 +49,15 @@ const PLAIN_ERRORS = [ {key: 'video-no-picture', message: 'No picture could be read from this video file. Choose another video.', title: ['This video could not be read', '无法读取这个视频'], summary: ['No picture could be read from this video file. Choose another video.', '无法从这个视频文件里读出画面，请换一个视频。']}, + {key: 'lora-unusable', message: 'This LoRA was not trained for the FreeVideo model and cannot be used.', + prefixes: ['LoRA target/shape incompatible with this VDN model', 'Full-difference LoRA target/shape incompatible with this VDN model', + 'Unsupported LoRA tensor (no patches are ignored): ', 'Adapter contains no LoRA factors', 'Expected matching 2-D LoRA factors', 'Incomplete LoRA pair'], + title: ['This LoRA cannot be used', '此 LoRA 无法使用'], + summary: ['This LoRA is not made for the FreeVideo model. Turn it off or remove it, then generate again.', '此 LoRA 不适用于 FreeVideo 的模型。请关闭或移除后重新生成。']}, {key: 'lora-int8', message: 'This LoRA changes parts of the model that the int8 version cannot merge yet, so it cannot be used for now. LoRAs that change only linear layers work.', - title: ['This LoRA cannot be used with the int8 model yet', '这个 LoRA 暂时不能用于 int8 模型'], - summary: ['This LoRA changes parts of the model that the int8 version cannot merge yet, so it cannot be used for now. LoRAs that change only linear layers work.', '这个 LoRA 修改了 int8 模型目前还不能合并的部分，暂时无法使用。只修改线性层的 LoRA 可以正常使用。']}, + title: ['This LoRA cannot be used', '此 LoRA 无法使用'], + summary: ['The int8 version cannot merge the layers this LoRA changes yet. Turn it off or remove it, then generate again.', + 'int8 版本暂时无法合并此 LoRA 修改的层。请关闭或移除后重新生成。']}, {key: 'fa4-update', message: 'The attention speed-up component on this computer needs an update. Run freevideo setup --fa4-guard in a terminal, then restart FreeVideo.', title: ['Runtime components need updating', '运行组件需要更新'], summary: ['The attention speed-up component on this computer needs an update. Run freevideo setup --fa4-guard in a terminal, then restart FreeVideo.', '注意力加速组件需要更新。请在终端运行 freevideo setup --fa4-guard，然后重新启动 FreeVideo。']}, @@ -80,7 +87,8 @@ function knownError(text) { ...(d.step_cache ? {step_cache: d.step_cache} : {})}; } catch { /* Malformed metadata stays in technical details. */ } } - return PLAIN_ERRORS.find(error => line === error.message || error.aliases?.includes(line))?.key ?? null; + return PLAIN_ERRORS.find(error => line === error.message || error.aliases?.includes(line) + || error.prefixes?.some(prefix => line.startsWith(prefix)))?.key ?? null; } // When a diagnostic report exists the card hides Export report; the same advice @@ -362,8 +370,7 @@ export function failureAdvice(value, t, known = knownError(value)) { ['A component needed to run FreeVideo could not load.', 'FreeVideo 所需的运行组件未能加载。'], ['Select "Repair" in the launcher under Settings > Environment. If it still fails, export the report and submit it in a GitHub issue.', '可在启动器“设置 → 环境”里点“修复”。仍然失败时，请导出报告并提交到 GitHub issue。']); return row('unknown', ['Generation stopped', '生成已停止'], - ['This generation did not finish.', '这次生成未完成。'], - ['Click Export report and submit it in a GitHub issue; we will look into it.', '请点“导出报告”，提交到 GitHub issue，我们会专门排查。']); + ['', ''], ['', '']); } let activeDialog = null; @@ -414,7 +421,7 @@ export function createErrorPanel(t) { .fv-failure code{display:inline-block;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:inherit;white-space:nowrap;background:var(--fv-raised);border-radius:var(--fv-r-xs,3px);padding:0 3px} .fv-failure-dialog{box-sizing:border-box;font-size:14px;width:min(560px,94vw);max-height:88vh;overflow:auto;padding:22px 24px;border:1px solid var(--fv-border,#45454a);border-radius:var(--fv-r-lg,16px);background:var(--fv-bg,#202126);color:var(--fv-ink,#eee);box-shadow:0 24px 80px #0008} .fv-failure-dialog::backdrop{background:#070a0db8;backdrop-filter:blur(6px)}.fv-failure-dialog>.fv-failure{border:0;margin:0 0 16px;padding:0}.fv-failure-dialog>button,.fv-failure-dialog .fv-failure-primary{display:block;margin-left:auto;background:var(--fv-accent,#6db8fa);border-color:var(--fv-accent,#6db8fa);color:var(--fv-bg,#111720);font-weight:var(--fv-semibold,600)} -.fv-failure .fv-extension-steps{font-size:14px;line-height:1.6}.fv-failure-reload{order:-1}.fv-failure-footer{display:flex;flex-wrap:wrap;align-items:center;gap:8px}.fv-failure-footer>.fv-failure-actions{margin-right:auto} +.fv-failure .fv-extension-steps{font-size:14px;line-height:1.6}.fv-failure-reload{order:-1}.fv-failure .fv-failure-copy{margin-top:8px}.fv-failure-footer{display:flex;flex-wrap:wrap;align-items:center;gap:8px}.fv-failure-footer>.fv-failure-actions{margin-right:auto} .fv-failure-footer button{min-height:var(--fv-h-sm,32px);border:1px solid var(--fv-border,#45454a);border-radius:var(--fv-r-sm,8px);background:var(--fv-raised,#303138);color:var(--fv-ink,#eee);padding:5px 12px;font:inherit;font-weight:var(--fv-medium,500);cursor:pointer}.fv-failure-footer .fv-failure-primary{margin-left:0}`; document.head.append(style); } @@ -423,26 +430,31 @@ export function createErrorPanel(t) { const title = document.createElement('strong'); title.textContent = t('Generation failed', '生成失败'); const copy = document.createElement('button'); copy.type = 'button'; copy.textContent = t('Copy error', '复制错误'); const download = document.createElement('button'); download.type = 'button'; download.textContent = t('Export report', '导出报告'); + // A generation that stopped offers two buttons: the report file, or the same facts as a GitHub issue. + const github = document.createElement('button'); github.type = 'button'; github.className = 'fv-failure-github'; + github.textContent = t('Submit to GitHub', '提交到 GitHub'); // Shown only for a browser extension, whose steps end with a reload. const reload = document.createElement('button'); reload.type = 'button'; reload.className = 'fv-failure-reload'; reload.textContent = t('Reload page', '刷新页面'); reload.hidden = true; reload.onclick = () => globalThis.location?.reload(); const actions = document.createElement('div'); actions.className = 'fv-failure-actions'; // Last in the DOM, first on screen (CSS order), so copy stays the first child. - actions.append(copy, download, reload); + actions.append(copy, download, reload, github); const explanation = document.createElement('p'), action = document.createElement('p'); const location = document.createElement('p'), cause = document.createElement('p'); cause.className = 'fv-failure-cause'; location.className = 'fv-failure-location'; const privacy = document.createElement('p'); privacy.className = 'fv-failure-privacy'; privacy.textContent = t('Paths and typed text are removed.', '已去掉路径和输入的文字。'); + const privacyInside = document.createElement('p'); privacyInside.className = 'fv-failure-privacy'; privacyInside.hidden = true; + privacyInside.textContent = t('Paths and typed text are removed.', '已去掉路径和输入的文字。'); const downloadStatus = document.createElement('p'); downloadStatus.setAttribute('role', 'status'); - let currentReport = null; + let currentReport = null, githubLora = false; const disclosure = document.createElement('details'), summary = document.createElement('summary'); summary.textContent = t('Technical details', '技术详情'); const detailLocation = document.createElement('p'); detailLocation.className = 'fv-failure-location'; detailLocation.hidden = true; const detailCause = document.createElement('p'); detailCause.className = 'fv-failure-cause'; detailCause.hidden = true; const details = document.createElement('textarea'); details.readOnly = true; details.setAttribute('aria-label', t('Error details', '错误详情')); details.spellcheck = false; - copy.onclick = async () => { + const copyDetails = async button => { let copied = false; try { await navigator.clipboard.writeText(details.value); copied = true; } catch { // Remote ComfyUI commonly uses plain HTTP, where Clipboard API is @@ -450,7 +462,23 @@ export function createErrorPanel(t) { disclosure.open = true; details.focus(); details.select(); try { copied = document.execCommand('copy'); } catch { /* manual Ctrl+C */ } } - copy.textContent = copied ? t('Copied', '已复制') : t('Press Ctrl+C', '请按 Ctrl+C'); + button.textContent = copied ? t('Copied', '已复制') : t('Press Ctrl+C', '请按 Ctrl+C'); + }; + copy.onclick = () => copyDetails(copy); + // A generation failure keeps the raw text inside the collapsed technical details, with its own copy button. + const copyInside = document.createElement('button'); copyInside.type = 'button'; copyInside.className = 'fv-failure-copy'; + copyInside.hidden = true; copyInside.onclick = () => copyDetails(copyInside); + // The issue address needs two browser-only modules. They load when the card is shown, so the click opens the + // page at once (a page opened after waiting can be blocked as a pop-up). + let issueModules = null; + const loadIssueModules = () => import('./feedback.js').then(feedback => import('./updates.js').then(updates => { issueModules = {feedback, updates}; })).catch(() => {}); + github.onclick = async () => { + if (!currentReport) return; + if (!issueModules) await loadIssueModules(); + if (!issueModules) return; + const draft = failureIssue(currentReport, element.freevideoKind, {lora: githubLora, t}); + const url = issueModules.feedback.feedbackIssueURL(draft.text, issueModules.updates.productVersion(), draft.summary, draft.title); + globalThis.open?.(url, '_blank', 'noopener'); }; download.onclick = () => { if (!currentReport) return; @@ -473,7 +501,7 @@ export function createErrorPanel(t) { if (url) setTimeout(() => URL.revokeObjectURL(url), 30000); } }; - disclosure.append(summary, detailLocation, detailCause, details); + disclosure.append(summary, detailLocation, detailCause, details, copyInside, privacyInside); head.append(title, actions); element.append(head, explanation, location, cause, action, disclosure, privacy, downloadStatus); return {element, show(value, popup = true, context = {}) { const report = createErrorReport(value, context); currentReport = report; @@ -490,6 +518,12 @@ export function createErrorPanel(t) { } if (advice.kind === 'unknown' && ['submission', 'queue'].includes(report.stage)) advice.action = t('Select "Copy error" to keep the error details.', '可点“复制错误”保留错误详情。'); + // A generation that stopped without a known cause is named by the stage it stopped in. + const generated = report.stage === 'execution' && advice.kind !== 'browser-extension'; + if (generated && advice.kind === 'unknown') { + const stage = failureStage(report); + advice.title = stage ? t(`Generation did not finish: ${stage[0]}`, `生成未完成：${stage[1]}`) : t('Generation did not finish', '生成未完成'); + } title.textContent = advice.title; explanation.textContent = advice.summary; explanation.hidden = !advice.summary; action.textContent = advice.action; element.querySelector('.fv-extension-steps')?.remove(); if (advice.kind === 'browser-extension') action.after(extensionSteps(advice)); @@ -521,17 +555,27 @@ export function createErrorPanel(t) { downloadStatus.textContent = ''; download.hidden = diagnosticReports.has(report); if (internal && download.hidden) - action.textContent = t('Use Download report in the preview and submit it in a GitHub issue; we will look into it.', - '请点预览区的“下载报告”，提交到 GitHub issue，我们会专门排查。'); + action.textContent = t('Use Download report in the preview and submit it in a GitHub issue.', + '请点预览区的“下载报告”，提交到 GitHub issue。'); else if (download.hidden) { // Advice that ends with "export the report" points at the report this card does offer. for (const [from, to] of EXPORT_TO_DOWNLOAD) if (action.textContent.includes(from)) action.textContent = action.textContent.replace(from, to); } + action.hidden = !action.textContent; privacy.textContent = download.hidden ? t('Select "Download report" in the preview to save the full diagnostic information.', '可在预览区点“下载报告”保存完整诊断信息。') : t('Paths and typed text are removed.', '已去掉路径和输入的文字。'); copy.textContent = advice.kind === 'sampling-table' ? t('Copy details', '复制详情') : t('Copy error', '复制错误'); + // A generation failure shows two buttons; copying moves into the technical details. + copy.hidden = generated; copyInside.hidden = !generated; copyInside.textContent = copy.textContent; + // For a generation failure the note on removed text sits in the technical details, beside the copy button. + privacy.hidden = generated; privacyInside.hidden = !generated; + const lora = advice.kind === 'lora-unusable' || advice.kind === 'lora-int8'; + // An int8 limit is not a judgement, so there is no mistake to report. + github.hidden = !generated || advice.kind === 'lora-int8'; githubLora = lora; + github.textContent = lora ? t('Report a mistake', '报告误判') : t('Submit to GitHub', '提交到 GitHub'); + if (generated && typeof window !== 'undefined') loadIssueModules(); if (popup) showFailureDialog(report, t); return report; }, clear() { currentReport = null; details.value = ''; element.hidden = true; }}; diff --git a/web/failure_issue.js b/web/failure_issue.js new file mode 100644 index 00000000..cd6fdd67 --- /dev/null +++ b/web/failure_issue.js @@ -0,0 +1,37 @@ +// What a failure card sends to GitHub: the redacted report's summary, never the report file, +// paths or typed text (the report was already cleaned). feedback.js turns it into the address. +const PHASES = [ + [/download|dependenc/i, ['downloading files', '下载文件']], + [/load/i, ['loading the model', '载入模型']], + [/prompt|text|encod|reference/i, ['encoding', '编码']], + [/sampl|denois|first|second|refine|pass|step/i, ['sampling', '采样']], + [/decod|vae/i, ['decoding', '解码']], + [/save|mux|output|write/i, ['saving the video', '保存视频']], +]; + +/** The stage a generation stopped in, as [English, Chinese] words, or null when the report does not say. */ +export function failureStage(report) { + const named = [report?.progress?.phase, report?.progress?.stage, + /^Stage: (\S+)/m.exec(String(report?.details || ''))?.[1]].filter(Boolean); + for (const name of named) + for (const [pattern, words] of PHASES) if (pattern.test(name)) return words; + return null; +} + +/** Title, text and summary for the prefilled issue. `lora` marks a LoRA the check judged wrongly. */ +export function failureIssue(report, kind, {lora = false, t = en => en} = {}) { + const stage = failureStage(report); + const lines = String(report?.message || '').split(/\r?\n/).filter(line => line.trim()); + // A traceback ends with the line that names the error; any other message starts with it. + const first = (/^Traceback/.test(lines[0] || '') ? lines[lines.length - 1] : lines[0]) || ''; + const text = [lora ? 'FreeVideo refused this LoRA. The verdict is reported as a misjudgment.' : 'FreeVideo reported a generation failure.', + `Stage: ${stage ? stage[0] : report?.stage || 'unknown'}`, `Error: ${first.trim().slice(0, 300)}`, + `Kind: ${kind || 'unknown'}`].join('\n'); + const types = (report?.nodes || []).map(node => node.type).filter(Boolean); + const summary = JSON.stringify({kind: kind || 'unknown', stage: report?.stage, exception_type: report?.exception_type, + known_error: report?.known_error, nodes: types, settings: report?.settings, progress: report?.progress, + tail: String(report?.details || '').slice(-1500)}); + const title = lora ? t('[Feedback] LoRA compatibility misjudged', '[Feedback] LoRA 兼容判断有误') + : t('[Feedback] Generation did not finish', '[Feedback] 生成未完成') + (stage ? t(`: ${stage[0]}`, `：${stage[1]}`) : ''); + return {title, text, summary}; +} diff --git a/web/feedback.js b/web/feedback.js index c2bc7e21..b4aba8fa 100644 --- a/web/feedback.js +++ b/web/feedback.js @@ -23,14 +23,14 @@ for (const name of ['setup.css', 'feedback.css']) { // The GitHub page for a suggestion, with the text (and, when asked for, the // summary) already in it. -export function feedbackIssueURL(text, version = null, diagnostics = '') { - return feedbackIssueParts(text, version, diagnostics).url; +export function feedbackIssueURL(text, version = null, diagnostics = '', title = '[Feedback]') { + return feedbackIssueParts(text, version, diagnostics, title).url; } // Same, and whether the end of the text had to be left off to fit the address. -export function feedbackIssueParts(text, version = null, diagnostics = '') { +export function feedbackIssueParts(text, version = null, diagnostics = '', title = '[Feedback]') { const build = (body, extra) => { - const params = new URLSearchParams({ template: 'feedback.yml', title: '[Feedback]', feedback: body }); + const params = new URLSearchParams({ template: 'feedback.yml', title, feedback: body }); if (version) params.set('version', `v${version}`); if (extra) params.set('diagnostics', extra); return `${ISSUE_FORM}?${params}`; diff --git a/web/freevideo.js b/web/freevideo.js index ad369b77..6ea9b72b 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -1,4 +1,5 @@ import { createErrorPanel, errorText } from './error_panel.js'; +import { installNativeErrorGuard } from './native_errors.js'; import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; import { openEditorView } from './editor_loader.js'; @@ -830,6 +831,7 @@ api.addEventListener('freevideo_prewarm', event => { if (value) app.graph?.getNodeById(value.node)?.freevideoShowPrewarm?.(value); }); +installNativeErrorGuard({api, app}); api.addEventListener('execution_error', event => { // An upstream loader/media node can stop FreeVideo before its own node is // entered. Retain that error even while the creative workspace is closed. diff --git a/web/health.js b/web/health.js index ffd8f989..071ada73 100644 --- a/web/health.js +++ b/web/health.js @@ -15,7 +15,7 @@ const launchVisit = query.get('freevideo') === 'launch'; // Every script this folder ships; a test keeps the list in step with web/*.js. export const MODULES = ['agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', 'editor.js', 'editor_actions.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js', - 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'library.js', 'motion.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', + 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_terms.js', 'request_assets.js', 'report_issue.js', 'studio_recent.js', 'result_actions.js', 'sampling_effort.js', 'setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'optional_inputs.js', 'studio_queue.js', 'studio_size.js', 'toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'cast_library.js', 'media_upload.js']; diff --git a/web/lora_check.js b/web/lora_check.js new file mode 100644 index 00000000..14db4599 --- /dev/null +++ b/web/lora_check.js @@ -0,0 +1,63 @@ +// What the LoRA check found, as words for the row; the check itself is /freevideo/lora/check. +const FAMILIES = {wan: 'Wan 2.x', hunyuan: 'HunyuanVideo', ltx: 'LTX-Video', sd: 'Stable Diffusion'}; +const size = shape => Array.isArray(shape) ? shape.join('×') : ''; + +/** One short reason, and the lines behind it for the "Details" fold. Null when the LoRA can be used. */ +export function describeVerdict(verdict, t) { + if (!verdict || verdict.usable !== false) return null; + const family = FAMILIES[verdict.family]; + let reason; + if (verdict.reason === 'unreadable') reason = t('the file could not be read. It may be incomplete; download it again.', '文件无法读取，可能未下载完整，请重新下载'); + else if (verdict.reason === 'format') reason = t('not a LoRA file FreeVideo can read', '不是 FreeVideo 能读取的 LoRA 文件'); + else if (verdict.reason === 'shape') reason = t('does not match the current model', '和当前模型的结构不匹配'); + else reason = family ? t(`made for ${family}`, `这是给 ${family} 训练的 LoRA`) : t('not made for the FreeVideo model', '不适用于 FreeVideo 的模型'); + const summary = verdict.summary || {}; + const lines = []; + if (verdict.reason === 'shape') { + const where = Number.isInteger(verdict.block) ? t(`Layer ${verdict.block}: `, `第 ${verdict.block} 层：`) : ''; + lines.push(where + t(`expected ${size(verdict.expected)}, file has ${size(verdict.actual)}`, `期望 ${size(verdict.expected)}，文件是 ${size(verdict.actual)}`)); + } + if (verdict.layer) lines.push(t('Layer: ', '层：') + verdict.layer); + if (verdict.count > 1) lines.push(t(`${verdict.count} layers do not match`, `共 ${verdict.count} 层不匹配`)); + if (Number.isInteger(summary.keys)) lines.push(t(`${summary.keys} tensors`, `共 ${summary.keys} 个张量`)); + for (const [key, shape] of summary.first || []) lines.push(`${key} ${size(shape)}`); + for (const [key, value] of Object.entries(summary.base || {})) lines.push(`${key}: ${value}`); + return {reason, lines}; +} + +/** The text and header summary for a prefilled issue; never the file or its path. */ +export function feedbackDraft(name, verdict, described, t = (en) => en) { + const base = String(name).split('/').pop(); + const text = ['FreeVideo marked this LoRA as unusable. The verdict is reported as a misjudgment.', `File: ${base}`, + `Check result: ${verdict.reason || 'unknown'}${verdict.family ? ` (${verdict.family})` : ''}${described ? `, ${described.reason}` : ''}`].join('\n'); + const summary = verdict.summary || {}; + const diagnostics = JSON.stringify({check: {reason: verdict.reason, family: verdict.family, layer: verdict.layer, block: verdict.block, + expected: verdict.expected, actual: verdict.actual, count: verdict.count}, + header: {keys: summary.keys, first: summary.first, base: summary.base}}); + return {title: (t('[Feedback] LoRA compatibility misjudged: ', '[Feedback] LoRA 兼容判断有误：') + base).slice(0, 120), text, diagnostics}; +} + +/** Verdicts by LoRA name, read once per list refresh. A check that fails to run says nothing, so the LoRA stays usable. */ +export function createVerdicts(fetchApi) { + let known = new Map(), pending = new Map(); + return { + peek: name => known.get(name), + clear() { known = new Map(); pending = new Map(); }, + async get(name) { + if (known.has(name)) return known.get(name); + if (!pending.has(name)) { + const mine = pending, request = (async () => { + try { + const response = await fetchApi('/freevideo/lora/check?' + new URLSearchParams({name}), {cache: 'no-store'}); + const verdict = response.ok ? await response.json() : null; + if (verdict && typeof verdict.usable === 'boolean' && mine === pending) known.set(name, verdict); + if (!verdict && mine === pending) pending.delete(name); + return verdict; + } catch { return null; } + })(); + pending.set(name, request); + } + return pending.get(name); + }, + }; +} diff --git a/web/native_errors.js b/web/native_errors.js new file mode 100644 index 00000000..d60baab9 --- /dev/null +++ b/web/native_errors.js @@ -0,0 +1,65 @@ +// ComfyUI answers every node error with its own overlay (or dialog) carrying a traceback, local +// paths and a link to ComfyUI's issue tracker. FreeVideo shows its own failure panel instead, so +// for errors that belong to FreeVideo those two are skipped while the event is delivered; the +// event itself still reaches ComfyUI's stores and every other listener. +const EVENT = 'execution_error'; +const DIALOG_KEY = 'global-execution-error'; + +/** True when the failing node is a FreeVideo node or feeds one (a loader or media node upstream). */ +export function ownsFailure(detail, graph) { + const failing = String(detail?.node_id ?? ''); + if (/^FreeVideo/.test(String(detail?.node_type ?? ''))) return true; + if (!failing) return false; + for (const node of graph?._nodes || []) { + if (!/^FreeVideo/.test(String(node.comfyClass || node.type || ''))) continue; + const pending = [node], seen = new Set(); + while (pending.length) { + const current = pending.pop(); + if (!current || seen.has(String(current.id))) continue; + seen.add(String(current.id)); + for (const input of current.inputs || []) { + const link = input.link != null ? current.graph?.links?.[input.link] : null; + if (link) pending.push(current.graph?.getNodeById?.(link.origin_id)); + } + } + if (seen.has(failing)) return true; + } + return false; +} + +function stores(app) { + try { + const pinia = app?.extensionManager?._p + || document.querySelector('#vue-app')?.__vue_app__?.config?.globalProperties?.$pinia; + return {errors: pinia?._s?.get('executionError'), dialogs: pinia?._s?.get('dialog')}; + } catch { return {}; } +} + +/** Skip ComfyUI's own error overlay and dialog for FreeVideo errors. Does nothing when the page lacks them. */ +export function installNativeErrorGuard({api, app, owns = detail => ownsFailure(detail, app?.graph)}) { + if (!api || api.freevideoNativeErrorGuard) return; + api.freevideoNativeErrorGuard = true; + // ComfyUI delivers server messages through dispatchCustomEvent (which calls EventTarget's own dispatchEvent). + const dispatch = api.dispatchCustomEvent; + if (typeof dispatch !== 'function') return; + api.dispatchCustomEvent = function (type, detail) { + if (type !== EVENT) return dispatch.call(this, type, detail); + let quiet = false; + try { quiet = !!owns(detail); } catch { /* an unreadable event is left to ComfyUI */ } + if (!quiet) return dispatch.call(this, type, detail); + const {errors, dialogs} = stores(app); + const original = []; + const replace = (store, name, make) => { + if (typeof store?.[name] !== 'function') return; + original.push([store, name, store[name]]); + store[name] = make(store[name]); + }; + replace(errors, 'showErrorOverlay', () => () => {}); + replace(dialogs, 'showDialog', keep => function (options, ...rest) { + return options?.key === DIALOG_KEY ? undefined : keep.call(this, options, ...rest); + }); + try { return dispatch.call(this, type, detail); } finally { + for (const [store, name, value] of original.reverse()) store[name] = value; + } + }; +} diff --git a/web/studio.css b/web/studio.css index e82dd7f3..1bb5e998 100644 --- a/web/studio.css +++ b/web/studio.css @@ -141,11 +141,11 @@ .fv-studio .fv-panel .fv-empty{display:grid;place-items:center;padding:22px 15px;min-height:92px;margin-top:12px;font-size:var(--fv-micro);white-space:pre-line;border-color:var(--fv-border);background:var(--fv-card)} .fv-studio .fv-panel .fv-preview{max-height:170px;margin:10px auto;object-fit:contain}.fv-studio .fv-panel .fv-card{background:var(--fv-card);border-color:var(--fv-border)} .fv-studio .fv-panel select{min-height:var(--fv-h-sm);padding:5px 8px;max-width:145px;width:auto}.fv-studio .fv-panel button{font-size:var(--fv-micro);padding:5px 10px;min-height:var(--fv-h-sm)} -.fv-lora-list{display:flex;flex-direction:column;gap:8px;margin:12px 0}.fv-lora-row{display:grid;grid-template-columns:18px minmax(0,1fr) 64px 30px;align-items:center;gap:7px} +.fv-lora-list{display:flex;flex-direction:column;gap:8px;margin:12px 0}.fv-lora-row{display:grid;grid-template-columns:18px minmax(0,1fr) 64px 30px;align-items:center;gap:7px}.fv-lora-row[data-sortable=true]{grid-template-columns:12px 18px minmax(0,1fr) 64px 30px;--lora-lead:3} .fv-lora-row select{min-width:0!important;max-width:100%!important;width:100%!important}.fv-lora-row input[type=number]{width:64px!important;padding:6px!important;font-variant-numeric:tabular-nums}.fv-lora-row input[type=checkbox]{accent-color:var(--fv-accent);width:15px;min-height:15px} -.fv-lora-range{grid-column:2/5;grid-row:3;justify-self:end;font-size:12px;line-height:16px;white-space:nowrap;color:var(--fv-muted)}.fv-lora-range[hidden]{display:none} -.fv-lora-row[data-enabled=false]{opacity:.45}.fv-lora-order{grid-column:2/5;display:flex;gap:5px;justify-content:flex-end} -.fv-lora-row .fv-lora-trigger{grid-column:2/5;grid-row:2;width:100%;min-width:0;min-height:28px;height:28px;box-sizing:border-box;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);padding:4px 8px;background:var(--fv-card);color:var(--fv-ink);font:inherit;font-size:var(--fv-micro)} +.fv-lora-range{grid-column:var(--lora-lead,2)/span 3;grid-row:3;justify-self:end;font-size:12px;line-height:16px;white-space:nowrap;color:var(--fv-muted)}.fv-lora-range[hidden]{display:none} +.fv-lora-row[data-enabled=false]{cursor:pointer}.fv-lora-row[data-enabled=false]>:not(.fv-lora-off):not(.fv-lora-grip){opacity:.45}.fv-lora-off{grid-column:var(--lora-lead,2)/span 3;grid-row:4;font-size:var(--fv-micro);color:var(--fv-muted)}.fv-lora-off[hidden]{display:none}.fv-lora-grip{width:12px;height:22px;cursor:grab;touch-action:none;color:var(--fv-muted);background:radial-gradient(circle,currentColor 1.3px,transparent 1.6px) 0 0/6px 7px;border-radius:3px}.fv-lora-grip:hover{color:var(--fv-ink)}.fv-lora-dragging{position:relative;z-index:1;background:var(--fv-card);box-shadow:0 4px 14px rgb(0 0 0/.35);border-radius:var(--fv-r-sm);cursor:grabbing}.fv-lora-dragging .fv-lora-grip{cursor:grabbing} +.fv-lora-row .fv-lora-trigger{grid-column:var(--lora-lead,2)/span 3;grid-row:2;width:100%;min-width:0;min-height:28px;height:28px;box-sizing:border-box;border:1px solid var(--fv-border);border-radius:var(--fv-r-sm);padding:4px 8px;background:var(--fv-card);color:var(--fv-ink);font:inherit;font-size:var(--fv-micro)} .fv-lora-trigger::placeholder{color:var(--fv-disabled)} .fv-studio .fv-lora-row button,.fv-lora-panel .fv-lora-row button{min-height:28px;padding:3px 7px}.fv-lora-note{font-size:var(--fv-micro);color:var(--fv-muted);text-align:center;margin:10px 0 0}.fv-lora-add{display:flex;justify-content:center;gap:8px} .fv-studio .fv-error{color:var(--fv-danger)}.fv-studio .fv-legacy{font-size:var(--fv-micro);color:var(--fv-muted);padding:12px 0} @@ -218,3 +218,6 @@ .fv-studio .fv-chain-part-tag[data-kind=error]{color:var(--fv-warning);white-space:normal;max-width:16em;text-align:right} @media(max-width:680px){.fv-studio .fv-chain-part{gap:8px;padding-left:4px}} .fv-studio .fv-request-assets{font-size:12px;line-height:1.5;margin:0 0 8px} + +.fv-lora-problem{grid-column:var(--lora-lead,2)/span 3;grid-row:4;display:flex;flex-direction:column;align-items:flex-start;gap:6px;font-size:var(--fv-micro)}.fv-lora-problem p{margin:0;color:var(--fv-danger)}.fv-lora-problem[hidden]{display:none}.fv-lora-why summary{cursor:pointer;color:var(--fv-muted)}.fv-lora-why pre{margin:6px 0 0;white-space:pre-wrap;word-break:break-all;color:var(--fv-muted);font:inherit;font-family:ui-monospace,monospace}.fv-lora-row[data-unusable=true] select{color:var(--fv-muted)} +.fv-lora-row[data-unusable=true]{cursor:default}.fv-lora-row[data-unusable=true] :is(input[type=number],input[type=checkbox]){opacity:.45}.fv-lora-row[data-unusable=true] select{opacity:1}.fv-lora-row[data-unusable=true] :is(.fv-lora-trigger,.fv-lora-range){display:none} diff --git a/web/studio.js b/web/studio.js index 79375190..c964f883 100644 --- a/web/studio.js +++ b/web/studio.js @@ -5,7 +5,7 @@ import {cleanOptionalPrompt} from './optional_inputs.js'; import { app } from '../../scripts/app.js'; import { api } from '../../scripts/api.js'; import { openSetup } from './setup.js'; -import { feedbackButton } from './feedback.js'; +import { feedbackButton, feedbackIssueURL } from './feedback.js'; import { wordmark } from './wordmark.js'; import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { viewSwitch, viewChanged, agentTabReady, openAutopilot } from './view_navigation.js'; @@ -26,6 +26,7 @@ import { attachMentionOverlay } from './cast_overlay.js'; import { openCastLibrary } from './cast_source.js'; import { uploadMedia } from './media_upload.js'; import { reconcileLoraTriggers } from './lora_triggers.js'; +import { createVerdicts, describeVerdict, feedbackDraft } from './lora_check.js'; import { createSamplingEffort, effortFor, effortsFrom, stepCacheOf } from './sampling_effort.js'; import { resultActions, previewBadge, icon, actionsUnavailable } from './result_actions.js'; import { createPromptEnhancer } from './prompt_enhance.js'; @@ -290,6 +291,10 @@ export function loraPanel(node, panel, {promptNode = null} = {}) { const actions = el('div', null, 'fv-lora-add'); panel.append(list, actions, note); let names = [], disposed = false, rendering = false, renderAgain = false; const drafts = new Map(), errors = new Map(), saves = new Map(); + let skipped = 0; + const idleNote = () => loraWarning() + (skipped ? t(skipped === 1 ? ' 1 other LoRA file in the folder is not in .safetensors format and is not listed.' : ` ${skipped} other LoRA files in the folder are not in .safetensors format and are not listed.`, ` 文件夹中另有 ${skipped} 个 LoRA 文件不是 .safetensors 格式，未列出。`) : ''); + const verdicts = createVerdicts((path, options) => api.fetchApi(path, options)); + let focusGrip = null; const read = () => { const rows = JSON.parse(value(node, 'adapters') || '[]'); if (!Array.isArray(rows) || rows.length > 32) throw new Error(t('Use at most 32 LoRAs', '最多使用 32 个 LoRA')); return rows; }; function syncPrompt(rows) { const targets = new Set([promptNode, ...(node.graph?._nodes || []).filter(target => @@ -326,10 +331,57 @@ export function loraPanel(node, panel, {promptNode = null} = {}) { for (const trigger of list.querySelectorAll('.fv-lora-trigger')) { if (trigger.dataset.name === name && !drafts.has(name)) trigger.value = triggerEntry(name).data?.words ?? ''; } - note.textContent = rows.map(row => errors.get(row.name)).find(Boolean) || loraWarning(); + note.textContent = rows.map(row => errors.get(row.name)).find(Boolean) || idleNote(); } catch (error) { note.textContent = error.message; } } const write = rows => { if (set(node, 'adapters', JSON.stringify(rows))) { syncPrompt(rows); window.dispatchEvent(new CustomEvent('freevideo-loras', {detail: node.id})); } }; + // A LoRA the model cannot take is switched off here, so it is not sent with the request. + function disableUnusable(name) { + if (disposed) return; + try { + const rows = read(); let changed = false; + for (const row of rows) if (row.name === name && row.enabled !== false && verdicts.peek(name)?.usable === false) { row.enabled = false; changed = true; } + if (changed) write(rows); + } catch (error) { note.textContent = error.message; } + } + const reorder = order => { + const rows = read(); + if (order.length !== rows.length || order.every((from, to) => from === to)) return; + write(order.map(from => rows[from])); + }; + // The grip moves the whole row with the pointer; arrow keys do the same one step at a time. + function sortGrip(wrap, index) { + const grip = el('span', null, 'fv-lora-grip'); grip.tabIndex = 0; grip.setAttribute('role', 'button'); + grip.setAttribute('aria-label', t('Drag to reorder', '拖动调整顺序')); grip.title = t('Drag to reorder', '拖动调整顺序'); + grip.onkeydown = event => { + const delta = event.key === 'ArrowUp' ? -1 : event.key === 'ArrowDown' ? 1 : 0; + const count = list.children.length; + if (!delta || index + delta < 0 || index + delta >= count) return; + event.preventDefault(); event.stopPropagation(); + const order = [...Array(count).keys()]; + [order[index], order[index + delta]] = [order[index + delta], order[index]]; + focusGrip = index + delta; reorder(order); + }; + grip.onpointerdown = event => { + if (event.button > 0) return; + event.preventDefault(); wrap.classList.add('fv-lora-dragging'); + const move = e => { + const next = [...list.children].find(row => row !== wrap && e.clientY < row.getBoundingClientRect().top + row.getBoundingClientRect().height / 2); + if (next) { if (wrap.nextElementSibling !== next) list.insertBefore(wrap, next); } + else if (list.lastElementChild !== wrap) list.append(wrap); + }; + const end = e => { + for (const [type, handler] of [['pointermove', move], ['pointerup', end], ['pointercancel', end]]) window.removeEventListener(type, handler); + wrap.classList.remove('fv-lora-dragging'); + const order = [...list.children].map(row => Number(row.dataset.index)); + if (e.type === 'pointercancel') { render(); return; } + reorder(order); + }; + // Moving the row in the DOM drops pointer capture, so the drag listens on the window. + for (const [type, handler] of [['pointermove', move], ['pointerup', end], ['pointercancel', end]]) window.addEventListener(type, handler); + }; + return grip; + } function render() { if (disposed) return; if (rendering) { renderAgain = true; return; } @@ -345,7 +397,12 @@ export function loraPanel(node, panel, {promptNode = null} = {}) { const select = el('select'); select.setAttribute('aria-label', t('LoRA model', 'LoRA 模型')); const pinned = recentLoras().filter(n => names.includes(n)); for (const name of [...new Set([...pinned, ...names, row.name])].filter(Boolean)) { const option = el('option', name); option.value = name; select.append(option); } - select.value = row.name; select.title = row.name; select.onchange = () => { row.name = select.value; rememberLora(row.name); write(rows); }; + select.value = row.name; select.title = row.name; + select.onchange = () => { + // Leaving a LoRA that could not be used turns the row back on for the new choice. + if (verdicts.peek(row.name)?.usable === false) row.enabled = true; + row.name = select.value; rememberLora(row.name); write(rows); + }; const {input: strength, mark: strengthMark} = loraStrengthBox(row.strength ?? 1, () => { if (strength.value !== '') { row.strength = Number(strength.value); write(rows); } }); strength.setAttribute('aria-label', t('LoRA strength', 'LoRA 强度')); const trigger = el('input', null, 'fv-lora-trigger'); trigger.type = 'text'; trigger.maxLength = 300; @@ -373,20 +430,39 @@ export function loraPanel(node, panel, {promptNode = null} = {}) { select.disabled = strength.disabled = trigger.disabled = row.enabled === false; const remove = button('×', () => { rows.splice(index, 1); write(rows); }); remove.setAttribute('aria-label', t('Remove LoRA', '移除 LoRA')); wrap.append(enabled, select, strength, remove, trigger, strengthMark); - if (rows.length > 1) { - const order = el('div', null, 'fv-lora-order'); - for (const [label, delta] of [['↑', -1], ['↓', 1]]) { - const move = button(label, () => { [rows[index], rows[index + delta]] = [rows[index + delta], rows[index]]; write(rows); }); - move.disabled = index + delta < 0 || index + delta >= rows.length; - move.setAttribute('aria-label', delta < 0 ? t('Move LoRA earlier', 'LoRA 前移') : t('Move LoRA later', 'LoRA 后移')); order.append(move); - } - wrap.append(order); - } + wrap.dataset.index = String(index); wrap.dataset.sortable = String(rows.length > 1); + if (rows.length > 1) wrap.prepend(sortGrip(wrap, index)); + const problem = el('div', null, 'fv-lora-problem'); problem.hidden = true; wrap.append(problem); + // A switched-off row says so, and a click anywhere on it turns it back on. + const off = el('div', t('Off. Click to turn on', '已停用，点击启用'), 'fv-lora-off'); off.hidden = row.enabled !== false; wrap.append(off); + wrap.onclick = event => { + if (row.enabled !== false || wrap.dataset.unusable === 'true' || event.target.closest('button,select,input,.fv-lora-grip')) return; + row.enabled = true; write(rows); + }; + const mark = verdict => { + const described = describeVerdict(verdict, t); + wrap.dataset.unusable = String(Boolean(described)); + problem.hidden = !described; off.hidden = off.hidden || Boolean(described); + if (!described) return; + enabled.checked = false; enabled.disabled = true; strength.disabled = trigger.disabled = true; select.disabled = false; + const fold = el('details', null, 'fv-lora-why'), why = el('pre', described.lines.join('\n')); + const report = button(t('Report a mistake', '报告误判'), () => { + const draft = feedbackDraft(row.name, verdict, described, t); + window.open(feedbackIssueURL(draft.text, productVersion(), draft.diagnostics, draft.title), '_blank', 'noopener'); + }); + fold.append(el('summary', t('Details', '详情')), why, report); + problem.replaceChildren(el('p', t('Cannot be used: ', '不能使用：') + described.reason), fold); + if (row.enabled !== false) queueMicrotask(() => disableUnusable(row.name)); + }; + const known = verdicts.peek(row.name); + if (known) mark(known); + else verdicts.get(row.name).then(verdict => { if (verdict && !disposed && wrap.isConnected && select.value === row.name) mark(verdict); }); list.append(wrap); loadTriggers(row.name).catch(error => { if (!disposed && trigger.isConnected) note.textContent = error.message; }); }); if (rows.length > 1) actions.append(clear); - note.textContent = rows.map(row => errors.get(row.name)).find(Boolean) || loraWarning(); + if (focusGrip != null) { list.querySelector(`.fv-lora-row[data-index="${focusGrip}"] .fv-lora-grip`)?.focus(); focusGrip = null; } + note.textContent = rows.map(row => errors.get(row.name)).find(Boolean) || idleNote(); } catch (error) { if (error instanceof DOMException) throw error; note.textContent = error.message; @@ -401,7 +477,7 @@ export function loraPanel(node, panel, {promptNode = null} = {}) { try { const rows = read(); if (rows.length >= 32) throw new Error(t('Use at most 32 LoRAs', '最多使用 32 个 LoRA')); if (!names.length) throw new Error(t('Put LoRA files (.safetensors) in the models/loras folder, then choose "Add".', '把 LoRA 文件（.safetensors）放进 models/loras 文件夹，再点“添加”。')); const next = recentLoras().find(n => names.includes(n) && !rows.some(row => row.name === n)) ?? names[0]; rememberLora(next); rows.push({name: next, strength: 1, enabled: true}); write(rows); } catch (error) { note.textContent = error.message; } }); async function refresh() { - try { const response = await api.fetchApi('/models/loras', {cache: 'no-store'}); if (!response.ok) throw new Error(response.statusText); const listed = await response.json(); names = (Array.isArray(listed) ? listed : []).filter(n => typeof n === 'string' && n.toLowerCase().endsWith('.safetensors')); render(); } + try { const response = await api.fetchApi('/models/loras', {cache: 'no-store'}); if (!response.ok) throw new Error(response.statusText); const listed = await response.json(); verdicts.clear(); skipped = (Array.isArray(listed) ? listed : []).filter(n => typeof n === 'string' && /\.(ckpt|pt|pth|bin|sft)$/i.test(n)).length; names = (Array.isArray(listed) ? listed : []).filter(n => typeof n === 'string' && n.toLowerCase().endsWith('.safetensors')); render(); } catch (error) { if (!disposed) note.textContent = error.message; } } // Only deselects: the files stay in the folder and the recent names are kept. From a1ff4f1db6a178bb853a916a0a80c85e5253554e Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 02/21] Publish the feedback issue form so the feedback button opens it, not an empty issue --- .github/ISSUE_TEMPLATE/feedback.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/feedback.yml diff --git a/.github/ISSUE_TEMPLATE/feedback.yml b/.github/ISSUE_TEMPLATE/feedback.yml new file mode 100644 index 00000000..7dedb9a8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feedback.yml @@ -0,0 +1,18 @@ +name: Feedback / 反馈 +description: Tell us what could be better / 说说哪里可以做得更好 +labels: ["feedback"] +body: + - type: textarea + id: feedback + attributes: + label: What could be better? / 哪里可以做得更好？ + validations: + required: true + - type: input + id: version + attributes: + label: Version / 版本 + - type: textarea + id: diagnostics + attributes: + label: Diagnostic info (optional) / 诊断信息（选填） From 2d61420cc72af296910a4745d50edddfc8cefd51 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 03/21] Failure panel: one layout for submission, queue and browser-extension failures Failure panel: one layout for submission, queue and browser-extension failures; ComfyUI side-panel error text written by FreeVideo - Failure panel: no action line for unknown submission and queue failures; tests follow - Failure panel: submission failure drops the sentence that repeats its title --- scripts/test_freevideo_entry.mjs | 3 +- web/error_panel.js | 47 ++++++++++++++++++-------------- web/failure_issue.js | 2 ++ web/freevideo.js | 7 +++-- web/native_errors.js | 12 ++++++-- 5 files changed, 45 insertions(+), 26 deletions(-) diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index b3ef8a6c..80c2d966 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -46,6 +46,7 @@ test('main browser entry registers both views and preserves node hooks', async t }, secondPassAction: () => new Element('button'), createErrorPanel: () => ({element: new Element('error'), show() {}, clear() {}}), + createErrorReport: () => ({}), adviceFor: () => ({advice: {}}), errorText: value => String(value), openStudio: node => opened.push(node), loraPanel: () => () => {}, loraWarning: () => 'LoRA warning', promptGuide: () => new Element('guide'), @@ -90,7 +91,7 @@ test('main browser entry registers both views and preserves node hooks', async t let source = await readFile(new URL('../web/freevideo.js', import.meta.url), 'utf8'); source = source.replace("import { openEditorView } from './editor_loader.js';", 'const openEditorView = (...args) => globalThis.__freevideoEntryTest.openEditorView?.(...args);'); source = source - .replace("import { createErrorPanel, errorText } from './error_panel.js';", 'const {createErrorPanel,errorText} = globalThis.__freevideoEntryTest;') + .replace("import { createErrorPanel, errorText, createErrorReport, adviceFor } from './error_panel.js';", 'const {createErrorPanel,errorText,createErrorReport,adviceFor} = globalThis.__freevideoEntryTest;') .replace('import { app } from "../../scripts/app.js";', 'const {app} = globalThis.__freevideoEntryTest;') .replace('import { api } from "../../scripts/api.js";', 'const {api} = globalThis.__freevideoEntryTest;') .replace("import { openStudio, loraPanel, loraWarning, promptGuide } from \"./studio.js\";", 'const {openStudio,loraPanel,loraWarning,promptGuide} = globalThis.__freevideoEntryTest;') diff --git a/web/error_panel.js b/web/error_panel.js index 5326c4b6..d3c67350 100644 --- a/web/error_panel.js +++ b/web/error_panel.js @@ -410,6 +410,30 @@ function showFailureDialog(report, t) { // cause drops it: the technical details and the report keep the original text. const shownCause = text => text.replace(/\s*Use PYTORCH_MPS_HIGH_WATERMARK_RATIO=[\d.]+ to disable upper limit for memory allocations(?: \(may cause system failure\))?\.?/g, ''); +/** The written title and explanation for a report, and whether it gets the packaged layout (not a rejected input). */ +export function adviceFor(report, t) { + const advice = report.browser_extension + ? browserExtensionAdvice(report.browser_extension, t) + : failureAdvice(report.details, t, report.known_error ?? null); + if (report.stage === 'submission' && advice.kind === 'unknown') { + advice.title = t('The task could not be submitted', '任务提交失败'); + advice.summary = ''; + } + if (report.stage === 'queue' && advice.kind === 'unknown') { + advice.title = t('The queue operation did not complete', '队列操作未完成'); + advice.summary = t('Check that ComfyUI is still running and connected, then retry.', '请确认 ComfyUI 仍在运行、连接正常后重试。'); + } + // Every failure except a rejected input (which the user fixes in place) has one layout: written title, + // Export report / Submit to GitHub, the raw error only inside the technical details. + const generated = advice.kind !== 'validation'; + // A generation that stopped without a known cause is named by the stage it stopped in. + if (generated && advice.kind === 'unknown' && report.stage === 'execution') { + const stage = failureStage(report); + advice.title = stage ? t(`Generation did not finish: ${stage[0]}`, `生成未完成：${stage[1]}`) : t('Generation did not finish', '生成未完成'); + } + return {advice, generated}; +} + export function createErrorPanel(t) { if (!document.getElementById('freevideo-error-style')) { const style = document.createElement('style'); style.id = 'freevideo-error-style'; @@ -505,25 +529,7 @@ export function createErrorPanel(t) { head.append(title, actions); element.append(head, explanation, location, cause, action, disclosure, privacy, downloadStatus); return {element, show(value, popup = true, context = {}) { const report = createErrorReport(value, context); currentReport = report; - const advice = report.browser_extension - ? browserExtensionAdvice(report.browser_extension, t) - : failureAdvice(report.details, t, report.known_error ?? null); - if (report.stage === 'submission' && advice.kind === 'unknown') { - advice.title = t('The task could not be submitted', '任务提交失败'); - advice.summary = t('The task could not be submitted; see the error details below.', '任务未提交成功，可查看下方错误详情。'); - } - if (report.stage === 'queue' && advice.kind === 'unknown') { - advice.title = t('The queue operation did not complete', '队列操作未完成'); - advice.summary = t('Check that ComfyUI is still running and connected, then retry.', '请确认 ComfyUI 仍在运行、连接正常后重试。'); - } - if (advice.kind === 'unknown' && ['submission', 'queue'].includes(report.stage)) - advice.action = t('Select "Copy error" to keep the error details.', '可点“复制错误”保留错误详情。'); - // A generation that stopped without a known cause is named by the stage it stopped in. - const generated = report.stage === 'execution' && advice.kind !== 'browser-extension'; - if (generated && advice.kind === 'unknown') { - const stage = failureStage(report); - advice.title = stage ? t(`Generation did not finish: ${stage[0]}`, `生成未完成：${stage[1]}`) : t('Generation did not finish', '生成未完成'); - } + const {advice, generated} = adviceFor(report, t); title.textContent = advice.title; explanation.textContent = advice.summary; explanation.hidden = !advice.summary; action.textContent = advice.action; element.querySelector('.fv-extension-steps')?.remove(); if (advice.kind === 'browser-extension') action.after(extensionSteps(advice)); @@ -543,8 +549,7 @@ export function createErrorPanel(t) { `${node.type || t('Node', '节点')} #${node.id}${error.input ? ' · ' + error.input : ''}: ${error.message}${error.details ? ': ' + error.details : ''}`)); const causeText = shownCause(reasons.length ? reasons.join('\n') : [report.message, report.error_details].filter(Boolean).join('\n')); const internal = advice.kind === 'unknown' && !['submission', 'queue'].includes(report.stage); - const causeInDetails = advice.kind === 'browser-extension' - || advice.kind !== 'validation' && !['submission', 'queue'].includes(report.stage); + const causeInDetails = generated; location.hidden = causeInDetails; detailLocation.hidden = !causeInDetails; detailLocation.textContent = causeInDetails ? location.textContent : ''; if (causeInDetails) location.textContent = ''; diff --git a/web/failure_issue.js b/web/failure_issue.js index cd6fdd67..66d3820f 100644 --- a/web/failure_issue.js +++ b/web/failure_issue.js @@ -11,6 +11,8 @@ const PHASES = [ /** The stage a generation stopped in, as [English, Chinese] words, or null when the report does not say. */ export function failureStage(report) { + if (report?.stage === 'submission') return ['submitting the task', '提交任务']; + if (report?.stage === 'queue') return ['queue operation', '队列操作']; const named = [report?.progress?.phase, report?.progress?.stage, /^Stage: (\S+)/m.exec(String(report?.details || ''))?.[1]].filter(Boolean); for (const name of named) diff --git a/web/freevideo.js b/web/freevideo.js index 6ea9b72b..f027031c 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -1,4 +1,4 @@ -import { createErrorPanel, errorText } from './error_panel.js'; +import { createErrorPanel, errorText, createErrorReport, adviceFor } from './error_panel.js'; import { installNativeErrorGuard } from './native_errors.js'; import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; @@ -831,7 +831,10 @@ api.addEventListener('freevideo_prewarm', event => { if (value) app.graph?.getNodeById(value.node)?.freevideoShowPrewarm?.(value); }); -installNativeErrorGuard({api, app}); +installNativeErrorGuard({api, app, describe: detail => { + const {advice} = adviceFor(createErrorReport(detail, {stage: 'execution', graph: app.graph}), text); + return [advice.title, advice.summary].filter(Boolean).join('\n'); +}}); api.addEventListener('execution_error', event => { // An upstream loader/media node can stop FreeVideo before its own node is // entered. Retain that error even while the creative workspace is closed. diff --git a/web/native_errors.js b/web/native_errors.js index d60baab9..3ff0f2e1 100644 --- a/web/native_errors.js +++ b/web/native_errors.js @@ -1,7 +1,8 @@ // ComfyUI answers every node error with its own overlay (or dialog) carrying a traceback, local // paths and a link to ComfyUI's issue tracker. FreeVideo shows its own failure panel instead, so // for errors that belong to FreeVideo those two are skipped while the event is delivered; the -// event itself still reaches ComfyUI's stores and every other listener. +// event itself still reaches ComfyUI's stores and every other listener. The stored record keeps its node +// but carries our written explanation instead of the traceback. const EVENT = 'execution_error'; const DIALOG_KEY = 'global-execution-error'; @@ -36,7 +37,7 @@ function stores(app) { } /** Skip ComfyUI's own error overlay and dialog for FreeVideo errors. Does nothing when the page lacks them. */ -export function installNativeErrorGuard({api, app, owns = detail => ownsFailure(detail, app?.graph)}) { +export function installNativeErrorGuard({api, app, owns = detail => ownsFailure(detail, app?.graph), describe = null}) { if (!api || api.freevideoNativeErrorGuard) return; api.freevideoNativeErrorGuard = true; // ComfyUI delivers server messages through dispatchCustomEvent (which calls EventTarget's own dispatchEvent). @@ -55,6 +56,13 @@ export function installNativeErrorGuard({api, app, owns = detail => ownsFailure( store[name] = make(store[name]); }; replace(errors, 'showErrorOverlay', () => () => {}); + // ComfyUI's side panel and the node's red frame read the recorded error. The frame follows node_id, which + // stays; the text becomes our written explanation, without the traceback (it carries local paths). + replace(errors, 'recordExecutionError', keep => function (record, ...rest) { + let written = ''; + try { written = describe ? String(describe(record) || '') : ''; } catch { /* keep ComfyUI's text */ } + return keep.call(this, written ? {...record, exception_message: written, traceback: [], current_inputs: undefined, current_outputs: undefined} : record, ...rest); + }); replace(dialogs, 'showDialog', keep => function (options, ...rest) { return options?.key === DIALOG_KEY ? undefined : keep.call(this, options, ...rest); }); From d372c2d0f495010cfff888bd472fa519d17fab78 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 04/21] Agent/cast: every character carries a sheet of all views plus the face close-up Agent/cast: every character carries a sheet of all views plus the face close-up; a continued part with several characters keeps each one's face --- freevideo_engine/agent/cast.py | 119 +++++++++++++++++++++++++- freevideo_engine/agent/comfy.py | 7 -- freevideo_engine/agent/drama.py | 100 ++++++++-------------- freevideo_engine/agent/guide.py | 4 +- freevideo_engine/agent/production.py | 3 +- freevideo_engine/agent/productions.py | 87 +------------------ freevideo_engine/agent/schema.py | 3 +- freevideo_engine/comfy_assets.py | 38 ++++++-- freevideo_engine/comfy_media.py | 13 ++- freevideo_engine/continuation.py | 18 ++++ web/cast_mention.js | 46 ++++++---- web/editor_references.js | 13 ++- web/freevideo.js | 4 +- 13 files changed, 264 insertions(+), 191 deletions(-) diff --git a/freevideo_engine/agent/cast.py b/freevideo_engine/agent/cast.py index 1b32163d..5f611628 100644 --- a/freevideo_engine/agent/cast.py +++ b/freevideo_engine/agent/cast.py @@ -40,6 +40,18 @@ CLOSE_UP = dict(aspect='1:1', seconds=4) SCENE = dict(aspect='16:9', seconds=6) +# The sheet of every view a shot carries for each person, put together here from the card's own pictures: up to six +# equal cells (three to a row) on a light grey ground, each picture scaled whole and centred, no text. +SHEET_VERSION = 1 +SHEET_ORDER = ('front', 'three_quarter', 'side', 'back', 'face') # then any other label, in card order +SHEET_CELL = (736, 828) +SHEET_GAP = 48 +SHEET_GROUND = (238, 238, 238) +SHEET_COLUMNS = 3 +SHEET_MAX = 6 +SHEET_SIZE = (SHEET_COLUMNS * SHEET_CELL[0] + (SHEET_COLUMNS + 1) * SHEET_GAP, 2 * SHEET_CELL[1] + 3 * SHEET_GAP) # 2400 x 1800 +_SHEETS_SEEN = set() + def reference_prompt(card, *, face=False): """The one place the reference clip prompts are written.""" @@ -99,6 +111,8 @@ def view(card): value['voice'] = dict(deepcopy(card['voice']), media=media_view(card['voice']['media']), clip=assets.view(card['voice']['clip']['file'])) value['generate_error'] = errors.with_reason(value.get('generate_error')) + if card.get('_sheet'): + value['sheet'] = media_view(card['_sheet']) # the sheet made from the pictures, when the card has none of its own return value @@ -364,7 +378,12 @@ def create(self, value, *, by='builtin', ref=None, made_for=None, local=True, cl def get(self, identity): with self.changed: - return view(self._record(identity)['card']) + card = deepcopy(self._record(identity)['card']) + if card.get('kind') == 'character': + sheet = self.sheet(card) + if sheet is not None: + card['_sheet'] = sheet + return view(card) def list(self, *, q=None, kind=None, origin=None, production=None, limit=50): with self.changed: @@ -544,10 +563,49 @@ def _unfinished(self, film): # Filming ------------------------------------------------------------------------------------------ def snapshot(self, identity): - """The current card for a submission, or None once it is deleted or unknown.""" + """The current card for a submission, or None once it is deleted or unknown. A card with several pictures + and no sheet of its own carries the one made from them as `_sheet` (never part of the public card).""" with self.changed: record = self.records.get(identity) - return deepcopy(record['card']) if record else None + card = deepcopy(record['card']) if record else None + if card is not None and card.get('kind') == 'character': + sheet = self.sheet(card) + if sheet is not None: + card['_sheet'] = sheet + return card + + def sheet(self, card): + """The sheet of every view made from this card's pictures, as a media item, or None when it has no sheet to + make (fewer than two pictures, or one cannot be read). The file is named by what it is made from, so a + picture changed gives a new one and the same pictures reuse the old.""" + if any(row['label'] == 'sheet' for row in card.get('images') or []): + return None + try: + port = self.jobs.port + rank = {label: i for i, label in enumerate(SHEET_ORDER)} + rows = sorted(enumerate(card.get('images') or []), key=lambda r: (rank.get(r[1]['label'], len(rank)), r[0])) + files = [port.input_path(row['media']['file']) for _, row in rows[:SHEET_MAX] + if isinstance(row['media'].get('file'), str)] + if len(files) < 2: + return None + pictures = [path.read_bytes() for path in files] # the same bytes are named and drawn, so a picture + digest = hashlib.sha256(str(SHEET_VERSION).encode()) # changed meanwhile cannot sit under the old name + for data in pictures: + digest.update(hashlib.sha256(data).digest()) + relative = 'freevideo/agent/%s/sheet-%s.png' % (card['id'], digest.hexdigest()[:16]) + target = port.input_path(relative, create=True) + if not _sheet_intact(target): + temporary = target.with_name(target.name + '.' + uuid.uuid4().hex[:8] + '.tmp') + try: + compose_sheet(pictures, temporary) + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + # A sheet is never deleted here: a queued shot or an earlier answer may still name it. + return dict(file=relative) + except Exception: # A sheet is a help: without one the person carries the face alone. + logging.getLogger(__name__).warning('no sheet for card %s', card.get('id'), exc_info=True) + return None def used(self, identities, film): """Record that a film uses these cards; a film never leaves used_by once it is there.""" @@ -881,6 +939,61 @@ def _fail(self, identity, hidden, failure): self.changed.notify_all() +def _sheet_intact(path): + """Whether this is a whole sheet: a file left half written or cut short is made again. A file already checked + and unchanged is not decoded twice.""" + try: + status = path.stat() + key = (str(path), status.st_size, status.st_mtime_ns) + if key in _SHEETS_SEEN: + return True + from PIL import Image + with Image.open(path) as opened: + if opened.size != SHEET_SIZE: + return False + opened.load() + except Exception: + return False + if len(_SHEETS_SEEN) > 256: + _SHEETS_SEEN.clear() + _SHEETS_SEEN.add(key) + return True + + +def compose_sheet(pictures, out): + """Write one PNG of these pictures (two to six, as file bytes or paths): always three columns by two rows of + equal cells, a light grey ground (cells with no picture stay empty), each picture scaled whole to fit its cell + and centred, nothing cropped, no text.""" + import io + from PIL import Image, ImageOps + (width, height), gap = SHEET_CELL, SHEET_GAP + sheet = Image.new('RGB', SHEET_SIZE, SHEET_GROUND) + for index, source in enumerate(pictures[:SHEET_MAX]): + with Image.open(io.BytesIO(source) if isinstance(source, bytes) else source) as opened: + picture = ImageOps.exif_transpose(opened) + if picture.mode in ('RGBA', 'LA', 'P'): + flat = Image.new('RGB', picture.size, SHEET_GROUND) + picture = picture.convert('RGBA') + flat.paste(picture, mask=picture.getchannel('A')) + picture = flat + picture = ImageOps.contain(picture.convert('RGB'), (width, height), Image.LANCZOS) + row, column = divmod(index, SHEET_COLUMNS) + sheet.paste(picture, (gap + column * (width + gap) + (width - picture.width) // 2, + gap + row * (height + gap) + (height - picture.height) // 2)) + sheet.save(out, format='PNG') + + +def continued_pictures(cards): + """The pictures a shot that continues the previous one carries for these cards (the current cards, from + `Cast.snapshot`), at most three: a single person carries the sheet then the face close-up; two or three people + carry one each, the face (else the front view, the three-quarter view, the first other picture), no sheet. + Each is a media item.""" + from . import drama + cards = list(cards)[:spec.CONTINUED_PICTURES] + return [media for card in cards + for media in drama.person_pictures(card, alone=len(cards) == 1, continues=True)][:spec.CONTINUED_PICTURES] + + def _unreadable(error): """A clip that is gone or cannot be decoded, rather than a fault of ours.""" if isinstance(error, (JobError, ValueError, OSError)): diff --git a/freevideo_engine/agent/comfy.py b/freevideo_engine/agent/comfy.py index 4d7f3201..96deda12 100644 --- a/freevideo_engine/agent/comfy.py +++ b/freevideo_engine/agent/comfy.py @@ -101,13 +101,6 @@ def capability_snapshot(self, query): cache = getattr(self.server, '_freevideo_capabilities', None) return cache.get(_query(query)) if hasattr(cache, 'get') else dict(UNAVAILABLE) - def prepared_precision(self): - """'int8' or 'fp8', the family of the model setup prepared for this machine; None while unknown.""" - cache = getattr(self.server, '_freevideo_capabilities', None) - snapshot = getattr(cache, 'snapshot', None) - saved = snapshot[0].get('prepared_format', 'int8') if snapshot else None - return next((p for p in ('int8', 'fp8') if isinstance(saved, str) and saved.startswith(p)), None) - def output_summary(self, relative): from .jobs import library_id from ..comfy_library import _report diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py index a9ce60e7..8908a9de 100644 --- a/freevideo_engine/agent/drama.py +++ b/freevideo_engine/agent/drama.py @@ -61,18 +61,9 @@ def __init__(self, problems): CAPTIONED = 3 # the score of a take with unwanted captions CARD = re.compile(r'c[0-9a-f]{12}') # characters..card names a cast library card # How a shot carries its cast cards. Change these after the reference tests; nothing else needs to. -PICTURE_ORDER = ('sheet', 'front', 'three_quarter', 'face') # then any other label, in card order -PICTURES_EACH = 6 -PICTURES_LEAST = 3 # What a person carried before six: a device that cannot hold more still shoots with these. -# Six pictures cost extra time on every step. A class of machine starts from six only when it was measured at 25% -# extra or less on every step, cold and warm: (card memory tier in GiB, people in the shot, weight precision). -# Everything else, and everything unmeasured, starts from three. Only Linux is measured. Two people went 45-79% over -# at 8 to 24 GiB, so only one-person classes can ever be listed. No class has passed yet. -SIX_PICTURES_CLASSES = frozenset() -SIX_PICTURES_TIERS_GIB = (8, 10, 12, 16, 20, 24, 32) # measured card sizes; a card takes the largest one it reaches -SIX_PICTURES_SYSTEMS = ('Linux',) -TIER_SLACK_GIB = .5 # A "24 GB" card reports a little under 24 GiB. -CONTINUED_ORDER = ('face', 'front', 'three_quarter') # a continuing shot's picture for each person +# Each person is two pictures: one sheet of every view (made by the cast library when the card has none), then +# the face close-up. A continuing shot with several people has no room for two each: it carries one picture a person. +FACE_ORDER = ('face', 'front', 'three_quarter') # the one picture standing for a person, then any other label, in card order MAX_VOICES = 3 # voice clips per shot, the model limit MAX_REFERENCES = 32 @@ -299,46 +290,51 @@ def subject_definitions(subjects): return 'subject_definitions: ' + ' '.join(parts) -def _pictures(card, each=PICTURES_EACH): - rank = {label: i for i, label in enumerate(PICTURE_ORDER)} - rows = sorted(enumerate(card.get('images') or []), key=lambda r: (rank.get(r[1]['label'], len(rank)), r[0])) - return [row['media'] for _, row in rows[:each]] +def _views(card): + """The card's pictures other than a sheet, best stand-in first: the face, the front, the three-quarter view, the rest.""" + rank = {label: i for i, label in enumerate(FACE_ORDER)} + rows = [(i, row) for i, row in enumerate(card.get('images') or []) if row['label'] != 'sheet'] + rows.sort(key=lambda r: (rank.get(r[1]['label'], len(rank)), r[0])) + return [row['media'] for _, row in rows] -def _continued_pictures(card, alone): - """The pictures a continuing shot carries for one person. +def sheet_of(card): + """The card's sheet of every view: one it carries itself, else the one the cast library made from its pictures.""" + own = next((row['media'] for row in card.get('images') or [] if row['label'] == 'sheet'), None) + return own or card.get('_sheet') - Alone in the shot: the sheet, then a close face (else the front view), because a sheet - alone makes the face too small. With others: one picture, face > front > three_quarter.""" - by = {} - for item in card.get('images') or []: - by.setdefault(item['label'], item['media']) - rest = [item['media'] for item in card.get('images') or [] if item['label'] not in CONTINUED_ORDER] - ordered = [by[label] for label in CONTINUED_ORDER if label in by] + rest - if not alone: - return ordered[:1] - sheet = by.get('sheet') - close = by.get('face') or by.get('front') or next((m for m in ordered if m is not sheet), None) - return [m for m in (sheet, close) if m is not None] or ordered[:1] +def person_pictures(card, *, alone=True, continues=False): + """The pictures one person carries into a shot, in order: the sheet, then the face close-up (the front view + when there is no face). Two of them on every device. A card that has no sheet (one picture, or none could be + made) carries its face alone (else the front view). A shot that continues the previous one with several + people carries only the face (else the front view) of each, with no sheet.""" + views = _views(card) + if continues and not alone: + return views[:1] or ([sheet_of(card)] if sheet_of(card) else []) + sheet = sheet_of(card) + if sheet is None: + return views[:1] + return [sheet] + [view for view in views[:1] if view.get('file') != sheet.get('file')] -def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH): + +def cast_references(play, shot, cards, *, continues=False): """Which cast cards a shot carries, as references and a prompt prefix. cards maps a card ID to the current card, or None once it is gone. The same - function serves plan previews and the moment a take is submitted. each is - the most pictures one person carries; pictures_each says what was used. + function serves plan previews and the moment a take is submitted. + pictures_each is the most pictures one person carries in this shot. cast_reference: None (no character in the shot has a card), 'own' (the shot names its own first frame or references; an empty list means no pictures), 'missing' (a card was deleted), 'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out entirely; 'voice_omitted' lists people whose pictures are carried but whose - voice did not fit. A shot that continues the previous one carries one picture - per person (face, else front, else three-quarter; a lone person gets the sheet and - then a face close-up) for at most CONTINUED_PICTURES people, and no voice clip: its voices carry on from the previous shot.""" + voice did not fit. A shot that continues the previous one carries no voice + clip (its voices carry on from the previous shot) and at most CONTINUED_PICTURES + pictures: two for a single person, else one for each of up to that many people.""" characters = play.get('characters', {}) keys = [k for k in cast(play, shot) if isinstance(characters[k].get('card'), str)] result = dict(cast_reference=None, people=[], omitted=[], voice_omitted=[], missing=[], references=[], - prefix=None, subjects=[], cards={}, pictures_each=each, clip_seconds=[]) + prefix=None, subjects=[], cards={}, pictures_each=None, clip_seconds=[]) if not keys: return result # A references list, even an empty one, is the shot's own choice: cards are added only when it is left out. @@ -350,18 +346,18 @@ def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH): spoken[line['who']] = spoken.get(line['who'], 0) + 1 # People with more lines come first when not everyone fits; ties keep the order of appearance. chosen, pictures, voices = [], 0, 0 - alone = continues and sum(1 for k in keys if cards.get(characters[k]['card']) is not None - and _pictures(cards[characters[k]['card']])) == 1 + alone = sum(1 for k in keys if cards.get(characters[k]['card']) is not None + and (cards[characters[k]['card']].get('images') or [])) == 1 for key in sorted(keys, key=lambda k: -spoken.get(k, 0)): identity = characters[key]['card'] card = cards.get(identity) if card is None: result['missing'].append(key) continue - images = _pictures(card, each) # a continuing shot ignores each: it carries at most 2, within the 3 it may hold + images = person_pictures(card, alone=alone, continues=continues) if continues: if images and len(chosen) < specs.CONTINUED_PICTURES: - chosen.append((key, card, _continued_pictures(card, alone), False)) + chosen.append((key, card, images, False)) else: result['omitted'].append(key) continue @@ -391,6 +387,7 @@ def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH): result['people'].append(dict(character=key, card=card['id'], revision=card['revision'], pictures=len(images), voice=voice)) result['references'].extend(clips) + result['pictures_each'] = max((len(images) for _, _, images, _ in chosen), default=0) or None if subjects: result['prefix'] = subject_definitions(subjects) result['subjects'] = subjects @@ -399,27 +396,6 @@ def cast_references(play, shot, cards, *, continues=False, each=PICTURES_EACH): return result -def pictures_start(machine, people=1): - """Pictures a person carries before the length check: six for a measured class, else three. - machine is a capability table's `machine` row; a missing or unreadable one is unmeasured.""" - try: - if machine['system'] in SIX_PICTURES_SYSTEMS: - vram = float(machine['vram_gib']) + TIER_SLACK_GIB - tier = max((t for t in SIX_PICTURES_TIERS_GIB if t <= vram), default=None) - if tier is not None and (tier, people, machine['precision']) in SIX_PICTURES_CLASSES: - return PICTURES_EACH - except (KeyError, TypeError, ValueError): - pass - return PICTURES_LEAST - - -def pictures_note(each, cut=True): - """One sentence when the length check carried fewer pictures than this device started from; otherwise None.""" - if each is None or each >= PICTURES_EACH or not cut: - return None - return 'Each person carries %d pictures in this shot; this device cannot hold more at this length.' % each - - PROMPT_LIMIT = 20000 DESCRIPTION_CUTS = (500, 250, 120, 0) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index ef9efb45..07b4cece 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -543,8 +543,8 @@ def connection_text(value): 'Changes need the revision you read (body revision or If-Match); stale_revision returns the current card in ' 'error.card. Fields the user changed return changed_by_user: keep the user\'s version.', 'Film with a card by setting characters..card. Cards are read when each take is submitted. Shots that ' - 'continue the previous one carry one card picture per person (a person alone: the sheet and a face close-up) for ' - 'at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. ' + 'continue the previous one carry the sheet and the face for a single person, else one picture each (the face, ' + 'else the front) for at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. ' 'A shot\'s own references replace the card pictures; references=[] sends none. ' 'A shot with a carded character is at most 15 s; continue it in the next shot to go longer.', 'POST /cast/{id}/generate makes reference pictures from the description as a normal job (face=true adds a close-up). ' diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py index d172622a..37b8b207 100644 --- a/freevideo_engine/agent/production.py +++ b/freevideo_engine/agent/production.py @@ -722,8 +722,7 @@ def view(state): row.update(cast_reference=latest.get('cast_reference'), cast_omitted=deepcopy(latest.get('cast_omitted', [])), cast_voice_omitted=deepcopy(latest.get('cast_voice_omitted', [])), cast_missing=deepcopy(latest.get('cast_missing', [])), - cast_pictures_each=latest.get('cast_pictures_each'), - cast_note=drama.pictures_note(latest.get('cast_pictures_each'), latest.get('cast_pictures_cut', True))) + cast_pictures_each=latest.get('cast_pictures_each')) row['takes'] = [] for take in shot['takes']: item = {key: deepcopy(take[key]) for key in ('take', 'job', 'generation', 'seed', 'state', 'stale', diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 64bcb6d4..fcfb811e 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -70,11 +70,6 @@ def pictures_each(resolved): return resolved['pictures_each'] if resolved['people'] else None -def pictures_cut(resolved): - """True when the length check took pictures away from what this device starts with.""" - return bool(resolved['people'] and resolved.get('pictures_cut')) - - class MaterialRejected(JobError): """External callers named material the content-only boundary does not admit: the job validation error.""" status = 422 @@ -171,7 +166,6 @@ def failure(view): class Productions: def __init__(self, jobs): self.jobs = jobs - self._machine = None # the capability table's machine row, once known self.directory = jobs.directory / 'productions' self.records = {} self.changed = threading.Condition(jobs.lock) @@ -375,7 +369,6 @@ def plan(self, request, *, local=True): row.update(cast_reference=resolved['cast_reference'], cast=resolved['people'], cast_omitted=resolved['omitted'], cast_voice_omitted=resolved['voice_omitted'], cast_missing=resolved['missing'], cast_pictures_each=pictures_each(resolved), - cast_note=drama.pictures_note(pictures_each(resolved), pictures_cut(resolved)), references=len(resolved['references'])) try: spec._text(row['prompt'], '/screenplay/shots/%d/prompt' % i, 20000) @@ -921,8 +914,7 @@ def _execute(self, identity, action): cast_omitted=list(resolved['omitted']), cast_voice_omitted=list(resolved['voice_omitted']), cast_missing=list(resolved['missing']), - cast_pictures_each=pictures_each(resolved), - cast_pictures_cut=pictures_cut(resolved)) + cast_pictures_each=pictures_each(resolved)) self._save(marker) # Durable before Jobs may accept it. self.jobs.submit(item, client='production:' + identity, parent=dict(production=identity, ref=ref)) except spec.SpecError as error: @@ -957,80 +949,9 @@ def _cast(self, state, take, item): return resolved def _carry(self, play, shot, cards, item, *, continues): - """The cast a shot carries: this device's starting number of pictures a person (six for a measured - class of card, people and precision, else three), fewer where it cannot hold them, down to three. Plan - previews and submissions both come here, so they agree.""" - if continues: - # A continuing shot carries one picture a person whatever the device holds. - resolved = drama.cast_references(play, shot, cards, continues=True, each=drama.PICTURES_LEAST) - resolved.update(pictures_each=None, pictures_cut=False) - return resolved - people = len(drama.cast_references(play, shot, cards, continues=continues, each=drama.PICTURES_LEAST)['people']) - start = each = self._pictures_start(item, people) - while True: - resolved = drama.cast_references(play, shot, cards, continues=continues, each=each) - if each <= drama.PICTURES_LEAST or item is None or not self._too_long(item, resolved): - resolved['pictures_cut'] = each < start - return resolved - each -= 1 - - def _pictures_start(self, item, people): - """Where the picture count starts for this many people on this machine, from the capability table's - graphics card, system and precision. Any canvas names the same machine, so a shot without a take asks - about the default canvas.""" - if self._machine is None: - from .length import capability_table - width, height = spec.canvas_size(item or {}) - table = capability_table(self.jobs.port, dict(width=width, height=height, task='ref2va', loras=False), wait=3.) - if table.get('status') != 'ready': - return drama.pictures_start(None) # Not worked out yet: three, and the next ask looks again. - precision = getattr(self.jobs.port, 'prepared_precision', lambda: None)() - # The machine does not change while this server runs. - self._machine = dict(table.get('machine') or {}, precision=precision if isinstance(precision, str) else None) - return drama.pictures_start(self._machine, people) - - def _too_long(self, item, resolved): - """True when this device cannot generate the shot with these references. A table that is not ready - (still computing, unavailable) or lacks this canvas says False and keeps the pictures; a ready table - that admits no length at this canvas, or a limit this shot's length cannot be planned under, says True.""" - pictures = sum(p['pictures'] for p in resolved['people']) - if not pictures: - return False - try: - from ..chain_plan import PlanError, plan as chain_plan - from .length import capability_table - media = dict(references=deepcopy(resolved['references'])) - normalized = spec.normalize(dict(deepcopy(item), media=media)) - rows = ([dict(kind='image', width=normalized['width'], height=normalized['height'])] * pictures - + [dict(kind='audio', seconds=seconds) for seconds in resolved['clip_seconds']]) - table = capability_table(self.jobs.port, normalized, wait=3., references=rows, - task='ref2va_av' if resolved['clip_seconds'] else 'ref2va') - if table.get('status') != 'ready': - return False - index = next((i for i, row in enumerate(table.get('sizes', [])) - if (row['width'], row['height']) == (normalized['width'], normalized['height'])), None) - tiers = table.get('tiers', {}) - tier = tiers.get(normalized['quality']) - if tier is None: - # Same plan and step cache first, then the same plan (see length.single_frame_limit). - same = [row for row in tiers.values() - if row.get('steps') == normalized['base_steps'] and row.get('two_pass') == normalized['two_pass'] - and (not normalized['two_pass'] or normalized['refine_steps'] == 3)] - tier = next((row for row in same if row.get('step_cache') == normalized.get('step_cache')), - same[0] if same else None) - maxima = tier.get('max_frames') if tier else None - if index is None or not isinstance(maxima, list) or index >= len(maxima): - return False - limit = maxima[index] - if limit is None: - return True # no length at all fits at this canvas with these pictures - try: - chain_plan(normalized['seconds'], max_frames=limit, continued=bool(normalized.get('continue_from'))) - except PlanError: - return True - return False - except Exception: # The limit is advisory: a failure here keeps the pictures and lets the job decide. - return False + """The cast a shot carries: each person's sheet and face close-up on every device (one picture a person when + a continuing shot has several people). Plan previews and submissions both come here, so they agree.""" + return drama.cast_references(play, shot, cards, continues=continues) def _current(self, play): """Cards as they are now; a card deleted since the film started is None (the shot uses text).""" diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 12ab2bc0..e19542c6 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -910,8 +910,7 @@ def production_response(): cast_reference=CAST_REFERENCE, cast_omitted=dict(type='array', items=string()), cast_voice_omitted=dict(type='array', items=string()), cast_missing=dict(type='array', items=string()), - cast_pictures_each=dict(type=['integer', 'null'], description='Pictures each person carries in this shot: six or three by graphics card class, fewer (down to three) when this device cannot hold the starting number at the shot length.'), - cast_note=dict(type=['string', 'null'], description='One sentence saying so, only when it was reduced from the starting number.'), + cast_pictures_each=dict(type=['integer', 'null'], description='Most pictures one person carries in this shot: two (the sheet of every view and the face close-up), or one each when a continuing shot has several people.'), takes=dict(type='array', items=dict(type='object', properties=dict(take=number(1, integer=True), job=REF, task={'oneOf': [JOB, {'type': 'null'}]}, generation=number(0, integer=True), seed=number(0, integer=True), stale={'type': 'boolean'}, error={'type': ['object', 'null']}, diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index f1d0e61d..ff304d4d 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -75,7 +75,11 @@ def resolve_assets(serialized, input_directory): root = Path(input_directory).resolve() result = {} for row in rows: - if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled'}: + if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast'}: + raise ValueError('Invalid media item') + # The character an @ mention added the row for; a continued part keeps one picture per character. + if 'cast' in row and (set(row['cast'] if isinstance(row['cast'], dict) else ('?',)) != {'id'} + or not isinstance(row['cast']['id'], str) or not 0 < len(row['cast']['id']) <= 128): raise ValueError('Invalid media item') if type(row.get('enabled', True)) is not bool: raise ValueError('Media enabled must be a boolean') @@ -101,7 +105,10 @@ def resolve_assets(serialized, input_directory): raise ValueError('Select only one ' + role + ' frame') result[role] = str(path) elif role == 'reference': - result.setdefault('references', []).append({'kind': kind, 'path': str(path)}) + reference = {'kind': kind, 'path': str(path)} + if cast_id(row) is not None: + reference['cast'] = cast_id(row) # only a continued part reads it (comfy_media.continued) + result.setdefault('references', []).append(reference) else: raise ValueError('Media role must be first, last or reference') # Mixing keyframes and references is checked when the video is made (comfy_media.export): @@ -110,15 +117,16 @@ def resolve_assets(serialized, input_directory): def continued_assets(serialized): - """The panel rows a continued part uses: enabled image references in panel order, at most three. + """The panel rows a continued part uses: enabled image references in panel order, at most three + (continuation.pick_pictures: with two or more characters, one picture each). Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks.""" - from .continuation import PICTURES_MAX + from .continuation import pick_pictures try: rows = json.loads(serialized) except (TypeError, ValueError): return '[]' - kept = [] + pictures = [] for row in rows if isinstance(rows, list) else (): if (isinstance(row, dict) and row.get('enabled', True) is True and row.get('role') == 'reference' and isinstance(row.get('file'), str)): @@ -126,9 +134,23 @@ def continued_assets(serialized): image = media_kind(row['file']) == 'image' except ValueError: image = False - if image and len(kept) < PICTURES_MAX: - kept.append(dict(file=row['file'], role='reference')) - return json.dumps(kept) + if image: + pictures.append(row) + return json.dumps([_kept_row(row) for row in pick_pictures(pictures, cast_id)]) + + +def cast_id(row): + """The character a panel row belongs to (the media list marks rows added by an @ mention), else None.""" + cast = row.get('cast') if isinstance(row, dict) else None + value = cast.get('id') if isinstance(cast, dict) else None + return value if isinstance(value, (str, int)) and not isinstance(value, bool) else None + + +def _kept_row(row): + kept = dict(file=row['file'], role='reference') + if cast_id(row) is not None: + kept['cast'] = dict(id=cast_id(row)) + return kept def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, **images): diff --git a/freevideo_engine/comfy_media.py b/freevideo_engine/comfy_media.py index dbf06460..2f8b8890 100644 --- a/freevideo_engine/comfy_media.py +++ b/freevideo_engine/comfy_media.py @@ -50,13 +50,15 @@ def continued(exported, notes=None): It starts from the footage, so a first or last frame image becomes one more picture after the reference pictures; pre-encoded conditioning, clips and sound are left out. When keyframes are used or pictures are left out, notes gets one line saying so.""" - from .continuation import PICTURES_MAX + from .continuation import PICTURES_MAX, pick_pictures media = (exported or {}).get('media') or {} kept = {key: value for key, value in media.items() if key not in ('first', 'last', 'references', 'conditioning_info')} pictures = [row for row in media.get('references', []) if row.get('kind') == 'image'] keyframes = [name for name in ('first', 'last') if media.get(name)] - candidates = pictures + [{'kind': 'image', 'path': media[name]} for name in keyframes] + # Characters added by an @ mention keep one picture each when there are several (pick_pictures). + chosen = pick_pictures(pictures, lambda row: row.get('cast')) + candidates = [_plain(row) for row in chosen] + [{'kind': 'image', 'path': media[name]} for name in keyframes] if candidates: kept['references'] = candidates[:PICTURES_MAX] lines = [continued_note(len(pictures), keyframes, len(candidates[:PICTURES_MAX]))] @@ -70,6 +72,11 @@ def continued(exported, notes=None): return {'media': kept} if len(kept) > 1 else {} +def _plain(row): + """A reference row without the panel's character mark, which the engine's request does not carry.""" + return {key: value for key, value in row.items() if key != 'cast'} + + def continued_note(pictures, keyframes, used): """One line for a continued part whose keyframes became pictures or whose pictures did not all fit.""" frames = 'the %s frame%s' % (' and '.join(keyframes), 's' if len(keyframes) > 1 else '') @@ -146,6 +153,8 @@ def export(run, canvas, *, first=None, last=None, references=None, loras=None, c torch.save(value, path) media['conditioning_info'] = info result['conditioning'] = str(path) + if not continuing and media.get('references'): + media['references'] = [_plain(row) for row in media['references']] if len(media) > 1: result['media'] = media return result diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py index 6a50d9f1..f432520c 100644 --- a/freevideo_engine/continuation.py +++ b/freevideo_engine/continuation.py @@ -40,6 +40,24 @@ UNSUPPORTED_MEDIA = UNSUPPORTED + ' Connect up to 3 pictures as references, and remove video, audio and pre-encoded inputs.' # A continued part keeps the cast's pictures, one per person and at most three people in a frame. PICTURES_MAX = 3 + + +def pick_pictures(rows, person): + """The pictures a continued part keeps from image rows in panel order: at most PICTURES_MAX. + + person(row) is the id of the character a row belongs to, or None. With two or more characters each keeps + only the last of its rows (the face close-up, or the one picture its card has) so that everyone is seen; the + other pictures follow. With none or one character the first rows are kept.""" + rows = list(rows) + last = {} + for index, row in enumerate(rows): + if person(row) is not None: + last[person(row)] = index + if len(last) < 2: + return rows[:PICTURES_MAX] + faces = [rows[index] for index in sorted(last.values())] + others = [row for row in rows if person(row) is None] + return (faces + others)[:PICTURES_MAX] INVALID_FRAMES = 'Frame count must be 5 + 17n and at least 39 (39, 56, 73, 90, ...).' INVALID_FILE = 'Choose an existing MP4 video using its absolute path.' diff --git a/web/cast_mention.js b/web/cast_mention.js index b332747d..c243c255 100644 --- a/web/cast_mention.js +++ b/web/cast_mention.js @@ -3,10 +3,10 @@ // of the media list that carry `cast: {id, name, description}`. When a request is made the names become // and the subject definitions go in front, the same wording the Agent's drama planner writes (agent/drama.py). export const MAX_PEOPLE = 3; -export const MAX_PICTURES_EACH = 6; +export const MAX_PICTURES_EACH = 2; export const MAX_REFERENCES = 32; -// Which of a card's pictures come first; the others follow in the card's own order. -export const PICTURE_ORDER = ['sheet', 'front', 'three_quarter', 'face']; +// Which of a card's pictures stands for the person after the sheet; the others follow in the card's own order. +export const PICTURE_ORDER = ['face', 'front', 'three_quarter']; // subjects: [{name, description, pictures: [first, last] | null, audio: number | null}]. This is the one JS place for // the wording; its Python twin is subject_definitions() and tests/fixtures/subject_definitions.json holds the cases @@ -34,15 +34,22 @@ export function subjectDefinitions(subjects) { const IMAGE = /\.(png|jpe?g|webp|bmp|tiff?)$/i; const enabled = row => row && row.enabled !== false; // The number H3 gives each reference picture: images in list order (connected inputs come after the list). -// A part that continues a video keeps the first CONTINUE_PICTURES of them (continuation.PICTURES_MAX). +// A part that continues a video keeps CONTINUE_PICTURES of them (continuation.PICTURES_MAX): the first ones, or, with two or +// more characters, the last picture of each (the face) and then the pictures of no character (continuation.pick_pictures). export const CONTINUE_PICTURES = 3; -function pictureNumbers(rows, limit = Infinity) { - const numbers = new Map(); - let count = 0; +function pictureNumbers(rows, continued = false) { + const picked = []; rows.forEach((row, index) => { - if (enabled(row) && row.role === 'reference' && IMAGE.test(row.file || '') && count < limit) numbers.set(index, ++count); + if (enabled(row) && row.role === 'reference' && IMAGE.test(row.file || '')) picked.push(index); }); - return numbers; + let kept = picked; + if (continued) { + const last = new Map(); + picked.forEach(index => { if (rows[index].cast?.id) last.set(rows[index].cast.id, index); }); + kept = last.size < 2 ? picked : [...last.values()].sort((a, b) => a - b).concat(picked.filter(index => !rows[index].cast?.id)); + kept = kept.slice(0, CONTINUE_PICTURES); + } + return new Map(kept.map((index, at) => [index, at + 1])); } // The characters present in the media list, in list order, with the rows that carry each one's pictures. @@ -86,8 +93,8 @@ export function mentionRanges(text, people) { // The prompt a request carries. Characters mentioned in the text are numbered in order of first mention; one whose // pictures are gone is written as a plain name. `active: false` (first/last frames) leaves every name plain too, since // pictures of people cannot go with those. `continued` (the part continues a video) lets the characters stay: the part -// keeps the first three reference pictures and its frames come after them, so a character whose pictures are not among -// those three is written as a plain name. +// keeps three reference pictures (see pictureNumbers) and its frames come after them, so a character whose pictures are +// not among those three is written as a plain name. export function expandCast(text, rows, {active = true, continued = false} = {}) { text = String(text ?? ''); const list = Array.isArray(rows) ? rows : []; @@ -95,7 +102,7 @@ export function expandCast(text, rows, {active = true, continued = false} = {}) const ranges = mentionRanges(text, people); if (!ranges.length) return {text, subjects: []}; if (!continued && list.some(row => enabled(row) && (row.role === 'first' || row.role === 'last'))) active = false; - const numbers = pictureNumbers(list, continued ? CONTINUE_PICTURES : Infinity); + const numbers = pictureNumbers(list, continued); const order = []; let result = '', at = 0; for (const range of ranges) { @@ -123,13 +130,20 @@ export function expandCast(text, rows, {active = true, continued = false} = {}) return {text: subjectDefinitions(subjects) + '\n\n' + result, subjects}; } -// The pictures of a card to put in the media list: the card's own order after the preferred labels, the first six. +// The pictures of a card to put in the media list, as the Agent carries them (agent/drama.py person_pictures): the sheet of +// every view (the card's own, else the one the library made), then the face close-up, else the front view. A card with no +// sheet (one picture, or none could be made) carries its face alone. export function cardPictures(card) { const rank = new Map(PICTURE_ORDER.map((label, index) => [label, index])); - return (card?.images || []).map((image, index) => ({image, index})) - .filter(({image}) => image?.media?.file && IMAGE.test(image.media.file)) + const usable = (card?.images || []).map((image, index) => ({image, index})) + .filter(({image}) => image?.media?.file && IMAGE.test(image.media.file)); + const views = usable.filter(({image}) => image.label !== 'sheet') .sort((a, b) => (rank.get(a.image.label) ?? PICTURE_ORDER.length) - (rank.get(b.image.label) ?? PICTURE_ORDER.length) || a.index - b.index) - .slice(0, MAX_PICTURES_EACH).map(({image}) => image.media.file); + .map(({image}) => image.media.file); + const sheet = usable.find(({image}) => image.label === 'sheet')?.image.media.file + || (IMAGE.test(card?.sheet?.file || '') ? card.sheet.file : null); + if (!sheet) return views.slice(0, 1); + return [sheet, ...views.slice(0, 1).filter(file => file !== sheet)]; } // Whether a character can be mentioned now; `reason` is the one thing in the way. diff --git a/web/editor_references.js b/web/editor_references.js index 4bd6f808..9476cbfc 100644 --- a/web/editor_references.js +++ b/web/editor_references.js @@ -4,8 +4,17 @@ const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; const IMAGE = /\.(png|jpe?g|webp|bmp|tiff?)$/i; const MAX = 32; -// The engine keeps the first three references of a part that continues. +// The engine keeps three references of a part that continues: the first three, or, with two or more characters, +// the last picture of each (the face) and then the others (continuation.pick_pictures). const CONTINUE_MAX = 3; +const personOf = ({row}) => typeof row.cast?.id === 'string' && row.cast.id ? row.cast.id : null; +function pickContinued(list) { + const last = new Map(); + list.forEach((entry, at) => { const person = personOf(entry); if (person) last.set(person, at); }); + if (last.size < 2) return list.slice(0, CONTINUE_MAX); + const faces = [...last.values()].sort((a, b) => a - b).map(at => list[at]); + return [...faces, ...list.filter(entry => !personOf(entry))].slice(0, CONTINUE_MAX); +} export function createEditorReferences({t, source, size = () => null, error = () => {}, notify = () => {}}) { const root = el('div', 'fv-ed-refs'), chips = el('div', 'fv-ed-ref-chips'); const add = el('button', 'fv-ed-ref-add'); add.type = 'button'; @@ -20,7 +29,7 @@ export function createEditorReferences({t, source, size = () => null, error = () const isImage = file => IMAGE.test(file || ''); // The pictures this part uses: all of them, or the first reference images of a part that continues // (sound or video references do not count there, as the engine skips them). - const used = () => continuing ? active().filter(({row}) => (row.role || 'reference') === 'reference' && isImage(row.file)).slice(0, CONTINUE_MAX) : active(); + const used = () => continuing ? pickContinued(active().filter(({row}) => (row.role || 'reference') === 'reference' && isImage(row.file))) : active(); const full = () => continuing ? used().length >= CONTINUE_MAX : active().length >= MAX; // On hover without a full stop; in the notice line with one. const waitText = () => t('Available after generation finishes', '生成完成后可改'); diff --git a/web/freevideo.js b/web/freevideo.js index f027031c..73655897 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -351,8 +351,8 @@ function mediaPanel(node, mount = null) { let uploading = false; serialized.serializeValue = () => { if (node.isUploading) throw new Error(text("Wait for media uploads to finish before running", "请等待素材上传完成后再生成")); - // The run takes only what it uses; the editing data of a picture stays in the workflow. - try { return JSON.stringify(read().map(({file, role, enabled}) => ({file, role, enabled}))); } catch { return serialized.value; } + // The run takes only what it uses (and which character a picture belongs to); the editing data of a picture stays in the workflow. + try { return JSON.stringify(read().map(({file, role, enabled, cast}) => ({file, role, enabled, ...(typeof cast?.id === 'string' && cast.id ? {cast: {id: cast.id}} : {})}))); } catch { return serialized.value; } }; async function upload(files) { if (node.isUploading || !files.length) return; From 5e2cc3dfb2dd0fd7bc5cdfd1176040c4ae180219 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 05/21] Add white-model guidance: Agent scene submit, page render and upload, ready-made scenes - Agent: reference videos may carry early_steps - Agent eval: whitebox_chain task, fake white-model server, scene wait - White model: accept any browser-encoded video by content, at least one second - White model tests: clips of a valid length - White model tests: keep clip colours in range - Whitebox: reference guides the first 3 steps; up to 3 scene retries - Whitebox: follow steps scale with first-pass length - Whitebox: a tried scene asks its conversation once it is ready - Whitebox: a webm recording is stored as a real mp4 - Whitebox: capability notes name the feature and the page - Whitebox: keep a recording mp4 cannot hold as it came --- freevideo_engine/agent/cli.py | 16 +- freevideo_engine/agent/errors.py | 1 + freevideo_engine/agent/guide.py | 17 +- freevideo_engine/agent/jobs.py | 8 +- freevideo_engine/agent/mcp.py | 8 +- freevideo_engine/agent/media.py | 1 + freevideo_engine/agent/routes.py | 106 ++++++++++- freevideo_engine/agent/schema.py | 48 ++++- freevideo_engine/agent/spec.py | 6 +- freevideo_engine/agent/whitebox.py | 211 +++++++++++++++++++++ freevideo_engine/agent/whitebox_presets.py | 26 +++ freevideo_engine/comfy_assets.py | 6 +- 12 files changed, 438 insertions(+), 16 deletions(-) create mode 100644 freevideo_engine/agent/whitebox.py create mode 100644 freevideo_engine/agent/whitebox_presets.py diff --git a/freevideo_engine/agent/cli.py b/freevideo_engine/agent/cli.py index 3d91a67e..06259869 100644 --- a/freevideo_engine/agent/cli.py +++ b/freevideo_engine/agent/cli.py @@ -121,6 +121,15 @@ def build_parser(): if name == 'operations': sub.add_argument('--operations-file', help='JSON array of operations (- = stdin)') schema.cli_arguments(sub, schema.image_schema(name)) + command = commands.add_parser('whitebox', help='Render a rough 3D white-model reference video in the user\'s open page') + whitebox = command.add_subparsers(dest='operation', required=True) + for name, (method, path, summary) in schema.WHITEBOX_OPERATIONS.items(): + sub = whitebox.add_parser(name, help=summary) + if '{id}' in path: + sub.add_argument('id') + if name == 'submit': + sub.add_argument('file', nargs='?', help='JSON {code, summary?} request (- = stdin); options override it') + schema.cli_arguments(sub, schema.whitebox_schema(name)) command = commands.add_parser('sessions', help='Share messages and plans with the user in a work conversation') sessions = command.add_subparsers(dest='operation', required=True) for name in ('list', 'create', 'events', 'post'): @@ -236,9 +245,10 @@ def source(value): schema.validate(schema.CAST, values['id']) path = path.replace('{id}', values['id']) result = client.get(path, **value) if method == 'GET' else client.send(method, path, value) - elif args.command in ('projects', 'images'): - table, shape_of, ident = ((schema.PROJECT_OPERATIONS, schema.project_schema, schema.PROJECT) - if args.command == 'projects' else (schema.IMAGE_OPERATIONS, schema.image_schema, schema.IMAGE)) + elif args.command in ('projects', 'images', 'whitebox'): + table, shape_of, ident = {'projects': (schema.PROJECT_OPERATIONS, schema.project_schema, schema.PROJECT), + 'images': (schema.IMAGE_OPERATIONS, schema.image_schema, schema.IMAGE), + 'whitebox': (schema.WHITEBOX_OPERATIONS, schema.whitebox_schema, schema.WHITEBOX)}[args.command] method, path, _ = table[args.operation] shape = shape_of(args.operation) value = read(args.file) if getattr(args, 'file', None) else {} diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index 79a676c6..1653aa5f 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -33,6 +33,7 @@ 'not_found': (404, False, 'The requested job, film or asset does not exist.', 'List /freevideo/agent/v1/jobs, /drama or /library and select an existing ID.'), 'no_video': (409, False, 'The source job has no finished video.', 'GET /freevideo/agent/v1/jobs/{id} and wait for a succeeded job containing result.video.'), 'not_ready': (503, True, 'The FreeVideo installation is not ready.', 'Open Settings → Install / repair and finish setup. If the installation token is unsafe or incomplete, restore a regular non-symlink agent/token owned by your account, or remove that token and restart FreeVideo.'), + 'page_required': (409, True, 'This needs the FreeVideo page open in a browser.', 'Ask the user to open FreeVideo in their browser, then retry.'), 'busy': (409, True, 'The requested resource is currently in use.', 'Wait retry_after_seconds, read GET /freevideo/agent/v1/queue, then retry the request.'), 'conflict': (409, False, 'The requested operation conflicts with the current state.', 'Read the current job with GET /freevideo/agent/v1/jobs/{id} before retrying.'), 'dependency_failed': (409, False, 'A required job did not succeed.', 'Read the dependency jobs, resume failed generation, then submit the dependent job with a new ref.'), diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 07b4cece..8789facf 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -4,7 +4,9 @@ own prompt rules; the enhancement job applies the same contract. """ from ..default_prompt import DEFAULT_PROMPT -from .schema import CAST_OPERATIONS, DRAMA_OPERATIONS, IMAGE_OPERATIONS, PROJECT_OPERATIONS, SESSION_OPERATIONS +from .schema import CAST_OPERATIONS, DRAMA_OPERATIONS, IMAGE_OPERATIONS, PROJECT_OPERATIONS, SESSION_OPERATIONS, WHITEBOX_OPERATIONS + +from .whitebox import SCENE_PROMPT SOURCE = 'https://github.com/MiniMax-AI/MiniMax-H3/tree/main/skills/h3-prompt-writing' @@ -133,6 +135,17 @@ 'A new paint layer is empty, so heal and clone there have nothing to work from; paint and gradients go on a paint layer above a locked picture.', 'On 409 image_changed, recompute from error.image and its current_revision. POST /images/{id}/render makes a PNG input file from the revision you read; use the job result file as a picture source.', ], + 'whitebox': [ + 'A white model is a rough 3D stand-in for the shot (boxes, camera, motion) rendered in the user\'s FreeVideo page. Used as a reference video that guides only the first steps, the final video keeps its layout and motion.', + 'Use it only when the user asked for it: the user message carries whitebox=true, or the user says so. Never choose it yourself, and never suggest it as a default.', + 'Write the scene with the prompt in whitebox_scene_prompt (replace {SHOT} with the user\'s one-sentence shot). POST /whitebox with the code and a one-sentence summary; it needs the page open and answers page_required otherwise: tell the user to open FreeVideo.', + 'Read GET /whitebox/{id} every few seconds. On failed, fix the code from error and send it again, at most 3 retries; if they all fail, tell the user and stop, with no silent fallback.', + 'On ready, ask the user to approve it first: write a confirm event with whitebox={work, seconds, summary}. After approval, generate with media.references=[{"library": work, "early_steps": K}] where K is about 3/8 of the first-pass steps rounded (8 steps -> 3, 20 steps -> 8; early_ratio in the ready answer), so the white model keeps the same share of the schedule on every quality level. The prompt describes the real look of the final video, not white boxes. Use no other kind of reference with it unless the user gave one.', + 'When the user wants a change, write a new scene from the edited description and repeat; the earlier white model stays in the library.', + ], + 'whitebox_scene_prompt': SCENE_PROMPT, + 'whitebox_routes': [dict(operation=name, method=method, path=path, description=summary) + for name, (method, path, summary) in WHITEBOX_OPERATIONS.items()], 'image_routes': [dict(operation=name, method=method, path=path, description=summary) for name, (method, path, summary) in IMAGE_OPERATIONS.items()], 'sessions': [ @@ -272,6 +285,8 @@ ('GET', '/library/{day}/{run}/file'): '下载已保存的视频', ('GET', '/library/{day}/{run}/thumbnail'): '查看已保存的视频缩略图', ('GET', '/queue'): '查看队列顺序及任务属于 Agent 还是用户', + ('POST', '/whitebox'): '发送你写好的白模场景代码，由用户打开的 FreeVideo 页面渲染成短视频并存入作品库；页面没打开返回 page_required', + ('GET', '/whitebox/{id}'): '查看白模场景进度：waiting、rendering、ready（work 是作品库编号，放进 media.references 并加 early_steps）或 failed（error）', ('GET', '/editor/sources/content'): '下载剪辑项目的源素材或缩略图；使用 GET /projects/{id} 返回的素材标识', ('GET', '/inbox'): '等待当前工作会话中用户的消息或答复，最多 50 秒；again=true 表示暂时没有，再调一次', ('POST', '/drama/plan'): '预览多镜头影片的剧本和各镜头拍摄条数，不保存、不提交。' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index b56e89b1..98be19e6 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -493,6 +493,8 @@ def __init__(self, port, directory, *, clock=now): self.projects = Projects(port.output_root) from .images import Images self.images = Images(self, self.directory.parent / 'images') + from .whitebox import Whitebox + self.whitebox = Whitebox(self.directory.parent / 'whitebox', port.output_root) self.load() from .productions import Productions self.productions = Productions(self) @@ -1515,8 +1517,10 @@ def resolve(): assets.append(dict(file=self.media_file(row, spec['media'][role], role, target=target), role=role)) for i, item in enumerate(spec['media'].get('references', []), 1): - assets.append(dict(file=self.media_file(row, item, 'reference-%02d' % i, target=target), - role='reference')) + asset = dict(file=self.media_file(row, item, 'reference-%02d' % i, target=target), role='reference') + if 'early_steps' in item: + asset['early_steps'] = item['early_steps'] + assets.append(asset) return assets assets = resolve() graph = specs.comfy_prompt(spec, node, prompt=prompt, assets=assets) diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index 321e363f..62ce0d27 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -71,6 +71,10 @@ def tool_list(): rows.extend(dict(name=PROJECT_TOOLS[name], title='Editing project: ' + name, description=summary, annotations=read if method == 'GET' else write) for name, (method, _, summary) in PROJECT_OPERATIONS.items()) + from .schema import WHITEBOX_OPERATIONS, WHITEBOX_TOOLS + rows.extend(dict(name=WHITEBOX_TOOLS[name], title='White model: ' + name, description=summary, + annotations=read if method == 'GET' else write) + for name, (method, _, summary) in WHITEBOX_OPERATIONS.items()) from .schema import IMAGE_OPERATIONS, IMAGE_TOOLS rows.extend(dict(name=IMAGE_TOOLS[name], title='Image: ' + name, description=summary, annotations=read if method == 'GET' else write) @@ -316,7 +320,7 @@ def tool(self, identity, name, args, token): images.append(dict(type='image', **value['previous'])) value['previous'] = True return value, images - from .schema import IMAGE_TOOLS, PROJECT_TOOLS, TOOL_ENDPOINTS + from .schema import IMAGE_TOOLS, PROJECT_TOOLS, TOOL_ENDPOINTS, WHITEBOX_TOOLS from .schema import SESSION_OPERATIONS, SESSION_TOOLS if name in SESSION_TOOLS.values(): operation = next(op for op, tool in SESSION_TOOLS.items() if tool == name) @@ -326,7 +330,7 @@ def tool(self, identity, name, args, token): if method == 'GET' and wait is not None: args['wait'] = wait return (c.get(path, **args) if method == 'GET' else c.post(path, args)), [] - if name in PROJECT_TOOLS.values() or name in IMAGE_TOOLS.values(): + if name in PROJECT_TOOLS.values() or name in IMAGE_TOOLS.values() or name in WHITEBOX_TOOLS.values(): method, path = TOOL_ENDPOINTS[name] if '{id}' in path: path = path.replace('{id}', args.pop('id')) diff --git a/freevideo_engine/agent/media.py b/freevideo_engine/agent/media.py index c2038acd..a274a13f 100644 --- a/freevideo_engine/agent/media.py +++ b/freevideo_engine/agent/media.py @@ -676,6 +676,7 @@ def _stream_seconds(stream, container): def _video_stream(container): import av + import av.stream attached = getattr(getattr(av.stream, 'Disposition', None), 'attached_pic', 0) for stream in container.streams.video: if not (attached and stream.disposition & attached): diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 69534e6b..a7b05b45 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -30,6 +30,8 @@ from .jobs import ACTIVE, DONE, JobError, Jobs, chain_error, library_view PREFIX = '/freevideo/agent/v1' +from .whitebox import MAX_VIDEO_BYTES as MAX_WHITEBOX_VIDEO +from .whitebox_presets import PRESETS, preset as whitebox_preset # What an empty wait tells the agent: the conversation is open for as long as the user keeps the page. KEEP_WAITING = ('Nothing new yet. The user is still in FreeVideo and may speak at any moment: call this wait ' 'again now. An empty wait never ends the conversation; do not end your turn or write a summary.') @@ -171,6 +173,13 @@ def features(continuation, vlm): **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen')}} +def whitebox_support(jobs): + """White-model guidance needs the user's open page to render, and the Mac engine cannot use it yet.""" + if quality_tiers.on_mac(): + return dict(available=False, needs_page=True, note='Mac 暂不支持白模。') + return dict(available=True, needs_page=True, page_open=jobs.whitebox.page_open(), note='白模预览在 FreeVideo 页面中显示，需保持页面打开。') + + def capabilities(port, jobs): from ..lora_triggers import list_triggers from .. import __version__ @@ -200,6 +209,7 @@ def capabilities(port, jobs): api_version=API_VERSION, freevideo_version=release or __version__, build=__version__, ready=ready, problem=problem, continuation=continuation, features=features(continuation, vlm), + whitebox=whitebox_support(jobs), device=dict(backend=machine.get('device_backend', 'cuda') if machine else None, vram_limit_gib=machine.get('vram_gib'), ram_limit_gib=machine.get('ram_gib')), kinds={'generate': 'Video from a prompt and optional keyframes or references', @@ -960,7 +970,10 @@ def write(): if store.stopped(sid): # Speaking again continues a conversation the user stopped earlier. store.append(sid, dict(type='control', action='resumed'), by='user') - row = store.append(sid, dict(type='message', text=value['text'], mid=value['id']), by='user') + message = dict(type='message', text=value['text'], mid=value['id']) + if value.get('whitebox'): + message['whitebox'] = True + row = store.append(sid, message, by='user') fold['sent'][value['id']] = row['seq'] fold['ids'][row['seq']] = value['id'] while len(fold['sent']) > 500: @@ -1900,6 +1913,97 @@ async def handler(request): for operation, (method, suffix, _) in contract.IMAGE_OPERATIONS.items(): getattr(routes, method.lower())(PREFIX + suffix)(image_handler(operation)) + def whitebox_handler(operation): + async def handler(request): + scenes = jobs.whitebox + if operation == 'get': + deadline = time.monotonic() + request[query_key].get('wait', 0) + while True: + view = await asyncio.to_thread(scenes.read, request.match_info['id']) + if view['status'] in ('ready', 'failed') or time.monotonic() >= deadline: + return reply(view) + await asyncio.sleep(.5) + value = await body(request) + if not scenes.page_open(): + raise JobError('page_required', 'No FreeVideo page is open to render the scene') + return reply(await asyncio.to_thread(scenes.create, value['code'], value.get('summary'), + request.get('agent_session')), 201) + return guarded(handler) + + for operation, (method, suffix, _) in contract.WHITEBOX_OPERATIONS.items(): + getattr(routes, method.lower())(PREFIX + suffix)(whitebox_handler(operation)) + + # The page renders scenes: it asks for work, then sends back the video or the reason it failed. + def whitebox_confirm(notice): + store = work_sessions() + if not notice or store is None: + return + try: + store.append(notice['session'], dict(type='confirm', id='cf' + secrets.token_hex(5), state='asked', + question=notice['question'], whitebox=dict(work=notice['work'], seconds=notice['seconds'], + summary=notice['summary'])), by='builtin') + except Exception: + logging.getLogger(__name__).warning('White-model confirmation failed', exc_info=True) + + @routes.get('/freevideo/agent/whitebox/next') + @guarded + async def whitebox_next(request): + return reply(dict(scene=await asyncio.to_thread(jobs.whitebox.claim))) + + @routes.get('/freevideo/agent/whitebox/presets') + @guarded + async def whitebox_presets(request): + return reply(dict(presets=[{k: row[k] for k in ('id', 'name', 'shot', 'prompt')} for row in PRESETS])) + + @routes.post('/freevideo/agent/whitebox/try') + @guarded + async def whitebox_try(request): + if request.content_type != 'application/json': + return problem('unsupported_media_type', 'Send JSON', 415) + value = await body(request) + row = whitebox_preset(value.get('preset') if isinstance(value, dict) else None) + if row is None: + raise JobError('not_found', 'No such ready-made scene') + sid = value.get('session') + notice = None + if sid is not None: + store = work_sessions() + if store is None or not isinstance(sid, str) or not store.has(chat['client'], sid): + raise JobError('not_found', 'No such conversation') + # The conversation gets the confirmation card once the page has rendered the scene. + notice = dict(question=row['prompt'], summary=row['name']) + scene = await asyncio.to_thread(jobs.whitebox.create, row['code'], row['shot'], sid, notice) + return reply(dict(scene, preset=row['id'], name=row['name'], prompt=row['prompt']), 201) + + @routes.post('/freevideo/agent/whitebox/{id}/video') + @guarded + async def whitebox_video(request): + scenes = jobs.whitebox + with tempfile.TemporaryDirectory(prefix='.whitebox-', dir=port.input_root) as temporary: + target = Path(temporary) / 'scene.mp4' + size = 0 + with target.open('xb') as output: + async for chunk in request.content.iter_chunked(256 * 1024): + size += len(chunk) + if size > MAX_WHITEBOX_VIDEO: + raise JobError('too_large', 'Upload exceeds limit', limit_bytes=MAX_WHITEBOX_VIDEO) + output.write(chunk) + if not size: + raise JobError('invalid_media', 'Supply the rendered video') + async with upload_decoders: + scene = await asyncio.to_thread(scenes.finish, request.match_info['id'], target) + await asyncio.to_thread(whitebox_confirm, scenes.take_notice(scene['id'])) + return reply(scene) + + @routes.post('/freevideo/agent/whitebox/{id}/failed') + @guarded + async def whitebox_failed(request): + if request.content_type != 'application/json': + return problem('unsupported_media_type', 'Send JSON', 415) + value = await body(request) + message = value.get('message') if isinstance(value, dict) else None + return reply(await asyncio.to_thread(jobs.whitebox.fail, request.match_info['id'], message)) + @routes.get(PREFIX + '/editor/sources/content') @guarded async def editor_source_content(request): diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index e19542c6..d72982ca 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -107,11 +107,15 @@ def edit_document(): 'x-editor-document': True} -def media(image=False, *, local=False, clip=False, video=False): +def media(image=False, *, local=False, clip=False, video=False, early=False): properties = dict(file=FILE, from_job=JOB, library=SAVED, frame=FRAME, edit={'oneOf': [edit_recipe(), edit_document()]}, label=string(80)) if local: properties['path'] = string(4096) + if early: + properties['early_steps'] = number(1, 32, integer=True, description= + 'A reference video only: it guides the first pass for this many steps and is dropped afterwards, ' + 'including from the second pass. Omitted: the video guides every step.') if clip: properties.pop('frame') properties.pop('edit') @@ -139,7 +143,7 @@ def common(): def generation_media(*, local=False, enhance=False): return obj({'first': nullable(media(True, local=local)), 'last': nullable(media(True, local=local)), - 'references': nullable(array(media(enhance, local=local), high=8 if enhance else 32))}, + 'references': nullable(array(media(enhance, local=local, early=not enhance), high=8 if enhance else 32))}, allOf=[{'if': {'required': ['references'], 'properties': {'references': {'type': 'array', 'minItems': 1}}}, 'then': {'properties': {'first': {'type': 'null'}, 'last': {'type': 'null'}}}}]) @@ -391,6 +395,13 @@ def route(summary, body=None, query=None, **kw): else: extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen') else {})) routes[method, suffix] = route(summary, **extra) + for name, (method, suffix, summary) in WHITEBOX_OPERATIONS.items(): + extra = dict(path=obj({'id': WHITEBOX}, ('id',))) if '{id}' in suffix else {} + if method == 'GET': + extra['query'] = whitebox_schema(name) + else: + extra.update(body=whitebox_schema(name), status=201) + routes[method, suffix] = route(summary, **extra) for name, (method, suffix, summary) in IMAGE_OPERATIONS.items(): shape = image_schema(name) extra = dict(path=obj({'id': IMAGE}, ('id',))) if '{id}' in suffix else {} @@ -408,12 +419,22 @@ def route(summary, body=None, query=None, **kw): ('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None), ('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY), ('/chat/delete', 'POST', obj({'session': SESSION}, ('session',))), - ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID, 'session': SESSION}, ('text', 'id'))), + ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID, 'session': SESSION, 'whitebox': {'type': 'boolean'}}, ('text', 'id'))), ('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'}, 'session': SESSION}, ('id', 'approve')))): routes[(method, '/freevideo/agent' + suffix)] = route('Studio Agent controls', body=body, auth=False, local=True) routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True), 'session': SESSION}) routes['GET', '/freevideo/agent/chat/video']['query'] = obj({'id': string(43, pattern=r'^[0-9]{4}-[0-9]{2}-[0-9]{2}/[0-9a-f]{32}$')}, ('id',)) + for method, suffix, summary, extra in ( + ('GET', '/next', 'The page asks for a white-model scene to render', {}), + ('GET', '/presets', 'The ready-made white-model scenes the page can try', {}), + ('POST', '/try', 'The page renders one ready-made white-model scene (no model writes code)', + dict(body=obj(dict(preset=string(32, pattern=r'^[a-z0-9-]{1,32}$'), session=SESSION), ('preset',)))), + ('POST', '/{id}/video', 'The page sends the rendered white-model video (the raw mp4 body)', + dict(path=obj({'id': WHITEBOX}, ('id',)))), + ('POST', '/{id}/failed', 'The page reports that a white-model scene could not be rendered', + dict(path=obj({'id': WHITEBOX}, ('id',)), body=obj(dict(message=string(500)), ('message',))))): + routes[method, '/freevideo/agent/whitebox' + suffix] = route(summary, auth=False, local=True, **extra) for suffix in ('', '/connect'): routes['GET', '/freevideo/agent' + suffix]['query'] = obj({'lang': enum(('zh', 'en'))}) for name, (method, suffix, summary) in SESSION_OPERATIONS.items(): @@ -539,6 +560,7 @@ def cast_view(): # changed with the page's commands (web/image_commands.js). Shapes follow the page's # own fields (lineHeight, shape width), not the /media/edit recipe's. IMAGE = string(34, pattern=r'^i-[0-9a-f]{32}$') +WHITEBOX = string(12, pattern=r'^wb[0-9a-f]{10}$') LAYER_ID = string(128, pattern=r'^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$') COLOR = string(32, pattern=r'^(?:#[0-9A-Fa-f]{3,4}|#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{8}|transparent)$') IMAGE_SIDE = number(1, 8192, integer=True) @@ -643,6 +665,20 @@ def image_document(): operations='freevideo_image_operations', render='freevideo_image_render') +WHITEBOX_OPERATIONS = { + 'submit': ('POST', '/whitebox', 'Send the white-model scene code you wrote (build(THREE), see GET /guide). The user\'s open FreeVideo page renders it to a short video that joins the library.'), + 'get': ('GET', '/whitebox/{id}', 'Read a white-model scene: waiting, rendering, ready (work = library ID for media.references with early_steps) or failed (error).'), +} +WHITEBOX_TOOLS = dict(submit='freevideo_whitebox_submit', get='freevideo_whitebox_read') + + +def whitebox_schema(name): + if name == 'submit': + return obj(dict(code=string(65536, description='JavaScript that defines build(THREE) and returns {scene, camera, update(t)}.'), + summary=string(500, description='One sentence on what the scene shows, for the confirm card.')), ('code',)) + return obj({'wait': WAIT}) + + def image_operation_fields(): """The image commands (IDC): op -> (fields, required, description).""" layer_id = LAYER_ID @@ -773,6 +809,7 @@ def session_event(*, stored=False, agent_only=False): 1, 20000), total_seconds=number(0)), ('type', 'shots'))] if not agent_only: variants[0]['properties']['mid'] = CHAT_ID + variants[0]['properties']['whitebox'] = {'type': 'boolean', 'description': 'The user chose a white-model preview for this request.'} from .errors import ERRORS variants += [obj(dict(type={'const': 'step'}, id=string(12, pattern=r'^st[0-9a-f]{10}$'), tool=string(200), summary=text, state=enum(('started', 'done', 'failed')), @@ -782,6 +819,8 @@ def session_event(*, stored=False, agent_only=False): obj(dict(type={'const': 'confirm'}, id=string(12, pattern=r'^cf[0-9a-f]{10}$'), question=text, estimate_seconds=number(0), state=enum(('asked', 'approved', 'declined')), action=enum(('start', 'reshoot')), refs=obj(dict(drama=PRODUCTION, shot=SLUG)), + whitebox=obj(dict(work=SAVED, seconds=number(0), summary=text), ('work',), + description='The rendered white-model preview the user is asked to approve.'), digest=string(64, pattern=r'^[0-9a-f]{64}$')), ('type', 'id', 'state'), **{'if': {'properties': {'state': {'const': 'asked'}}}, 'then': {'required': ['question']}}), @@ -1253,6 +1292,7 @@ def schemas(): cast={name: cast_schema(name) for name in CAST_OPERATIONS}, cast_view=cast_view(), projects={name: project_schema(name) for name in PROJECT_OPERATIONS}, images={name: image_schema(name) for name in IMAGE_OPERATIONS}, + whitebox={name: whitebox_schema(name) for name in WHITEBOX_OPERATIONS}, sessions=dict(event=session_event(stored=True), meta=session_view(), **{name: session_schema(name) for name in SESSION_OPERATIONS}), confirmations=confirm_settings(), image_result=image_result()) @@ -1377,6 +1417,8 @@ def openapi(): for name, (method, suffix, _) in CAST_OPERATIONS.items()}) TOOL_ENDPOINTS.update({PROJECT_TOOLS[name]: (method, suffix) for name, (method, suffix, _) in PROJECT_OPERATIONS.items()}) +TOOL_ENDPOINTS.update({WHITEBOX_TOOLS[name]: (method, suffix) + for name, (method, suffix, _) in WHITEBOX_OPERATIONS.items()}) TOOL_ENDPOINTS.update({IMAGE_TOOLS[name]: (method, suffix) for name, (method, suffix, _) in IMAGE_OPERATIONS.items()}) TOOL_ENDPOINTS.update({SESSION_TOOLS[name]: SESSION_OPERATIONS[name][:2] diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 75ee1404..a148bd7f 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -68,13 +68,13 @@ def new_id(): return 'j' + secrets.token_hex(6) -def media_item(value, path, *, image=False, frame_default=None): +def media_item(value, path, *, image=False, frame_default=None, early=False): if isinstance(value, str): value = {'file': value} value = dict(value) if isinstance(value, dict) else value if isinstance(value, dict) and frame_default is not None and any(k in value for k in ('from_job', 'library')): value.setdefault('frame', frame_default) - errors = contract.issues(contract.media(image, local=True), value, path) + errors = contract.issues(contract.media(image, local=True, early=early), value, path) if errors: raise SpecError(errors) return value @@ -152,7 +152,7 @@ def generation_media(value): return {} result = {role: media_item(value[role], '/media/' + role, image=True) for role in FRAMES if value.get(role) is not None} if value.get('references'): - result['references'] = [media_item(v, '/media/references/%d' % i) for i, v in enumerate(value['references'])] + result['references'] = [media_item(v, '/media/references/%d' % i, early=True) for i, v in enumerate(value['references'])] return result diff --git a/freevideo_engine/agent/whitebox.py b/freevideo_engine/agent/whitebox.py new file mode 100644 index 00000000..95a157bd --- /dev/null +++ b/freevideo_engine/agent/whitebox.py @@ -0,0 +1,211 @@ +"""White-model scenes: the Agent writes the scene, the user's page renders it, the video joins the library. + +The video is a rough stand-in for the real shot (boxes, camera, motion). A confirmed one is then used as a +reference video that only guides the first steps (early_steps), so the final video keeps its layout and motion.""" +import json +import os +import secrets +import shutil +import threading +import time +import uuid +from pathlib import Path + +from . import media +from .jobs import JobError + +ID = r'^wb[0-9a-f]{10}$' +MAX_CODE = 64 * 1024 +MAX_VIDEO_BYTES = 64 * 1024 * 1024 +MAX_SECONDS = 12 +MIN_SECONDS = 1 +KEPT = 20 +PAGE_FRESH = 15. # A page that asked for work this recently is open. +RENDER_LEASE = 150. # A claimed scene nobody finished goes back to the queue once, then fails. +EARLY_STEPS = 3 # For an 8-step first pass; other lengths scale by 3/8 (20 steps -> 8). +FOLLOW_RATIO = 3 / 8 +SCENE_PROMPT = """You are a 3D blocking programmer. From one shot description, write a Three.js scene that builds a white model with as few shapes as possible: only boxes, cylinders, spheres, cones and planes, with no textures, no external files and no text. It is a composition and motion reference for a video model, not a finished picture: shapes must be recognisable, with plausible positions, scale and motion paths. + +Shot description: {SHOT} + +Output rules (follow strictly): +1. Output only one ES module that exports `export function build(THREE)` and returns `{ scene, camera, update }`. +2. `update(t)`: t is in seconds, 0 to 5.0. Every object and the camera depend only on t (a pure function: no Date, no Math.random, no accumulated state). The whole action in the description must finish within 5 seconds, and the camera and the main subject stay visible throughout. +3. Right-handed, y up, metres, true scale (a person about 1.7 m, a cat about 0.3 m, a room about 3 m high). +4. Camera: `PerspectiveCamera(fov, 960/544, 0.1, 200)`; set its position and `lookAt` inside `update` on every frame; move it with smooth interpolation (smoothstep or a spline): no jitter, no jumps. +5. Materials: always `new THREE.MeshStandardMaterial({ color, roughness: 1 })`, one simple colour per object that differs from its neighbours (a striking colour for the subject; grey, beige or light blue for the background). Set `mesh.name`; the subject's name starts with `hero`. +6. Lights: one HemisphereLight and one DirectionalLight with shadows; set `scene.background` to a light sky or wall colour. No fog, no post-processing. +7. At most 80 meshes. Build everything in `build`; `update` only changes positions, rotations and scales. No dependency other than the module itself. +8. Write motion as interpolation between key points (position, orientation, scale where needed); no physics simulation. + +Return only the code. + +Fix list for a retry: THREE.Geometry no longer exists (use BufferGeometry classes); never create objects in `update`; camera position and lookAt target must differ; if too dark use hemisphere light >= 1.2 and directional >= 1.4; if the camera enters an object, move its path farther or higher.""" + + +def _remux(source, target): + """A webm recording becomes a real mp4 without re-encoding: the library opens every work as video.mp4.""" + import av + with av.open(str(source)) as incoming, av.open(str(target), 'w', format='mp4') as outgoing: + video = incoming.streams.video[0] + stream = outgoing.add_stream_from_template(video) + for packet in incoming.demux(video): + if packet.dts is None: + continue + packet.stream = stream + outgoing.mux(packet) + + +class Whitebox: + def __init__(self, directory, output_root, clock=time.time): + self.directory, self.root, self.clock = Path(directory), Path(output_root), clock + self.lock = threading.RLock() + self.seen = None + self.records = {} + try: + for path in sorted(self.directory.glob('wb*.json')): + try: + record = json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError): + continue + if isinstance(record, dict) and record.get('id') == path.stem: + self.records[record['id']] = record + except OSError: + pass + + # Storage ------------------------------------------------------------------------------------------- + def _save(self, record): + self.directory.mkdir(parents=True, exist_ok=True) + path = self.directory / (record['id'] + '.json') + temporary = path.with_name(path.name + '.tmp') + temporary.write_text(json.dumps(record, ensure_ascii=False), encoding='utf-8') + os.replace(temporary, path) + stale = sorted(self.records.values(), key=lambda r: r['created_at'])[:-KEPT] + for old in stale: + self.records.pop(old['id'], None) + (self.directory / (old['id'] + '.json')).unlink(missing_ok=True) + + def _get(self, identity): + record = self.records.get(identity) + if record is None: + raise JobError('not_found', 'No such white-model scene') + return record + + def _expire(self, record): + """A claim the page never finished: back to waiting once, then failed.""" + if record['status'] == 'rendering' and self.clock() - record['claimed_at'] > RENDER_LEASE: + if record['attempts'] < 2: + record['status'] = 'waiting' + else: + record.update(status='failed', error='The page did not finish rendering the scene.') + self._save(record) + + def view(self, record): + self._expire(record) + result = {k: record[k] for k in ('id', 'status', 'summary', 'created_at')} + result['page_open'] = self.page_open() + if record['status'] == 'ready': + result.update(work=record['work'], seconds=record['seconds'], early_steps=EARLY_STEPS, early_ratio=FOLLOW_RATIO) + if record['status'] == 'failed': + result['error'] = record['error'] + return result + + def page_open(self): + return self.seen is not None and self.clock() - self.seen <= PAGE_FRESH + + # Agent --------------------------------------------------------------------------------------------- + def create(self, code, summary, session=None, notice=None): + if not isinstance(code, str) or not code.strip() or len(code.encode('utf-8')) > MAX_CODE: + raise JobError('invalid_request', 'Scene code must be nonempty text of at most %d bytes' % MAX_CODE) + if 'build' not in code: + raise JobError('invalid_request', 'Scene code must define build(THREE)') + with self.lock: + record = dict(id='wb' + secrets.token_hex(5), status='waiting', code=code, summary=str(summary or '')[:500], + session=session, created_at=self.clock(), claimed_at=None, attempts=0, work=None, + seconds=None, error=None, notice=notice) + self.records[record['id']] = record + self._save(record) + return self.view(record) + + def take_notice(self, identity): + """What the server should tell the user's conversation once the scene is ready; handed out once.""" + with self.lock: + record = self._get(identity) + notice = record.get('notice') + if record['status'] != 'ready' or not notice: + return None + record['notice'] = None + self._save(record) + return dict(notice, session=record['session'], work=record['work'], seconds=record['seconds']) + + def read(self, identity): + with self.lock: + return self.view(self._get(identity)) + + # Page ---------------------------------------------------------------------------------------------- + def claim(self): + """The oldest waiting scene with its code, or None; asking at all tells the Agent a page is open.""" + with self.lock: + self.seen = self.clock() + for record in sorted(self.records.values(), key=lambda r: r['created_at']): + self._expire(record) + if record['status'] == 'waiting': + record.update(status='rendering', claimed_at=self.clock(), attempts=record['attempts'] + 1) + self._save(record) + return dict(id=record['id'], code=record['code'], summary=record['summary']) + return None + + def fail(self, identity, message): + with self.lock: + record = self._get(identity) + if record['status'] in ('ready', 'failed'): + return self.view(record) + record.update(status='failed', error=str(message or 'The scene failed to render')[:500]) + self._save(record) + return self.view(record) + + def finish(self, identity, upload): + """Store the page's rendered mp4 as a library work and mark the scene ready.""" + with self.lock: + record = self._get(identity) + if record['status'] == 'ready': + return self.view(record) + if record['status'] != 'rendering': + raise JobError('conflict', 'The scene is not being rendered') + upload = Path(upload) + if upload.stat().st_size > MAX_VIDEO_BYTES: + raise JobError('too_large', 'The rendered video is too large') + try: + info, suffix = media.validate_upload(upload) + except (OSError, ValueError, SyntaxError): + raise JobError('invalid_media', 'The rendered video cannot be decoded') from None + # The page records with whatever its browser can encode (H.264, VP9, mp4 or webm): content decides, not the name. + if info['kind'] != 'video' or suffix not in ('.mp4', '.webm') or not MIN_SECONDS <= info['seconds'] <= MAX_SECONDS: + raise JobError('invalid_media', 'The rendered video must last %d to %d seconds' % (MIN_SECONDS, MAX_SECONDS)) + day = time.strftime('%Y-%m-%d') + folder = self.root / 'FreeVideo' / day / uuid.uuid4().hex + folder.mkdir(parents=True) + try: + try: + if suffix != '.webm': + raise ValueError('already mp4') + _remux(upload, folder / 'video.mp4') + except Exception: + # Codecs mp4 cannot hold (VP8) keep the recording as it came; the content still decodes. + shutil.copyfile(upload, folder / 'video.mp4') + (folder / 'prompt.txt').write_text(record['summary'], encoding='utf-8') + fps = info.get('fps') or 24. + report = dict(success=True, geometry=dict(width=info['width'], height=info['height'], + frames=info.get('frames') or round(info['seconds'] * fps), fps=fps, seconds=info['seconds']), + request_seconds=0, video={}, encoding={}, whitebox=dict(scene=record['id'])) + request = folder / 'video.request.json' + temporary = request.with_name(request.name + '.tmp') + temporary.write_text(json.dumps(report), encoding='utf-8') + os.replace(temporary, request) + except BaseException: + shutil.rmtree(folder, ignore_errors=True) + raise + with self.lock: + record.update(status='ready', work='%s/%s' % (day, folder.name), seconds=info['seconds']) + self._save(record) + return self.view(record) diff --git a/freevideo_engine/agent/whitebox_presets.py b/freevideo_engine/agent/whitebox_presets.py new file mode 100644 index 00000000..5ea8f0d1 --- /dev/null +++ b/freevideo_engine/agent/whitebox_presets.py @@ -0,0 +1,26 @@ +"""Ready-made white-model scenes (scene code written to the scene prompt); 'try' and the built-in AI render one without a model writing code.""" + +PRESETS = ( + dict(id='plane', name='纸飞机飞出教室窗外', shot='一架纸飞机从教室后排飞过一排排课桌，穿过敞开的窗户飞向窗外的操场，镜头跟在纸飞机后面。', + prompt='A realistic classroom scene: a white paper airplane glides from the back of the classroom over rows of wooden desks and blue chairs, passes through the open window in the far wall and flies out over a sunny sports field with a running track, while the camera follows just behind the plane. Natural daylight, realistic materials, smooth camera motion. Soft ambient classroom sounds fading into a light outdoor breeze.', + code="export function build(THREE) {\n const scene = new THREE.Scene();\n scene.background = new THREE.Color(0xb7d7f0);\n const camera = new THREE.PerspectiveCamera(65, 960 / 544, 0.1, 200);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1, side: THREE.DoubleSide });\n const add = (geo, mat, x, y, z, name) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || '';\n m.castShadow = true; m.receiveShadow = true; scene.add(m); return m;\n };\n scene.add(new THREE.HemisphereLight(0xffffff, 0x8a7f70, 1.2));\n const sun = new THREE.DirectionalLight(0xfff4e0, 2.0);\n sun.position.set(12, 10, -4); sun.castShadow = true;\n sun.shadow.camera.left = -15; sun.shadow.camera.right = 15; sun.shadow.camera.top = 15; sun.shadow.camera.bottom = -15;\n sun.shadow.mapSize.set(2048, 2048); scene.add(sun);\n // room x -4..4, z 0..14 (window wall at z = 0), height 3\n add(new THREE.PlaneGeometry(8, 14).rotateX(-Math.PI / 2), std(0xb08a5e), 0, 0, 7, 'floor');\n add(new THREE.PlaneGeometry(8, 14).rotateX(Math.PI / 2), std(0xffffff), 0, 3, 7, 'ceiling');\n add(new THREE.BoxGeometry(0.2, 3, 14), std(0xe6dfc8), -4, 1.5, 7, 'wall_left');\n add(new THREE.BoxGeometry(0.2, 3, 14), std(0xe6dfc8), 4, 1.5, 7, 'wall_right');\n add(new THREE.BoxGeometry(8, 3, 0.2), std(0xe6dfc8), 0, 1.5, 14, 'wall_back');\n // window wall with a 2.4 x 1.4 opening, sill at 1.0\n add(new THREE.BoxGeometry(8, 1.0, 0.2), std(0xe6dfc8), 0, 0.5, 0, 'wall_below');\n add(new THREE.BoxGeometry(8, 0.6, 0.2), std(0xe6dfc8), 0, 2.7, 0, 'wall_above');\n add(new THREE.BoxGeometry(2.8, 1.4, 0.2), std(0xe6dfc8), -2.6, 1.7, 0, 'wall_l');\n add(new THREE.BoxGeometry(2.8, 1.4, 0.2), std(0xe6dfc8), 2.6, 1.7, 0, 'wall_r');\n add(new THREE.BoxGeometry(2.4, 0.05, 0.3), std(0xffffff), 0, 1.0, 0, 'sill');\n add(new THREE.BoxGeometry(5.5, 1.2, 0.05), std(0x2f5b43), -1.5, 1.6, 13.85, 'blackboard');\n // desks: 3 columns x 4 rows\n for (let c = 0; c < 3; c++) for (let r = 0; r < 4; r++) {\n const x = -2.4 + c * 2.4, z = 4 + r * 2.2;\n add(new THREE.BoxGeometry(1.1, 0.06, 0.6), std(0xc98f4f), x, 0.75, z, 'desk_top');\n add(new THREE.BoxGeometry(0.06, 0.75, 0.06), std(0x777777), x - 0.5, 0.375, z - 0.25, 'desk_leg');\n add(new THREE.BoxGeometry(0.06, 0.75, 0.06), std(0x777777), x + 0.5, 0.375, z + 0.25, 'desk_leg');\n add(new THREE.BoxGeometry(0.45, 0.05, 0.45), std(0x4d7cc9), x, 0.45, z + 0.7, 'chair_seat');\n add(new THREE.BoxGeometry(0.45, 0.45, 0.05), std(0x4d7cc9), x, 0.7, z + 0.92, 'chair_back');\n }\n // outside: grass, running track, goal\n add(new THREE.PlaneGeometry(120, 120).rotateX(-Math.PI / 2), std(0x6fae56), 0, -0.02, -50, 'field');\n add(new THREE.PlaneGeometry(30, 4).rotateX(-Math.PI / 2), std(0xc4654a), 0, 0.0, -12, 'track');\n add(new THREE.BoxGeometry(6, 3, 0.2), std(0xffffff), 0, 1.5, -36, 'goal');\n const plane = new THREE.Group(); plane.name = 'hero_paper_plane'; scene.add(plane);\n const wing = new THREE.Mesh(new THREE.ConeGeometry(0.16, 0.5, 3).rotateX(-Math.PI / 2).scale(1.6, 0.35, 1), std(0xffffff));\n wing.castShadow = true; plane.add(wing);\n const ss = x => x * x * (3 - 2 * x);\n const pts = [[0.4, 1.7, 11.5], [0.2, 1.55, 8.5], [-0.3, 1.5, 6.0], [0.0, 1.55, 2.5], [0.0, 1.6, 0.0], [0.3, 1.5, -4], [0.8, 1.2, -10], [1.4, 0.9, -16]]\n .map(p => new THREE.Vector3(...p));\n const curve = new THREE.CatmullRomCurve3(pts, false, 'centripetal');\n const at = t => curve.getPoint(ss(Math.min(1, t / 5)) * 0.6 + Math.min(1, t / 5) * 0.4);\n return {\n scene, camera,\n update(t) {\n const p = at(t), q = at(t + 0.05);\n plane.position.copy(p); plane.lookAt(q); plane.rotateZ(0.25 * Math.sin(t * 2));\n const behind = at(Math.max(0, t - 0.45));\n camera.position.set(behind.x * 0.5, behind.y + 0.4, behind.z + 1.5);\n camera.lookAt(p.x, p.y, p.z - 2);\n },\n };\n}\n"), + dict(id='coaster', name='第一人称过山车俯冲', shot='第一人称过山车：先爬到最高点停顿，然后俯冲而下，穿过一个大回环，最后冲向远处的游乐园。', + prompt='A first-person roller coaster ride at a real amusement park: the cart climbs a steep red steel track to the very top, pauses for a heartbeat, then plunges down in a steep drop, whips through a large vertical loop, and races out toward the colourful rides and a Ferris wheel in the distance. Bright sunny day, green trees, realistic steel and paint, strong sense of speed, smooth camera. Rushing wind and the clatter of the wheels on the track.', + code='export function build(THREE) {\n const scene = new THREE.Scene();\n const camera = new THREE.PerspectiveCamera(65, 960 / 544, 0.1, 300);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1 });\n const add = (geo, mat, x, y, z, name, parent) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || "";\n m.castShadow = true; m.receiveShadow = true; (parent || scene).add(m); return m;\n };\n const ss = x => x * x * (3 - 2 * x);\n const lerp = (a, b, u) => a + (b - a) * u;\n const clamp01 = x => Math.min(1, Math.max(0, x));\n const sun = new THREE.DirectionalLight(0xffffff, 1.8);\n sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048);\n sun.shadow.camera.left = -60; sun.shadow.camera.right = 60; sun.shadow.camera.top = 60; sun.shadow.camera.bottom = -60; sun.shadow.camera.far = 200;\n scene.add(sun);\n\n scene.background = new THREE.Color(0xbfdcf0);\n scene.add(new THREE.HemisphereLight(0xdfeeff, 0x6a7a5a, 1.1)); sun.position.set(-40, 80, 30);\n add(new THREE.PlaneGeometry(600, 600).rotateX(-Math.PI / 2), std(0x7fa86a), 0, 0, 0, "ground");\n // track as a curve: climb, crest, drop, loop, run-out\n const pts = [[0,2,60],[0,12,40],[0,30,20],[0,44,6],[0,46,-4],[0,40,-12],[0,18,-24],[0,4,-34],[0,3,-46],[0,10,-56],[0,18,-50],[0,14,-40],[0,3,-48],[0,2,-70],[0,2,-110]];\n const curve = new THREE.CatmullRomCurve3(pts.map(p => new THREE.Vector3(...p)));\n const N = 120;\n for (let i = 0; i < N; i++) {\n const p = curve.getPoint(i / N), q = curve.getPoint((i + 1) / N);\n const seg = add(new THREE.BoxGeometry(2.6, 0.25, 1.1 * p.distanceTo(q) + 0.2), std(0xd23b2e), p.x, p.y, p.z, "track");\n seg.lookAt(q);\n if (i % 6 === 0 && p.y > 2.5) add(new THREE.BoxGeometry(0.35, p.y, 0.35), std(0xe8e8e8), p.x, p.y / 2 - 0.2, p.z, "pillar");\n }\n for (let i = 0; i < 60; i++) { const a = i * 2.399, r = 20 + (i * 13) % 70; add(new THREE.ConeGeometry(2.2, 7, 6), std(0x3f7a3a), Math.cos(a) * r, 3.5, -40 + Math.sin(a) * r * 1.6, "tree"); }\n // distant park\n const cols = [0xf2c14e, 0x4e9bd6, 0xd65e8b, 0x8bd64e];\n for (let i = 0; i < 14; i++) add(new THREE.BoxGeometry(5 + i % 3, 3 + (i * 5) % 6, 5), std(cols[i % 4]), -30 + i * 5, 2, -150 - (i % 3) * 8, "park");\n const wheel = add(new THREE.TorusGeometry(12, 0.5, 6, 20), std(0xf0f0f0), 30, 14, -140, "ferris");\n add(new THREE.ConeGeometry(2.5, 5, 8), std(0xe8412c), -20, 6, -125, "tent");\n const hero = new THREE.Group(); hero.name = "hero_cart"; scene.add(hero);\n add(new THREE.BoxGeometry(1.6, 0.6, 2.4), std(0xf2c14e), 0, 0, 0, "hero_cart_body", hero);\n const ease = u => u < 0.28 ? 0.55 * (u / 0.28) * 0.5 * 0 + ss(u / 0.28) * 0.46 : u < 0.36 ? 0.46 + (u - 0.28) / 0.08 * 0.02 : 0.48 + ss((u - 0.36) / 0.64) * 0.52;\n return { scene, camera, update(t) {\n const u = ease(clamp01(t / 5));\n wheel.rotation.z = t * 0.2;\n const p = curve.getPoint(u * 0.98), look = curve.getPoint(Math.min(1, u * 0.98 + 0.03));\n hero.position.copy(p); hero.lookAt(look);\n const dir = look.clone().sub(p).normalize(); dir.y *= 0.55; camera.position.set(p.x, p.y + 1.1, p.z); camera.lookAt(p.x + dir.x * 10, p.y + 1.1 + dir.y * 10, p.z + dir.z * 10);\n camera.rotation.z = 0;\n } };\n}\n'), + dict(id='drone', name='无人机穿过楼间再拉升', shot='无人机贴着两排高楼之间的街道低空飞行，随后拉升冲出楼顶，镜头跟在无人机后上方。', + prompt="A cinematic drone shot in a real city: a small red quadcopter drone flies low along a narrow street between two rows of tall concrete and glass buildings, then climbs steeply and bursts out above the rooftops into the open blue sky, while the camera follows behind and above the drone. Natural sunlight, soft shadows, realistic materials, smooth camera motion. Ambient city wind sound and the soft hum of the drone's rotors.", + code="export function build(THREE) {\n const scene = new THREE.Scene();\n scene.background = new THREE.Color(0xbcd6ea);\n const camera = new THREE.PerspectiveCamera(60, 960 / 544, 0.1, 200);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1 });\n const add = (geo, mat, x, y, z, name) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || '';\n m.castShadow = true; m.receiveShadow = true; scene.add(m); return m;\n };\n scene.add(new THREE.HemisphereLight(0xdfeeff, 0x666a70, 1.1));\n const sun = new THREE.DirectionalLight(0xffffff, 1.8);\n sun.position.set(-30, 50, 20); sun.castShadow = true;\n sun.shadow.camera.left = -60; sun.shadow.camera.right = 60; sun.shadow.camera.top = 60; sun.shadow.camera.bottom = -60;\n sun.shadow.camera.far = 150; sun.shadow.mapSize.set(2048, 2048);\n scene.add(sun);\n add(new THREE.PlaneGeometry(200, 200).rotateX(-Math.PI / 2), std(0x5d6168), 0, 0, 0, 'ground');\n add(new THREE.PlaneGeometry(9, 200).rotateX(-Math.PI / 2), std(0x3b3e44), 0, 0.02, 0, 'street');\n const palette = [0xc9b8a0, 0x9fb1c4, 0xd6cfc4, 0xa8a39a, 0xb9c4b0];\n for (let side of [-1, 1]) {\n for (let i = 0; i < 9; i++) {\n const z = -80 + i * 17 + (side > 0 ? 6 : 0);\n const h = 14 + ((i * 7 + (side > 0 ? 3 : 0)) % 5) * 5;\n add(new THREE.BoxGeometry(14, h, 14), std(palette[(i + (side > 0 ? 2 : 0)) % 5]), side * 12.5, h / 2, z, 'building');\n }\n }\n const hero = new THREE.Group(); hero.name = 'hero_drone'; scene.add(hero);\n const body = new THREE.Mesh(new THREE.BoxGeometry(0.9, 0.25, 0.9), std(0xe8412c)); body.castShadow = true; hero.add(body);\n for (const [x, z] of [[-0.7, -0.7], [0.7, -0.7], [-0.7, 0.7], [0.7, 0.7]]) {\n const rotor = new THREE.Mesh(new THREE.CylinderGeometry(0.4, 0.4, 0.03, 12), std(0x222222));\n rotor.position.set(x, 0.17, z); hero.add(rotor);\n }\n const ss = x => x * x * (3 - 2 * x);\n const lerp = (a, b, u) => a + (b - a) * u;\n const heroAt = t => {\n const u = Math.min(1, t / 5);\n const z = lerp(60, -60, u);\n const rise = ss(Math.min(1, Math.max(0, (u - 0.55) / 0.35)));\n return new THREE.Vector3(1.5 * Math.sin(u * 6), lerp(3.2, 42, rise), z);\n };\n return {\n scene, camera,\n update(t) {\n const p = heroAt(t);\n hero.position.copy(p);\n const ahead = heroAt(t + 0.1);\n hero.rotation.set(0.25 * Math.min(1, (ahead.z - p.z) / -0.1 * 0.02), 0, 0);\n hero.rotation.x = -0.18; hero.rotation.z = 0.12 * Math.sin(t * 1.3);\n const u = Math.min(1, t / 5);\n const lift = ss(Math.min(1, Math.max(0, (u - 0.55) / 0.35)));\n camera.position.set(p.x * 0.4, p.y + 2.2 + 3 * lift, p.z + 9 + 2 * lift);\n camera.lookAt(p.x, p.y + 0.3, p.z - 4);\n },\n };\n}\n"), + dict(id='cat', name='橘猫跳上书架', shot='一只橘猫在客厅地板上蹲下蓄力，沿着错落的书架层板一级一级跳上去，最后蹲在最高一层，镜头侧面平移并略微上摇跟随。', + prompt='A ginger cat crouches on a living room floor, gathers itself and leaps up a staggered wooden bookshelf, jumping from one shelf to the next between rows of books, and finally settles on the top shelf, while the camera pans sideways and tilts up slightly to follow. Warm indoor daylight, realistic fur and wood, soft shadows, smooth camera motion. Soft room tone and the light thud of paws on wood.', + code='export function build(THREE) {\n const scene = new THREE.Scene();\n const camera = new THREE.PerspectiveCamera(65, 960 / 544, 0.1, 300);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1 });\n const add = (geo, mat, x, y, z, name, parent) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || "";\n m.castShadow = true; m.receiveShadow = true; (parent || scene).add(m); return m;\n };\n const ss = x => x * x * (3 - 2 * x);\n const lerp = (a, b, u) => a + (b - a) * u;\n const clamp01 = x => Math.min(1, Math.max(0, x));\n const sun = new THREE.DirectionalLight(0xffffff, 1.8);\n sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048);\n sun.shadow.camera.left = -60; sun.shadow.camera.right = 60; sun.shadow.camera.top = 60; sun.shadow.camera.bottom = -60; sun.shadow.camera.far = 200;\n scene.add(sun);\n\n scene.background = new THREE.Color(0xe9e2d6);\n scene.add(new THREE.HemisphereLight(0xfff4e6, 0x8a7f72, 1.2)); sun.position.set(6, 10, 8);\n add(new THREE.PlaneGeometry(14, 10).rotateX(-Math.PI / 2), std(0xb88f65), 0, 0, 0, "floor");\n add(new THREE.BoxGeometry(14, 4, 0.2), std(0xe9e2d6), 0, 2, -1.2, "wall");\n // bookshelf: two side boards, staggered shelves\n add(new THREE.BoxGeometry(0.1, 2.6, 0.5), std(0x7a5232), -1.4, 1.3, -0.9, "side");\n add(new THREE.BoxGeometry(0.1, 2.6, 0.5), std(0x7a5232), 1.4, 1.3, -0.9, "side");\n const shelves = [[0.6, -0.9], [1.2, -0.4], [1.8, 0.2], [2.4, 0.6]];\n const stepsX = [-0.9, -0.2, 0.5, 1.0], ys = [0.55, 1.05, 1.55, 2.1];\n for (let i = 0; i < 4; i++) {\n add(new THREE.BoxGeometry(1.0, 0.06, 0.5), std(0x9a6b3f), stepsX[i], ys[i] - 0.03, -0.9, "shelf");\n for (let k = 0; k < 3; k++) add(new THREE.BoxGeometry(0.12, 0.28, 0.3), std([0x3c6ea8, 0xb04a4a, 0x4a8a5a][(i + k) % 3]), stepsX[i] - 0.3 + k * 0.2, ys[i] + 0.14, -1.0, "book");\n }\n add(new THREE.BoxGeometry(0.9, 0.5, 0.5), std(0x6b7b8c), -2.6, 0.25, 0.6, "sofa_block");\n const cat = new THREE.Group(); cat.name = "hero_cat"; scene.add(cat);\n const orange = std(0xe58a2e);\n const body = add(new THREE.BoxGeometry(0.42, 0.22, 0.2), orange, 0, 0.18, 0, "hero_cat_body", cat);\n add(new THREE.SphereGeometry(0.11, 10, 8), orange, 0.26, 0.28, 0, "hero_cat_head", cat);\n add(new THREE.ConeGeometry(0.035, 0.08, 4), orange, 0.28, 0.4, 0.06, "hero_cat_ear", cat);\n add(new THREE.ConeGeometry(0.035, 0.08, 4), orange, 0.28, 0.4, -0.06, "hero_cat_ear", cat);\n const tail = add(new THREE.CylinderGeometry(0.03, 0.03, 0.4, 6), orange, -0.34, 0.3, 0, "hero_cat_tail", cat);\n tail.rotation.z = 0.8;\n // waypoints: floor crouch, jump to each shelf\n const wp = [[-1.9, 0.0, 0.1], [-1.9, 0.0, 0.1], [stepsX[0], ys[0], -0.9], [stepsX[1], ys[1], -0.9], [stepsX[2], ys[2], -0.9], [stepsX[3], ys[3], -0.9], [stepsX[3], ys[3], -0.9]];\n const times = [0, 0.7, 1.4, 2.3, 3.2, 4.1, 5.0];\n const pos = t => { let i = 0; while (i < 5 && t > times[i + 1]) i++;\n const u = clamp01((t - times[i]) / (times[i + 1] - times[i])); const a = wp[i], b = wp[i + 1];\n const arc = (i >= 1 && i <= 4) ? Math.sin(Math.PI * u) * 0.45 : 0;\n const e = ss(u); return new THREE.Vector3(lerp(a[0], b[0], e), lerp(a[1], b[1], u) + arc, lerp(a[2], b[2], e)); };\n return { scene, camera, update(t) {\n const p = pos(t); cat.position.copy(p);\n cat.scale.y = (t < 0.7) ? 1 - 0.35 * Math.sin(Math.PI * clamp01(t / 0.7)) : 1;\n cat.rotation.z = (t > 0.7 && t < 4.1) ? 0.35 : 0;\n camera.position.set(p.x * 0.6, 0.9 + p.y * 0.5, 2.7);\n camera.lookAt(p.x * 0.85, 0.35 + p.y * 0.85, -0.9);\n } };\n}\n'), + dict(id='car', name='赛车入弯漂移环绕', shot='一辆红色赛车在直道尽头高速入弯，车尾漂移甩出，镜头围着赛车做半圈环绕，最后跟着车冲出弯道。', + prompt='A red race car hits the end of a straight at full speed and throws itself into a corner, the rear sliding out in a controlled drift with a little tyre smoke, while the camera orbits half a circle around the car and then follows it out of the bend. Sunny circuit with red and white kerbs, grandstands and green grass, realistic paint and tarmac, strong sense of speed, smooth camera. Roaring engine and the squeal of tyres.', + code='export function build(THREE) {\n const scene = new THREE.Scene();\n const camera = new THREE.PerspectiveCamera(65, 960 / 544, 0.1, 300);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1 });\n const add = (geo, mat, x, y, z, name, parent) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || "";\n m.castShadow = true; m.receiveShadow = true; (parent || scene).add(m); return m;\n };\n const ss = x => x * x * (3 - 2 * x);\n const lerp = (a, b, u) => a + (b - a) * u;\n const clamp01 = x => Math.min(1, Math.max(0, x));\n const sun = new THREE.DirectionalLight(0xffffff, 1.8);\n sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048);\n sun.shadow.camera.left = -60; sun.shadow.camera.right = 60; sun.shadow.camera.top = 60; sun.shadow.camera.bottom = -60; sun.shadow.camera.far = 200;\n scene.add(sun);\n\n scene.background = new THREE.Color(0xbdd8ee);\n scene.add(new THREE.HemisphereLight(0xdfeeff, 0x6a7a5a, 1.1)); sun.position.set(-30, 60, 25);\n add(new THREE.PlaneGeometry(400, 400).rotateX(-Math.PI / 2), std(0x6f9a58), 0, 0, 0, "grass");\n // road: straight along -z then curving right (to +x)\n const R = 22, cz = -30;\n add(new THREE.BoxGeometry(10, 0.05, 80), std(0x3d4046), 0, 0.03, 10, "road");\n const arc = []; for (let i = 0; i <= 24; i++) { const a = Math.PI * 0.5 * i / 24; arc.push(i); \n add(new THREE.BoxGeometry(10, 0.05, 2 * R * Math.PI / 2 / 24 + 0.6), std(0x3d4046), R - R * Math.cos(a), 0.03, cz - R * Math.sin(a), "bend").rotation.y = -a; }\n add(new THREE.BoxGeometry(10, 0.05, 80), std(0x3d4046), R + 40, 0.03, cz - R, "road2").rotation.y = Math.PI / 2;\n for (let i = 0; i < 16; i++) { const a = Math.PI * 0.5 * i / 15; add(new THREE.BoxGeometry(0.6, 0.6, 0.6), std(i % 2 ? 0xffffff : 0xe8412c), (R + 6) - (R + 6) * Math.cos(a) , 0.3, cz - (R + 6) * Math.sin(a), "kerb"); }\n for (let i = 0; i < 8; i++) add(new THREE.BoxGeometry(3, 5, 3), std(0xc9b8a0), -25 + (i % 2) * 4, 2.5, -60 + i * 6, "stand");\n const hero = new THREE.Group(); hero.name = "hero_car"; scene.add(hero);\n add(new THREE.BoxGeometry(1.8, 0.55, 4.2), std(0xd9261c), 0, 0.5, 0, "hero_car_body", hero);\n add(new THREE.BoxGeometry(1.3, 0.45, 1.6), std(0x20242a), 0, 0.95, -0.2, "hero_car_cabin", hero);\n for (const [x, z] of [[-1, -1.4], [1, -1.4], [-1, 1.4], [1, 1.4]]) { const w = add(new THREE.CylinderGeometry(0.4, 0.4, 0.3, 12), std(0x111111), x, 0.4, z, "hero_car_wheel", hero); w.rotation.z = Math.PI / 2; }\n const path = t => { const u = clamp01(t / 5); const s = u * (80 + R * Math.PI / 2 + 30) - 40; // arclength from z=40\n if (s < 40 - 0) return { x: 0, z: 40 - s, h: Math.PI }; // heading -z (rotation.y = PI points model -z? model faces -z when rotation 0)\n return null; };\n const pos = t => { const u = clamp01(t / 5), L1 = 70, L2 = R * Math.PI / 2, L3 = 40; const s = ss(u * 0.2 + u * 0.8) * (L1 + L2 + L3);\n if (s < L1) return { x: 0, z: 40 - s, yaw: 0 };\n if (s < L1 + L2) { const a = (s - L1) / R; return { x: R - R * Math.cos(a), z: cz - R * Math.sin(a) + 0, yaw: -a }; }\n const d = s - L1 - L2; return { x: R + d, z: cz - R, yaw: -Math.PI / 2 }; };\n return { scene, camera, update(t) {\n const p = pos(t); const drift = Math.sin(Math.PI * clamp01((t - 1.2) / 2.4)) * 0.55;\n hero.position.set(p.x, 0, p.z); hero.rotation.y = p.yaw - drift;\n const orbit = Math.PI * 0.9 * ss(clamp01((t - 1.0) / 3.0));\n const ang = Math.PI * 0.85 - orbit + p.yaw * -0.0;\n const rad = 9 - 2 * Math.sin(Math.PI * clamp01(t / 5));\n camera.position.set(p.x + Math.sin(ang + (p.yaw)) * rad * -1 * -1, 2.2 + 1.5 * Math.sin(Math.PI * clamp01(t / 5)), p.z + Math.cos(ang + p.yaw) * rad);\n camera.lookAt(p.x, 0.8, p.z);\n } };\n}\n'), + dict(id='door', name='推门长镜头', shot='短剧长镜头：镜头从走廊里缓缓推向一扇半掩的木门，推开门进入昏黄的客厅，一直推到沙发后的落地窗前停住。', + prompt='A short-drama long take: the camera glides down a dim hallway toward a half-open wooden door, pushes the door open and enters a warmly lit living room, and keeps moving forward past a sofa until it stops in front of the tall window at the far end. Warm tungsten light, soft evening glow from the window, realistic wood, fabric and plaster, smooth steady camera. Quiet room tone and a faint creak of the door.', + code='export function build(THREE) {\n const scene = new THREE.Scene();\n const camera = new THREE.PerspectiveCamera(65, 960 / 544, 0.1, 300);\n const std = c => new THREE.MeshStandardMaterial({ color: c, roughness: 1 });\n const add = (geo, mat, x, y, z, name, parent) => {\n const m = new THREE.Mesh(geo, mat); m.position.set(x, y, z); m.name = name || "";\n m.castShadow = true; m.receiveShadow = true; (parent || scene).add(m); return m;\n };\n const ss = x => x * x * (3 - 2 * x);\n const lerp = (a, b, u) => a + (b - a) * u;\n const clamp01 = x => Math.min(1, Math.max(0, x));\n const sun = new THREE.DirectionalLight(0xffffff, 1.8);\n sun.castShadow = true; sun.shadow.mapSize.set(2048, 2048);\n sun.shadow.camera.left = -60; sun.shadow.camera.right = 60; sun.shadow.camera.top = 60; sun.shadow.camera.bottom = -60; sun.shadow.camera.far = 200;\n scene.add(sun);\n\n scene.background = new THREE.Color(0x2b2622);\n scene.add(new THREE.HemisphereLight(0xffe9c8, 0x6a5a48, 1.8)); sun.position.set(-6, 8, -4); sun.intensity = 1.6;\n const wall = std(0xcfc4b0), floor = std(0x8a6a4a);\n // corridor z 6 .. 0, living room z 0 .. -9\n add(new THREE.PlaneGeometry(4, 6).rotateX(-Math.PI / 2), floor, 0, 0, 3, "corr_floor");\n add(new THREE.PlaneGeometry(7, 9).rotateX(-Math.PI / 2), floor, 0, 0, -4.5, "room_floor");\n add(new THREE.PlaneGeometry(7, 9).rotateX(Math.PI / 2), std(0xe4dccc), 0, 2.8, -4.5, "room_ceiling");\n add(new THREE.PlaneGeometry(4, 6).rotateX(Math.PI / 2), std(0xe4dccc), 0, 2.8, 3, "corr_ceiling");\n for (const x of [-2, 2]) add(new THREE.BoxGeometry(0.15, 2.8, 6), wall, x, 1.4, 3, "corr_wall");\n for (const x of [-3.5, 3.5]) add(new THREE.BoxGeometry(0.15, 2.8, 9), wall, x, 1.4, -4.5, "room_wall");\n add(new THREE.BoxGeometry(1.0, 0.7, 0.15), wall, 0, 2.45, 0, "door_lintel");\n add(new THREE.BoxGeometry(1.5, 2.8, 0.15), wall, -1.25, 1.4, 0, "door_wall_l");\n add(new THREE.BoxGeometry(1.5, 2.8, 0.15), wall, 1.25, 1.4, 0, "door_wall_r");\n // wall with window at far end\n add(new THREE.BoxGeometry(3.4, 2.8, 0.15), wall, -1.8, 1.4, -9, "far_l"); add(new THREE.BoxGeometry(3.4, 2.8, 0.15), wall, 1.8, 1.4, -9, "far_r");\n add(new THREE.BoxGeometry(0.2, 2.8, 0.15), wall, 0, 1.4, -9, "far_mid");\n add(new THREE.PlaneGeometry(0.1, 0.1), std(0xbfd8f0), 0, 1.4, -9.1, "sky_glow").scale.set(70, 28, 1);\n const hinge = new THREE.Group(); hinge.position.set(-0.5, 0, 0); scene.add(hinge);\n add(new THREE.BoxGeometry(1.0, 2.1, 0.06), std(0x6b4a2e), 0.5, 1.05, 0, "door", hinge);\n hinge.rotation.y = 0.3;\n add(new THREE.BoxGeometry(2.2, 0.8, 0.9), std(0x6b7f94), 0, 0.4, -6.2, "sofa");\n add(new THREE.BoxGeometry(2.2, 0.5, 0.2), std(0x6b7f94), 0, 0.95, -6.6, "sofa_back");\n add(new THREE.BoxGeometry(1.2, 0.35, 0.7), std(0x5a3a22), 0, 0.18, -4.6, "table");\n add(new THREE.CylinderGeometry(0.25, 0.25, 1.6, 10), std(0x3d6b45), 2.6, 0.8, -7.5, "plant");\n add(new THREE.BoxGeometry(0.9, 1.0, 0.1), std(0xb05a3a), -3.4, 1.6, -3, "picture");\n add(new THREE.BoxGeometry(0.5, 1.7, 0.4), std(0x2b2b30), -3.0, 0.85, -1.5, "hero_figure");\n return { scene, camera, update(t) {\n const u = clamp01(t / 5);\n hinge.rotation.y = 0.3 + 1.1 * ss(clamp01((t - 1.0) / 1.2));\n const z = lerp(5.5, -4.6, ss(u * 0.85 + 0.15 * u));\n camera.position.set(0.15 * Math.sin(u * 3), 1.55 - 0.1 * ss(clamp01((u - 0.6) / 0.4)), z);\n camera.lookAt(0.1 * Math.sin(u * 3 + 0.5), 1.4, z - 4);\n } };\n}\n'), +) + + +def preset(identity): + return next((row for row in PRESETS if row['id'] == identity), None) diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index ff304d4d..ecbf3115 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -75,7 +75,7 @@ def resolve_assets(serialized, input_directory): root = Path(input_directory).resolve() result = {} for row in rows: - if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast'}: + if not isinstance(row, dict) or set(row) - {'file', 'role', 'enabled', 'cast', 'early_steps'}: raise ValueError('Invalid media item') # The character an @ mention added the row for; a continued part keeps one picture per character. if 'cast' in row and (set(row['cast'] if isinstance(row['cast'], dict) else ('?',)) != {'id'} @@ -108,6 +108,10 @@ def resolve_assets(serialized, input_directory): reference = {'kind': kind, 'path': str(path)} if cast_id(row) is not None: reference['cast'] = cast_id(row) # only a continued part reads it (comfy_media.continued) + if 'early_steps' in row: + from .media_request import validate_early_steps + validate_early_steps(dict(row, kind=kind)) + reference['early_steps'] = row['early_steps'] result.setdefault('references', []).append(reference) else: raise ValueError('Media role must be first, last or reference') From 29ab41ac2bcd2949e2eed187d127f44fea9e2e17 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 06/21] Queue editor continuations on the server --- freevideo_engine/chain_runner.py | 77 +++++++++++++++++++++++++++++--- freevideo_engine/chain_store.py | 21 +++++++-- web/editor.js | 34 +++++++++----- web/editor_model.js | 11 +++-- 4 files changed, 119 insertions(+), 24 deletions(-) diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index 64e809df..2194ded2 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -118,12 +118,61 @@ def _sources(self, record, block): if isinstance(value, str): source = next(row for row in record['blocks'] if row['id'] == value) sources[key] = dict(block=value, output=source['output']) + elif isinstance(value, dict) and 'chain' in value: + source = self._referenced_block(value) + sources[key] = dict(work=source['output'], output=source['output']) elif value is not None: sources[key] = dict(work=value['work'], output=value['work']) else: sources[key] = None return sources + def _referenced_block(self, value): + source = self.records.get(value['chain']) + if source is None: + # A recent dependent may outlive the list's bounded startup window. + try: + self.load(value['chain']) + except KeyError: + raise ValueError('Referenced chain does not exist') from None + source = self.records[value['chain']] + block = next((row for row in source['blocks'] if row['id'] == value['block']), None) + if block is None: + raise ValueError('Referenced chain block does not exist') + return block + + def _dependency_state(self, record): + waiting = False + for block in record['blocks']: + for key in ('after', 'before'): + value = block[key] + if not isinstance(value, dict) or 'chain' not in value: + continue + try: + source = self._referenced_block(value) + except ValueError: + return 'interrupted' + parent = self.records[value['chain']] + if parent['state'] in ('cancelling', 'cancelled', 'failed', 'interrupted'): + return 'interrupted' + if parent['state'] == 'done': + if source['state'] != 'done' or not source['output']: + return 'interrupted' + else: + waiting = True + return 'queued' if waiting else 'ready' + + def _interrupt_dependants(self): + # Repeat so a stopped ancestor reaches descendants in any receipt order. + changed = True + while changed: + changed = False + for waiting in list(self.records.values()): + if waiting['state'] == 'queued' and self._dependency_state(waiting) == 'interrupted': + waiting['state'] = 'interrupted' + self._save(waiting) + changed = True + def _elapsed(self, block): start = self.starts.get(block['prompt_id']) return max(0., self.clock() - start) if start is not None else 0. @@ -161,13 +210,21 @@ def _depths(self, record, seen=()): rest.append(identity) continue depth[identity] = 1 + sum(depth[value] if isinstance(value, str) - else self._work_parts(value['work'], seen + (record['id'],)) + else self._source_parts(value, seen + (record['id'],)) for value in sources) if len(rest) == len(pending): raise ValueError('Block references cannot contain a cycle or self reference') pending = rest return depth + def _source_parts(self, value, seen): + if 'chain' not in value: + return self._work_parts(value['work'], seen) + self._referenced_block(value) + if value['chain'] in seen: + raise ValueError('Chain references cannot contain a cycle') + return self._depths(self.records[value['chain']], seen)[value['block']] + def _work_parts(self, work, seen): key = _work_key(work) for record in list(self.records.values()): @@ -209,6 +266,7 @@ def cancel(self, chain_id): if record['state'] in store.ACTIVE and record['state'] != 'cancelling': record['state'] = 'cancelling' self._save(record) + self._interrupt_dependants() self.wake.set() return self._view(record) @@ -469,6 +527,8 @@ def _complete(self, record, block, status): self.success.pop(prompt_id, None) self.cancel_sent.discard(prompt_id) self._save(record) + if record['state'] in ('cancelled', 'failed', 'interrupted'): + self._interrupt_dependants() def step(self): """Run one scheduler pass; tests can drive the same loop without sleeping.""" @@ -490,11 +550,16 @@ def step(self): self._save(record) if self.active is not None and self.records[self.active]['state'] not in store.ACTIVE: self.active = None - while self.active is None and self.queue: - identity = self.queue.pop(0) - waiting = self.records[identity] - if waiting['state'] == 'queued' or waiting['state'] == 'cancelling' and waiting['output_state'] == 'joining': - self.active = identity + self._interrupt_dependants() + for queued_id in list(self.queue): + waiting = self.records[queued_id] + if waiting['state'] not in store.ACTIVE: + self.queue.remove(queued_id) + elif self.active is None and (waiting['state'] == 'queued' + and self._dependency_state(waiting) == 'ready' + or waiting['state'] == 'cancelling' and waiting['output_state'] == 'joining'): + self.queue.remove(queued_id) + self.active = queued_id identity = self.active for recovering_id, snapshot in recovering: if not self.closed.is_set(): diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py index 468e49e5..2c2c5d47 100644 --- a/freevideo_engine/chain_store.py +++ b/freevideo_engine/chain_store.py @@ -30,7 +30,7 @@ MAX_PARTS = 5 -def reference(value): +def reference(value, queue=False): if value is None or isinstance(value, str) and value: return value if isinstance(value, dict) and set(value) == {'work'}: @@ -41,6 +41,12 @@ def reference(value): and '..' not in posix.parts and '..' not in windows.parts and posix.suffix.lower() == '.mp4'): return deepcopy(value) + if queue and isinstance(value, dict) and set(value) == {'chain', 'block'}: + if (isinstance(value['chain'], str) and re.fullmatch(r'c-[0-9a-f]{12}', value['chain']) + and isinstance(value['block'], str) and value['block']): + return deepcopy(value) + if queue: + raise ValueError('References must be block IDs, relative MP4 work paths, or queued chain outputs') raise ValueError('References must be block IDs or relative MP4 work paths without ..') @@ -73,6 +79,9 @@ def validate(payload): raise ValueError('Invalid extra data or client ID') if ('blocks' in payload) == ('seconds' in payload): raise ValueError('Specify blocks or seconds, not both') + queue = payload.get('queue', False) + if type(queue) is not bool: + raise ValueError('Queue must be a bool') planned = split = None join = payload.get('join', 'seconds' in payload) if type(join) is not bool: @@ -118,7 +127,7 @@ def validate(payload): part = row.get('part', identity) if not isinstance(part, str): raise ValueError('Block part must be a string') - after, before = reference(row.get('after')), reference(row.get('before')) + after, before = reference(row.get('after'), queue), reference(row.get('before'), queue) text, frames = row.get('text'), row.get('frames') if not isinstance(text, str) or len(text) > 20000: raise ValueError('Block text must be a string of at most 20000 characters') @@ -164,6 +173,8 @@ def visit(identity): snapshot=dict(prompt=deepcopy(prompt), extra_data=deepcopy(extra)), blocks=blocks, plan=planned, join=join, output=None, output_state='pending' if join else 'none', output_error=None, output_blocks=0, split=split) + if queue: + record['queue'] = True try: json.dumps(record, allow_nan=False) except (ValueError, TypeError, OverflowError) as error: @@ -194,7 +205,9 @@ def summary(record): def detail(record): - return dict(summary(record), blocks=deepcopy(record['blocks']), plan=deepcopy(record['plan'])) + return dict(summary(record), blocks=deepcopy(record['blocks']), plan=deepcopy(record['plan']), + generation_size={key: record['snapshot']['prompt'][record['node']]['inputs'].get(key) + for key in ('width', 'height')}) def save(directory, record): @@ -277,7 +290,7 @@ def load_all(directory, limit=200, *, identity=None): continuation=None, skip_frames=0, skip_audio_samples=0) snapshot = record['snapshot'] normalized = validate(dict(node=record['node'], client_id=record['client_id'], **snapshot, - blocks=[{key: row[key] for key in ('id', 'part', 'after', 'before', 'fill', 'context')} + queue=record.get('queue', False), blocks=[{key: row[key] for key in ('id', 'part', 'after', 'before', 'fill', 'context')} | row['inputs'] for row in record['blocks']], join=record['join'])) if record['state'] not in CHAIN_STATES: diff --git a/web/editor.js b/web/editor.js index d0dda2c8..eab8237f 100644 --- a/web/editor.js +++ b/web/editor.js @@ -337,8 +337,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given } catch (value) { if (value instanceof model.EditError) return null; throw value; } } const jobActive = () => generationBusy || [...jobStatuses.values()].some(job => ['queued', 'running', 'cancelling'].includes(job.state)); - // Parts pressed while they cannot be sent yet: the engine is still starting, or they continue a part - // that is not finished. They are planned at once, shown queued, and sent in order when they can be. + // Only an engine that has not answered yet leaves parts held locally. + // Once ready, the server owns waiting parts and their dependencies. // Parts, and ranges not yet planned into parts, whose request is being sent. const held = [], sending = new Set(), sendingRanges = new Set(); const overlaps = (a, b) => a.start < b.start + b.frames && a.start + a.frames > b.start; @@ -352,10 +352,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given return state.record.tracks[0].clips.some(clip => (generating(clip) || isHeld(clip.id) || sending.has(clip.id)) && clip.start < range.start + range.frames && clip.start + clip.frames > range.start); } - function waitingFor(selection) { + function waitingFor() { if (!store.generationAvailable) return 'ready'; - const before = partBefore(selectionRange(selection).start); - return generationBusy || generating(before) || before && isHeld(before.id) ? 'before' : null; + return null; } // The nearest spot at or after the chosen one where a part of the same length fits. function emptySpot(selection) { @@ -424,9 +423,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given const entry = held[0], clips = entry.ids.map(id => model.find(state.record, id)).filter(clip => clip && clip.pending && !generating(clip)); if (!clips.length) { held.shift(); markHeld(entry.ids, false); rememberView(); continue; } const selection = {kind: 'clip', ids: clips.map(clip => clip.id)}, before = partBefore(Math.min(...clips.map(clip => clip.start))); - if (generating(before)) break; held.shift(); markHeld(entry.ids, false); rememberView(); - if (before && !before.source && before.pending) { genbar?.say(t('The part before did not finish, so this one did not start.', '上一段没有完成，这一段没有开始。')); continue; } + if (before && !before.source && before.pending && !before.job) { genbar?.say(t('The part before did not finish, so this one did not start.', '上一段没有完成，这一段没有开始。')); continue; } try { await actions.generate({text: entry.text, selection}); } catch (value) { const reason = error(value, 'generate'); if (reason && !['chain_too_long', 'chain_part_too_short'].includes(reason.reason)) genbar?.say(reason.say); } } @@ -771,8 +769,13 @@ export function openEditor({api, app, from = null, panel = {}, components: given // leaves whatever the user has moved on to, and what they typed there. submittedSelection = shown && state.selection === selection ? state.select({kind: 'clip', ids}) : state.selection; await completeSources(); const paths = Object.fromEntries([...store.sources].map(([key, value]) => [key, value.path])); - const request = model.generationRequest(prepared, ids, {text, paths}); - const follow = continuedSize(model.generationBefore(prepared, ids), store.sources), edit = capability.generationSize || generationSize(state.record); + const request = model.generationRequest(prepared, ids, {text, paths, queue: true}); + const beforeKeys = model.generationBefore(prepared, ids), sizes = new Map(store.sources); + for (const key of beforeKeys.filter(key => key?.startsWith('chain:'))) { + const job = await store.chain(key.split(':')[1]); + sizes.set(key, job?.generation_size); + } + const follow = continuedSize(beforeKeys, sizes), edit = capability.generationSize || generationSize(state.record); Object.assign(frozen.prompt[frozen.node].inputs, follow || {}); // Parts made at another size than the edit's have that size's limit. const limit = (follow && (follow.width !== edit.width || follow.height !== edit.height) @@ -790,7 +793,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const castRows = panel.references?.available() ? panel.references.rows?.() ?? [] : []; const blocks = request.blocks.map(({before, ...block}, i) => ({...block, seed: seed + i, text: expandCast(block.text, castRows, {continued: Boolean(block.after)}).text})); - const job = await store.startChain({...frozen, blocks}); + const job = await store.startChain({...frozen, blocks, queue: true}); if (!job && !store.generationAvailable) { if (epoch === projectEpoch) { const restoreSelection = submittedSelection !== priorSelection && state.selection === submittedSelection; @@ -823,6 +826,17 @@ export function openEditor({api, app, from = null, panel = {}, components: given async press(text) { if (disposed || closing || initializing) return {say: t('Opening the edit…', '正在打开…')}; let selection = state.selection, said = null; + if (actions.selectionHeld()) return {say: t('Queued', '排队中')}; + if (busySelection(selection)) return {say: t('This part is generating.', '这一段正在生成。')}; + // A preceding POST must supply its durable ID before a dependant can reference it. + const epoch = projectEpoch, awaiting = selectionRange(selection); + if (generationBusy) { + if (awaiting) sendingRanges.add(awaiting); + updateTransport(); + try { while (generationBusy) await generationDone; } + finally { sendingRanges.delete(awaiting); } + } + if (disposed || closing || epoch !== projectEpoch) return; if (!selectionRange(selection)) return {say: t('Drag on the timeline to choose a part first.', '先在时间线上拖出一段。')}; if (actions.selectionHeld()) return {say: t('Queued', '排队中')}; if (busySelection(selection)) return {say: t('This part is generating.', '这一段正在生成。')}; diff --git a/web/editor_model.js b/web/editor_model.js index 9bb4507b..1fa226c9 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -690,6 +690,8 @@ function remember(clip, source = null) { // A range drawn on its own starts a new video, even when it touches the part before it. const continuing = (clip, before) => !!clip && !!before && before.id !== clip.id && end(before) === clip.start && (clip.follows === before.id || (clip.part !== null && clip.part === before.part)); +const chainSource = clip => !!clip.job && !!clip.pending; +const continuationKey = clip => chainSource(clip) ? `chain:${clip.job}:${clip.id}` : clip.source ? key(clip.source) : null; const wholeEnd = clip => !!clip.source && clip.extend && clip.in + clip.frames === clip.length; const partBefore = (record, clip) => trackBy(record, 'v1').clips.find(before => continuing(clip, before)) ?? null; // Whether the part is drawn attached to the one before it on its track. @@ -943,7 +945,7 @@ function generationPlan(record, clips) { return videos.flatMap(video => videoPlan(record, video, replacing)); } function videoPlan(record, clips, replacing) { - const first = clips[0], before = trackBy(record, 'v1').clips.find(clip => continuing(first, clip) && wholeEnd(clip)); + const first = clips[0], before = trackBy(record, 'v1').clips.find(clip => continuing(first, clip) && (chainSource(clip) || wholeEnd(clip))); // Revealing a predecessor's hidden tail reduces the new visible range. const credit = clips.every(clip => clip.target_frames !== undefined && !clip.source) ? clips.reduce((sum, clip) => sum + clip.target_frames - clip.frames, 0) : 0; @@ -998,7 +1000,7 @@ export function prepare(record, ids) { export function generationBefore(record, ids) { const clips = generationClips(record, ids); return generationPlan(record, clips).filter(row => !row.before || !clips.some(clip => clip.id === row.before.id)) - .map(row => row.before ? key(row.before.source) : null); + .map(row => row.before ? continuationKey(row.before) : null); } // An imported video is continued from the copy the editor plays, which is upright and at 24 fps. function continuedPath(source, path) { @@ -1006,7 +1008,7 @@ function continuedPath(source, path) { check(path === `FreeVideo/.editor/imports/${source.id}/preview.mp4`, 'source'); return path; } -export function generationRequest(record, ids, {text, paths = {}} = {}) { +export function generationRequest(record, ids, {text, paths = {}, queue = false} = {}) { check(object(paths), 'source'); if (text !== undefined) check(validText(text), 'text'); const clips = generationClips(record, ids), rows = generationPlan(record, clips), plan = {}, blocks = []; @@ -1014,7 +1016,8 @@ export function generationRequest(record, ids, {text, paths = {}} = {}) { let after = null; if (row.before) { if (clips.some(clip => clip.id === row.before.id)) after = row.before.id; - else { const path = paths[key(row.before.source)]; check(typeof path === 'string' && path.length > 0, 'source', 'missing source path'); after = {work: continuedPath(row.before.source, path)}; } + else if (queue && chainSource(row.before)) after = {chain: row.before.job, block: row.before.id}; + else { const path = paths[continuationKey(row.before)]; check(typeof path === 'string' && path.length > 0, 'source', 'missing source path'); after = {work: continuedPath(row.before.source, path)}; } } blocks.push({id: row.clip.id, part: row.clip.part ?? row.clip.id, after, before: null, text: text ?? row.clip.text, frames: row.request_frames}); plan[row.clip.id] = {target_frames: row.target_frames, net_frames: row.net_frames, request_frames: row.request_frames, audio: row.clip.link}; From eaa14d5c6cf880fb5d326c96be78af0b82104e4a Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 07/21] Add edit deletion and discard untouched empty edits - Edit deletion: final wording and danger-coloured confirm button --- freevideo_engine/comfy_editor.py | 85 +++++++++++++++++++++++++++++++- web/editor.css | 3 ++ web/editor.js | 78 ++++++++++++++++++++++++++--- web/editor_lifecycle.js | 26 ++++++++++ web/editor_model.js | 8 ++- web/editor_store.js | 23 ++++++++- 6 files changed, 213 insertions(+), 10 deletions(-) create mode 100644 web/editor_lifecycle.js diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index 57053c62..1d88c671 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -43,6 +43,8 @@ # A note is one line: the page refuses the same characters (editor_model.js NOTE_BREAK) and the same ends. _NOTE_BREAK = re.compile('[\x00-\x1f\x7f-\x9f\u2028\u2029]') _STATE = threading.RLock() +_SESSIONS = {} # (output root, project id) -> {session token: monotonic expiry} +_SESSION_SECONDS = 60 _TRANSCODING = threading.Lock() _FAILED_IMPORTS = {} # terminal statuses awaiting writable storage _HEADERS = {'Cache-Control': 'no-store'} @@ -69,6 +71,12 @@ class TooLarge(ValueError): pass +class DeleteRefused(ValueError): + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + def _whole(value, low=0, high=2**53 - 1): return type(value) in (int, float) and low <= value <= high and value == int(value) @@ -314,6 +322,11 @@ def bounded(value, depth=0): raise ValueError('Invalid size mode') if size_mode == 'fixed': result['size_mode'] = size_mode + if 'keep_empty' in record: + if type(record['keep_empty']) is not bool: + raise ValueError('Invalid empty edit retention') + if record['keep_empty']: + result['keep_empty'] = True result.update(marker_fields(record)) if 'replacements' in record: replacements = record['replacements'] @@ -574,7 +587,8 @@ def list_projects(root): cover = sources(root, [source_key(first['source'])]) if first and first['source'] else {} rows.append(dict(id=identity, name=record['name'], updated_at=datetime.fromtimestamp(info.st_mtime, timezone.utc).isoformat(), frames=duration(record), fps=FPS, width=record['width'], height=record['height'], - thumbnail=next(iter(cover.values()), {}).get('thumbnail'))) + thumbnail=next(iter(cover.values()), {}).get('thumbnail'), + **({'keep_empty': True} if record.get('keep_empty') else {}))) return rows @@ -584,6 +598,59 @@ def load(root, identity): return dict(project=record, sources=sources(root, references(record))) +def project_session(root, identity, session, release=False): + """A tab renews its short lease while the edit is open; crashed tabs expire.""" + with _STATE: + path = _project_path(root, identity) + if not isinstance(session, str) or not re.fullmatch(r'[0-9a-f]{32}', session): + raise ValueError('Invalid editor session') + key = (str(Path(root).resolve()), identity) + now = time.monotonic() + for old, owners in list(_SESSIONS.items()): + live = {token: until for token, until in owners.items() if until > now} + if live: + _SESSIONS[old] = live + else: + _SESSIONS.pop(old, None) + if release: + _SESSIONS.get(key, {}).pop(session, None) + else: + if not path.is_file(): + raise MissingProject('Saved project unavailable') + _SESSIONS.setdefault(key, {})[session] = now + _SESSION_SECONDS + + +def delete_project(root, identity, session=None): + """Delete only this edit's retained directory, never its source media.""" + with _STATE: + path = _project_path(root, identity) + if session is not None and (not isinstance(session, str) or not re.fullmatch(r'[0-9a-f]{32}', session)): + raise ValueError('Invalid editor session') + if not path.parent.exists(): + return + key = (str(Path(root).resolve()), identity) + if any(token != session and until > time.monotonic() for token, until in _SESSIONS.get(key, {}).items()): + raise DeleteRefused('project_in_use') + if path.exists(): + record = _read_project(root, identity) + tracks = [*record['tracks'], *(track for tx in record.get('replacements', []) for track in [*tx['originals'], *tx['clips']])] + clips = [clip for track in tracks for clip in track['clips']] + from .comfy_chain import RUNNER + for clip in clips: + active = clip.get('pending') in ('queued', 'running') + if RUNNER is not None and clip.get('job'): + try: + active = RUNNER.get(clip['job'])['state'] in ('queued', 'running', 'cancelling') + except KeyError: + pass + if active: + raise DeleteRefused('generation_active') + # _project_path rejects links in the owned path. rmtree does not follow + # links inside it, and the state lock excludes publication by save(). + shutil.rmtree(path.parent) + _SESSIONS.pop(key, None) + + def load_by_work(root, work): if not _work(work): raise ValueError('Invalid creation id') @@ -1173,6 +1240,8 @@ async def guarded(request): return web.json_response(dict(error='Local page required'), status=403, headers=_HEADERS) try: return await handler(request) + except DeleteRefused as error: + return web.json_response(dict(error='Cannot delete this edit', reason=error.reason), status=409, headers=_HEADERS) except Conflict as error: return web.json_response(dict(revision=error.revision, **({'id': error.identity} if error.identity else {})), status=409, headers=_HEADERS) except TooLarge: @@ -1216,6 +1285,20 @@ async def persist(request): return web.json_response(dict(saved=False, retry=True, **_disk_full(error)), status=503, headers=_HEADERS) return web.json_response(dict(project=result), headers=_HEADERS) + @endpoint('post', '/freevideo/editor/project/delete') + async def remove_project(request): + body = await _body(request) + await asyncio.to_thread(delete_project, root(), body.get('id'), body.get('session')) + return web.json_response(dict(deleted=True), headers=_HEADERS) + + @endpoint('post', '/freevideo/editor/project/session') + async def use_project(request): + body = await _body(request) + if type(body.get('release', False)) is not bool: + raise ValueError('Invalid session action') + await asyncio.to_thread(project_session, root(), body.get('id'), body.get('session'), body.get('release', False)) + return web.json_response(dict(ok=True), headers=_HEADERS) + @endpoint('get', '/freevideo/editor/sources') async def source_lookup(request): keys = request.query.getall('key', []) diff --git a/web/editor.css b/web/editor.css index f570f3dc..9b17abda 100644 --- a/web/editor.css +++ b/web/editor.css @@ -162,6 +162,9 @@ .fv-editor .fv-ed-name-menu button:disabled{opacity:.4;cursor:not-allowed} .fv-ed-name-menu [role=separator]{flex:none;height:1px;margin:5px 4px;background:var(--fv-border)} .fv-ed-rename{width:100%;margin-bottom:4px} +.fv-editor .fv-ed-name-menu .fv-ed-delete-confirm{color:var(--fv-danger)} +.fv-editor .fv-ed-name-menu .fv-ed-delete-confirm:not(:disabled):is(:hover,:focus-visible){background:var(--fv-danger-subtle)} +.fv-ed-delete-question{margin:6px 10px 10px;line-height:1.5;overflow-wrap:anywhere;white-space:normal} .fv-ed-notice{position:absolute;bottom:340px;right:16px;max-width:min(560px,calc(100vw - 32px));border:1px solid var(--fv-border);background:var(--fv-card);padding:10px 14px;border-radius:8px;z-index:9;color:var(--fv-ink)} @media(max-width:680px){ .fv-studio.fv-editor{grid-template-columns:minmax(0,1fr);grid-template-rows:auto minmax(max(0px,calc(180px - var(--fv-ed-description-growth,0px))),1fr) 40px var(--fv-ed-generation-row,auto) 40px 236px;overflow:hidden} diff --git a/web/editor.js b/web/editor.js index eab8237f..3195f362 100644 --- a/web/editor.js +++ b/web/editor.js @@ -6,6 +6,7 @@ import {wordmark} from './wordmark.js'; import {openSetup} from './setup.js'; import {feedbackButton} from './feedback.js'; import * as model from './editor_model.js'; +import {discardEmptyEdit, hiddenEmptyEdit, deleteEditText} from './editor_lifecycle.js'; import {createEditorState} from './editor_state.js'; import {createEditorStore, sourceKey, mediaURL} from './editor_store.js'; import {generationSize, continuedSize} from './editor_generation.js'; @@ -560,6 +561,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given } async function replaceProject(record, key = null) { rememberView(); viewReady = false; + await store.activateProject(record.id).catch(() => {}); viewKey = record.id || key || 'draft:' + (globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`); let remembered = editorViews.get(viewKey); if (!remembered) try { remembered = JSON.parse(localStorage.getItem(viewPrefix + viewKey)); } catch {} @@ -613,17 +615,72 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (!clips.length || clips.some(clip => !clip || clip.source || clip.pending !== 'draft' || clip.job) || entry.ids.some(isHeld)) continue; held.push({ids: [...entry.ids], text: entry.text}); markHeld(entry.ids, true); } - viewReady = true; updateTransport(); if (remembered?.held?.length) rememberView(); if (held.length) void startHeld(); + viewReady = true; + if (record.id) try { localStorage.setItem('fv-editor-last', record.id); } catch {} + updateTransport(); if (remembered?.held?.length) rememberView(); if (held.length) void startHeld(); } async function switchProject(id) { - if (switching) return; switching = true; updateTop(); + if (switching || closing || id && id === state.record.id) return; switching = true; element.inert = true; updateTop(); try { await queue.flush(); if (queue.dirty) { return; } const record = id ? await store.load(id) : initial(); - if (!disposed && record) { await replaceProject(record); if (!id) { queue.enqueue(state.record); } menu.hidden = true; } - } catch (value) { error(value); } - finally { switching = false; updateTop(); } + if (!disposed && record) { + if (emptyEdit()) await discardProject({automatic: true}); + await replaceProject(record); if (!id) { queue.enqueue(state.record); } menu.hidden = true; + } + } catch (value) { + const reason = deleteEditText(t)[value?.reason]; + if (reason) notify(reason); else error(value); + } finally { switching = false; element.inert = false; updateTop(); } + } + const emptyEdit = () => discardEmptyEdit(state.record, initial(), {busy: jobActive(), hasMedia: store.uploading}); + function forgetProject() { + const keys = new Set([state.record.id, viewKey].filter(Boolean)); + for (const key of keys) editorViews.delete(key); + if (keys.has(lastDraft)) lastDraft = null; + try { + for (const key of keys) localStorage.removeItem(viewPrefix + key); + if (keys.has(localStorage.getItem('fv-editor-last'))) localStorage.removeItem('fv-editor-last'); + if (keys.has(globalThis.sessionStorage?.getItem('fv-editor-draft'))) globalThis.sessionStorage?.removeItem('fv-editor-draft'); + } catch {} + viewReady = false; // replaceProject/close must not remember the deleted edit again. + } + async function discardProject({automatic = false} = {}) { + if (state.record.id) try { await store.deleteProject(state.record.id); } + catch (value) { + // The last window to leave will remove a shared empty edit. Earlier + // windows must still be able to leave, quietly releasing their lease. + if (!automatic || value?.reason !== 'project_in_use') throw value; + } + queue.reset(); forgetProject(); + } + async function deleteCurrentProject() { + if (switching || closing) return; + switching = true; element.inert = true; updateTop(); + try { + const words = deleteEditText(t); + if (jobActive()) throw {reason: 'generation_active'}; + await queue.flush(); releaseStamp(); + if (queue.dirty) { notify(saveErrorText(queue.failure, t)); return; } + // Load the replacement before deleting, so a read failure keeps the edit open. + const rows = (await store.projects()).filter(row => row.id !== state.record.id) + .sort((a, b) => String(b.updated_at).localeCompare(String(a.updated_at))); + const record = rows.length ? await store.load(rows[0].id) : initial(); + if (!record) throw new Error(words.delete_failed); + await discardProject(); + await replaceProject(record); menu.hidden = true; title.setAttribute('aria-expanded', 'false'); title.focus(); + } catch (value) { + const words = deleteEditText(t); notify(words[value?.reason] || words.delete_failed); + } finally { switching = false; element.inert = false; updateTop(); } + } + function confirmDelete() { + const words = deleteEditText(t, state.record.name); + menu.replaceChildren(); + menu.append(el('p', 'fv-ed-delete-question', words.confirm)); + const remove = button(words.remove, deleteCurrentProject, 'fv-ed-delete-confirm'); + const cancel = button(words.cancel, showMenu); + menu.append(remove, cancel); cancel.focus(); } async function showMenu() { menu.replaceChildren(); menu.hidden = false; title.setAttribute('aria-expanded', 'true'); @@ -635,9 +692,10 @@ export function openEditor({api, app, from = null, panel = {}, components: given const fresh = button(t('New', '新建'), () => switchProject(null)); fresh.setAttribute('role', 'menuitem'); menu.append(name, fresh); pictureTools.appendMenu(menu); + const remove = button(deleteEditText(t).item, confirmDelete, 'fv-ed-delete'); remove.setAttribute('role', 'menuitem'); menu.append(remove); try { // The current edit is already named in the box at the top. - const rows = (await store.projects()).filter(row => row.id !== state.record.id); if (disposed || menu.hidden) return; + const rows = (await store.projects()).filter(row => row.id !== state.record.id && !hiddenEmptyEdit(row, initial())); if (disposed || menu.hidden || !menu.contains(name)) return; if (rows.length) { const line = el('div'); line.setAttribute('role', 'separator'); menu.append(line); } for (const row of rows) { const item = button(row.name || new Date(row.updated_at).toLocaleDateString(), () => switchProject(row.id), 'fv-ed-project'); item.setAttribute('role', 'menuitem'); item.title = item.textContent; menu.append(item); } } catch (value) { error(value); } @@ -1211,6 +1269,14 @@ export function openEditor({api, app, from = null, panel = {}, components: given saveWarning.textContent = t('Not saved yet. Switching again will lose recent changes.', '还没保存好，再切换会丢掉最近的改动'); saveWarning.hidden = false; return false; } + if (!queue.dirty && emptyEdit()) { + try { await discardProject({automatic: true}); } + catch (value) { + notify(deleteEditText(t)[value?.reason] || deleteEditText(t).delete_failed); + closing = false; element.inert = false; closePromise = null; return false; + } + } + await store.activateProject(null).catch(() => {}); rememberView(); disposed = true; guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); textTools.dispose(); adjust.dispose(); notes.dispose(); pictureEdit.dispose(); audio.dispose(); headerFit.dispose(); diff --git a/web/editor_lifecycle.js b/web/editor_lifecycle.js new file mode 100644 index 00000000..eed7b512 --- /dev/null +++ b/web/editor_lifecycle.js @@ -0,0 +1,26 @@ +const untitled = name => ['Untitled edit', '未命名剪辑'].includes(name); +const sorted = value => value && typeof value === 'object' + ? Array.isArray(value) ? value.map(sorted) : Object.fromEntries(Object.keys(value).sort().map(key => [key, sorted(value[key])])) : value; + +export function discardEmptyEdit(record, initial, {busy = false, hasMedia = false} = {}) { + if (busy || hasMedia || record.keep_empty || !untitled(record.name) + || record.tracks.some(track => track.clips.length) || record.replacements?.length) return false; + // Compare settings, track controls and sparse project fields with initial(). + const content = value => { + const {id, revision, serial, name, ...rest} = value; + return rest; + }; + return JSON.stringify(sorted(content(record))) === JSON.stringify(sorted(content(initial))); +} + +export const hiddenEmptyEdit = (row, initial) => row.frames === 0 && !row.keep_empty && untitled(row.name) + && row.width === initial.width && row.height === initial.height; + +export const deleteEditText = (t, name = '') => ({ + item: t('Delete edit', '删除剪辑'), + confirm: t(`Delete "${name}"? This can't be undone.`, `删除“${name}”？删除后无法恢复。`), + remove: t('Delete', '删除'), cancel: t('Cancel', '取消'), + project_in_use: t('This edit is open in another window. Close it there first.', '这个剪辑在另一个窗口中打开，请先在那里关闭。'), + generation_active: t('This edit is still generating. You can delete it when it finishes.', '正在生成，完成后才能删除。'), + delete_failed: t('Could not delete this edit. Please try again.', '删除失败，请重试。'), +}); diff --git a/web/editor_model.js b/web/editor_model.js index 1fa226c9..5f8b4d19 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -101,6 +101,7 @@ export function setFlag(record, key, value) { const next = serialize(record), settings = {...(next.settings || {})}; if (value) settings[key] = true; else delete settings[key]; next.settings = Object.keys(settings).length ? settings : null; + if (JSON.stringify(next.settings) !== JSON.stringify(record.settings)) next.keep_empty = true; return parse(next); } export function addAudioTrack(record) { @@ -194,6 +195,8 @@ export function parse(value) { width: value.width, height: value.height, settings, serial: value.serial, tracks}; check(value.size_mode === undefined || value.size_mode === 'first' || value.size_mode === 'fixed', 'size_mode'); result.size_mode = value.size_mode ?? 'first'; + check(value.keep_empty === undefined || typeof value.keep_empty === 'boolean', 'invalid'); + if (value.keep_empty) result.keep_empty = true; Object.assign(result, markerFields(value)); if (value.replacements !== undefined) { check(Array.isArray(value.replacements) && value.replacements.length > 0 && value.replacements.length <= RULES.maxClips, 'invalid'); @@ -613,8 +616,9 @@ export function setVolume(record, id, volume) { } return finish(next); } -export const rename = (record, name) => finish({...copy(record), name}); -export const setPictureSize = (record, width, height, size_mode = 'fixed') => finish({...copy(record), width, height, size_mode}); +export const rename = (record, name) => finish({...copy(record), name, ...(name !== record.name ? {keep_empty: true} : {})}); +export const setPictureSize = (record, width, height, size_mode = 'fixed') => finish({...copy(record), width, height, size_mode, + ...(width !== record.width || height !== record.height || size_mode !== record.size_mode ? {keep_empty: true} : {})}); export const setPicture = pictureEditor({copy, trackBy, check, pictureFit, finish}); export const setPictureFit = (record, id, fit) => setPicture(record, id, {fit: pictureFit(fit)}); diff --git a/web/editor_store.js b/web/editor_store.js index 86a804ee..d24c54d4 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -179,6 +179,17 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener let generationAvailable = false, generationGone = false, generationProbe = null, generationRetry = null, generationAttempts = 0; let disposed = false; const uploads = new Set(); const session = holdSession(); + let activeProject = null, sessionTimer = null, sessionWrites = Promise.resolve(); + function activateProject(id) { + clock.clearTimeout(sessionTimer); sessionTimer = null; + const previous = activeProject; activeProject = id; + sessionWrites = sessionWrites.catch(() => {}).then(async () => { + if (previous && previous !== id) await json('/freevideo/editor/project/session', {...post({id: previous, session: session.session, release: true}), keepalive: true}); + if (id) await json('/freevideo/editor/project/session', post({id, session: session.session})); + }); + if (id && !disposed) sessionTimer = clock.setTimeout(() => { activateProject(activeProject).catch(() => {}); }, 20000); + return sessionWrites; + } const notice = (key, en, zh) => { if (!once.has(key)) { once.add(key); notify(t(en, zh)); } }; const request = (path, options) => api.fetchApi(path, options); async function json(path, options, fallback = null) { @@ -222,6 +233,15 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener ? injected({clock, sources, request: json, mediaURL: value => mediaURL(api, value)}) : injected; const store = { sources, cache, + activateProject, + get uploading() { return uploads.size > 0; }, + async deleteProject(id) { + await sessionWrites; + const response = await request('/freevideo/editor/project/delete', post({id, session: session.session})); + if (!response.ok) throw await saveFailure(response); + memoryProjects.delete(id); + if (activeProject === id) { activeProject = null; clock.clearTimeout(sessionTimer); sessionTimer = null; } + }, get generationAvailable() { return standIn ? standIn.generationAvailable ?? true : generationAvailable; }, // The server said for certain it cannot generate (not just a missed answer that is asked again). get generationGone() { return !standIn && generationGone; }, @@ -302,6 +322,7 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener onSaved: (record, memory, local) => { memoryProjects.set(record.id || local, copy({...state.record, id: record.id, revision: record.revision})); state.adopt({id: record.id, revision: record.revision}); + if (record.id && record.id !== activeProject) activateProject(record.id).catch(() => {}); try { localStorage.setItem('fv-editor-last', record.id || local); } catch {} if (memory) memoryProjects.set(record.id || local, copy(state.record)); }, @@ -460,7 +481,7 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener installEars: () => json('/freevideo/editor/subtitle-pack', post({action: 'install', language: true})), cancelEarsPack: () => json('/freevideo/editor/subtitle-pack', post({action: 'cancel'})), watchEarsPack(fn) { return watch(() => json('/freevideo/editor/subtitle-pack'), fn); }, - dispose() { disposed = true; session.dispose(); for (const poller of pollers) poller.dispose(); pollers.clear(); watchers.clear(); + dispose() { disposed = true; activateProject(null).catch(() => {}); session.dispose(); for (const poller of pollers) poller.dispose(); pollers.clear(); watchers.clear(); for (const upload of [...uploads]) { upload.release(); upload.xhr.abort(); upload.reject(new Error('editor_closed')); } }, }; return store; From d06f56f2b8c363563070081453654382d893545a Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 08/21] Editor: restore cover follows the same open rule as the workspace Editor: restore cover follows the same open rule as the workspace; finishing stage and project name on reopen --- scripts/test_freevideo_entry.mjs | 23 ++++++++++++-- web/editor.js | 14 +++++++-- web/editor_entry.js | 3 +- web/editor_genbar.js | 38 +++++++++++++++++++--- web/freevideo.js | 30 ++++++++++++------ web/health.js | 4 ++- web/launcher.js | 5 ++- web/workspace_restore.js | 54 ++++++++++++++++++++++++++++++++ 8 files changed, 151 insertions(+), 20 deletions(-) create mode 100644 web/workspace_restore.js diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index 80c2d966..7a8bceb3 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -29,6 +29,10 @@ class Element { test('main browser entry registers both views and preserves node hooks', async t => { const extensions = [], opened = [], installed = [], events = new EventTarget(); let refreshed = 0, compatibilityChecks = 0, updateChecks = 0, doneNotifyStarts = 0, rememberedView = null; + const openingEvents = []; + let restoreReleases = 0; + let launchReady = null; + const opening = {setName: name => openingEvents.push(['name', name]), cover: () => openingEvents.push(['cover']), finish: () => openingEvents.push(['finish'])}; const casts = []; let progressView, progressOptions, realProgressFactory; const app = { @@ -73,6 +77,7 @@ test('main browser entry registers both views and preserves node hooks', async t startDoneNotify: () => { doneNotifyStarts++; }, askDesktopPermission: () => {}, installNavigation: callback => installed.push(callback), refreshNavigation: () => refreshed++, preferredView: () => rememberedView, + beginWorkspaceRestore: () => { openingEvents.push(['begin']); return opening; }, finishWorkspaceRestore: () => { restoreReleases++; }, waitForLauncherRestore: () => launchReady, attachReferencePicker() {}, referenceItems: () => [], syncReferencePrompt() {}, shareButton: () => new Element('button'), reportFileName: () => 'FreeVideo-report-20261008-101530.json', rememberPromptDraft() {}, savePromptDraft: async () => false, openImageEditor: async () => null, icon: () => '', @@ -89,6 +94,7 @@ test('main browser entry registers both views and preserves node hooks', async t } try { let source = await readFile(new URL('../web/freevideo.js', import.meta.url), 'utf8'); + source = source.replace("import { beginWorkspaceRestore, finishWorkspaceRestore, waitForLauncherRestore } from './workspace_restore.js';", 'const {beginWorkspaceRestore,finishWorkspaceRestore,waitForLauncherRestore} = globalThis.__freevideoEntryTest;'); source = source.replace("import { openEditorView } from './editor_loader.js';", 'const openEditorView = (...args) => globalThis.__freevideoEntryTest.openEditorView?.(...args);'); source = source .replace("import { createErrorPanel, errorText, createErrorReport, adviceFor } from './error_panel.js';", 'const {createErrorPanel,errorText,createErrorReport,adviceFor} = globalThis.__freevideoEntryTest;') @@ -225,8 +231,10 @@ test('main browser entry registers both views and preserves node hooks', async t assert.equal(node.widgets[0].value, 8, 'Migrate the previous private force-checkbox slot'); node.buttons[0].callback(); const quiet = opened.length; + const releasesBefore = restoreReleases; extension.afterConfigureGraph(); assert.equal(opened.length, quiet, 'a tab restored with the page does not open the workspace on its own'); + assert.equal(restoreReleases, releasesBefore + 1, 'the page-load cover is released when nothing opens'); events.dispatchEvent(new Event('pointerdown')); extension.afterConfigureGraph(); assert.deepEqual(opened, [node, node]); @@ -248,10 +256,21 @@ test('main browser entry registers both views and preserves node hooks', async t assert.equal(workflows, 1); assert.equal(opened.length, before + 2); rememberedView = 'editor'; const restored = []; - dependencies.openEditorView = (value, options) => restored.push({node:value, studio:options.studio}); + let finishLaunch, finishEditor; + launchReady = new Promise(resolve => { finishLaunch = resolve; }); + const ready = new Promise(resolve => { finishEditor = resolve; }); + dependencies.openEditorView = (value, options) => { restored.push({node:value, studio:options.studio}); options.onProjectLoaded({name: 'saved project'}); return {ready}; }; + const beforeRestore = opened.length; extension.afterConfigureGraph(); + assert.equal(restored.length, 0, 'wait for launcher/session restoration before choosing the workspace'); + finishLaunch(); await Promise.resolve(); await Promise.resolve(); assert.equal(restored[0].node, node); - assert.ok(restored[0].studio, 'restore opens the studio before the editor'); + assert.equal(restored[0].studio, undefined, 'restore enters the editor directly'); + assert.equal(opened.length, beforeRestore, 'the creation panel never flashes during a successful recovery'); + assert.deepEqual(openingEvents, [['begin'], ['name', 'saved project'], ['cover']]); + finishEditor(); await Promise.resolve(); await Promise.resolve(); + assert.deepEqual(openingEvents.at(-1), ['finish'], 'keep the opening status and update suppression until editor.ready'); + launchReady = null; // The lazy loader owns failed imports: it notifies, reports through onFailure and resolves null. dependencies.openEditorView = async (node, options) => { options.onFailure(new Error('editor unavailable')); return null; }; extension.afterConfigureGraph(); await Promise.resolve(); diff --git a/web/editor.js b/web/editor.js index 3195f362..dafaabe0 100644 --- a/web/editor.js +++ b/web/editor.js @@ -219,10 +219,15 @@ export function openEditor({api, app, from = null, panel = {}, components: given let submissionFailure = null; let blockStates = null, progressObserver = null, progressConnection = null; function barJob(job) { + if (!job) return job; const latest = [...runProgress.values()].filter(message => message.chain.id === job.id) + .filter(message => job.blocks?.some(block => block.id === message.chain.block && ['running', 'queued'].includes(block.state))) .sort((a, b) => (a.received_at || 0) - (b.received_at || 0)).at(-1); const words = panel.progressWords?.() || null; - return latest ? {...job, chainProgress: latest.chain, words} : words ? {...job, words} : job; + const loadingClip = !job.blocks?.some(block => ['pending', 'queued', 'running'].includes(block.state)) + && job.blocks?.some(block => block.state === 'done' && block.output && ['queued', 'running'].includes(model.attemptClip(state.record, block.id)?.pending)); + return {...job, ...(loadingClip ? {state: 'running', loadingClip: true} : {}), + ...(latest ? {chainProgress: latest.chain, progressMessage: latest} : {}), words}; } function paintJob(job) { const messages = blockMessages([...runProgress.values()].filter(message => message.chain.id === job.id), @@ -230,7 +235,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given const values = editorBlockStates(job, messages, blockStates); for (const block of job.blocks || []) { const clip = model.attemptClip(state.record, block.id); if (!clip || clip.job !== job.id) continue; - const value = {...values[block.id], run: `${job.id}:${block.prompt_id || block.attempts || 0}`}; + const loading = block.state === 'done' && block.output && ['queued', 'running'].includes(clip.pending); + const value = {...values[block.id], ...(loading ? {state: 'generating', fraction: .99} : {}), run: `${job.id}:${block.prompt_id || block.attempts || 0}`}; + if (value.state === 'generating') value.fraction = Math.min(.99, value.fraction || 0); for (const pair of model.pair(state.record, block.id)) { progress.set(pair.id, value); tracks?.setProgress(pair.id, value); } } } @@ -492,6 +499,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given return ['queued', 'running', 'cancelling'].includes(previous.state); failures.reconcile(job, previous); mostRecentJob = job.id; jobStatuses.set(job.id, job); paintJob(job); + genbar?.update(barJob(job)); const completed = new Set(); for (const block of job.blocks || []) { const clip = model.attemptClip(state.record, block.id); if (!clip || clip.job !== job.id) continue; @@ -551,6 +559,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given value.held = waiting; localStorage.setItem(viewPrefix + key, JSON.stringify({selection: value.selection, text: value.text, ...(waiting.length ? {held: waiting} : {})})); if (state.record.id) localStorage.setItem('fv-editor-last', key); + localStorage.setItem('fv-editor-opening', JSON.stringify({id: key, name: state.record.name})); } catch {} // An untouched edit belongs to this tab until it has a saved identity. lastDraft = !state.record.id && state.record.tracks.every(track => !track.clips.length) ? key : null; @@ -570,6 +579,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given player.pause(); for (const stop of jobWatchers.values()) stop(); jobWatchers.clear(); jobStatuses.clear(); progress.clear(); runProgress.clear(); failures.clear(); progressConnection?.reset(); mostRecentJob = null; submissionFailure = null; genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); suppressSave = true; try { state.replace(record); savedContent = content(record); } finally { suppressSave = false; } + panel.onProjectLoaded?.(record); const width = tracksHost.clientWidth - 148; if (width > 0) state.setZoom(model.openingZoom(width, record)); tracks?.setScrollLeft(0); // An empty edit opens with one part's length selected at the start, ready to generate. diff --git a/web/editor_entry.js b/web/editor_entry.js index 2c9aab27..4b52140d 100644 --- a/web/editor_entry.js +++ b/web/editor_entry.js @@ -23,9 +23,10 @@ function editorPanel(node) { return {node: String(node.id), prompt: value.output, extra_data: {extra_pnginfo: {workflow: value.workflow || {}}}}; }}; } -export function openEditorView(node, {studio, work = null, extend = false} = {}) { +export function openEditorView(node, {studio, work = null, extend = false, onProjectLoaded} = {}) { studio?.querySelectorAll('video,audio').forEach(media => media.pause()); const panel = editorPanel(node); + if (onProjectLoaded) panel.onProjectLoaded = onProjectLoaded; const progressElement = studio?.querySelector('.fv-generation-progress'); if (progressElement) { panel.progressElement = progressElement; diff --git a/web/editor_genbar.js b/web/editor_genbar.js index 728a6a4c..a953c54d 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -6,6 +6,35 @@ const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60 ? t(`about ${Math.max(1, Math.ceil(seconds))} s`, `约 ${Math.max(1, Math.ceil(seconds))} 秒`) : t(`about ${Math.ceil(seconds / 60)} minutes`, `约 ${Math.ceil(seconds / 60)} 分钟`); +// Provisional copy keys for Iris; decode/save sentences follow Studio's stage vocabulary. +export const CLIP_FINALIZATION_COPY = { + prepareDecode: ['Preparing video decoding', '正在准备解码视频'], + loadDecoder: ['Loading the video decoder', '正在载入视频解码器'], + decodeVideo: ['Decoding the video', '正在解码视频'], + decodeAudio: ['Decoding the audio', '正在解码音频'], + saveVideo: ['Saving the video', '正在保存视频'], + openClip: ['Loading the generated clip', '正在载入生成片段'], + joinClips: ['Combining the clips', '正在合成片段'], +}; +export function clipFinalization(job) { + if (!job || !['running', 'queued'].includes(job.state)) return null; + const message = job.progressMessage || job.progress || {}, blocks = job.blocks || []; + const phase = message.phase || message.timing_phase, label = message.label || ''; + let key; + if (job.joining) key = 'joinClips'; + else if (job.loadingClip || phase === 'complete' || message.overall?.status === 'complete' + || blocks.some(block => block.state === 'running' && block.fraction >= 1)) key = 'openClip'; + else if (phase === 'save' || /^Saving (MP4 and audio|decoded video|decoded audio)$/.test(label)) key = 'saveVideo'; + else if (phase === 'sample_finalize') key = 'prepareDecode'; + else if (phase === 'decode' || message.timing_phase === 'decode') key = /audio/i.test(label) && !/video and audio/i.test(label) ? 'decodeAudio' + : label === 'Loading video VAE' ? 'loadDecoder' : 'decodeVideo'; + if (!key) return null; + // Decode/save and joining retain engine cancellation. A saved final part waiting + // for its receipt/media cannot be cancelled; later queued parts still can. + const current = message.chain?.block || blocks.find(block => block.state === 'running')?.id; + const later = blocks.some(block => block.id !== current && ['pending', 'queued', 'running'].includes(block.state)); + return {key, cancelable: key !== 'openClip' || !job.loadingClip && later}; +} // After the next frame is on screen; a page nobody sees has no frame to wait for. const painted = () => typeof document === 'undefined' || document.visibilityState !== 'visible' || typeof requestAnimationFrame !== 'function' ? Promise.resolve() : new Promise(resolve => requestAnimationFrame(() => { const {port1, port2} = new MessageChannel(); port1.onmessage = () => { port1.close(); resolve(); }; port2.postMessage(null); })); @@ -88,7 +117,8 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { generate.classList.toggle('fv-primary', selected && !busy); // A cancel already accepted is not offered again; the bar says it is stopping until the run ends. const cancelling = status?.state === 'cancelling' && !held; - cancel.hidden = !busy || opening || cancelling; resume.hidden = !stopped; + const finalization = clipFinalization(status); + cancel.hidden = !busy || opening || cancelling || !held && finalization?.cancelable === false; resume.hidden = !stopped; const waiting = status?.state === 'queued' && !(status.blocks || []).some(block => ['queued', 'running'].includes(block.state) && block.prompt_id); if (opening) hint.textContent = t('Starting…', '正在开始…'); else if (held || waiting) hint.textContent = t('Queued', '排队中'); @@ -103,8 +133,8 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { // A single part needs no number; with nothing else to say it reads "Preparing". // The creation panel's own time when it has one, so the bar and the panel never differ; the job's sum until the panel has one. const left = words?.over ? null : words?.left ?? remaining; - const time = words?.stall || (words?.over ? t('Taking longer than expected', '比预计久，仍在生成') : words?.time || (left != null ? timeEstimate(left, t) : '')); - const stage = words?.stage || (count === 1 && !time ? t('Preparing', '正在准备') : ''); + const time = finalization ? '' : words?.stall || (words?.over ? t('Taking longer than expected', '比预计久，仍在生成') : words?.time || (left != null ? timeEstimate(left, t) : '')); + const stage = finalization ? t(...CLIP_FINALIZATION_COPY[finalization.key]) : words?.stage || (count === 1 && !time ? t('Preparing', '正在准备') : ''); const spans = [['fv-ed-gen-part', count > 1 ? t(`Part ${number}/${count}`, `第 ${number}/${count} 段`) : ''], ['fv-ed-gen-stage', stage], ['fv-ed-gen-time', time]]; const shown = spans.filter(([, text]) => text).map(([cls, text], index) => [cls, index ? ` · ${text}` : text]); @@ -223,7 +253,7 @@ export function editorBlockStates(job, messages = [], resolve = null) { : block.fraction; return [block.id, {state: running && live ? 'generating' : states[block.state] || 'waiting', ...(['pending', 'queued'].includes(block.state) && !live ? {queued: true} : {}), - fraction: Math.max(0, Math.min(1, fraction || 0)), + fraction: Math.max(0, Math.min(block.state === 'done' ? 1 : .99, fraction || 0)), remainingSeconds: running && live ? live.remaining_seconds ?? live.overall?.remaining_seconds ?? block.remaining_seconds ?? null : block.remaining_seconds ?? null}]; })); diff --git a/web/freevideo.js b/web/freevideo.js index 73655897..8afceb42 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -3,6 +3,7 @@ import { installNativeErrorGuard } from './native_errors.js'; import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; import { openEditorView } from './editor_loader.js'; +import { beginWorkspaceRestore, finishWorkspaceRestore, waitForLauncherRestore } from './workspace_restore.js'; import { openStudio, loraPanel, loraWarning, promptGuide } from "./studio.js"; import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { createProgressConnection } from './progress_connection.js'; @@ -797,16 +798,27 @@ app.registerExtension({ const rememberedView = preferredView(); const node = (app.graph?._nodes || []).find(isGenerate); // The user's own workflow is shown as it is. A FreeVideo workflow opens its workspace once the - // launcher or the user asked for it, never on its own for a tab restored with the page. - if (!node) { collapseWorkspaces(); return; } - if (app.graph?.extra?.freevideo_studio && rememberedView !== 'nodes' && (launchVisit || userActed)) { - requestAnimationFrame(() => { - const studio = showStudio(node); - if (studio && rememberedView === 'editor') { - void openEditorView(node, {studio, onFailure: error => health({type: 'error', where: 'editor', error})}); + // launcher or the user asked for it, never on its own for a tab restored with the page. The startup + // cover follows this same decision and is always released here when the workspace will not open. + const opens = !!node && app.graph?.extra?.freevideo_studio && rememberedView !== 'nodes' && (launchVisit || userActed); + if (!opens) { finishWorkspaceRestore(); if (!node) collapseWorkspaces(); return; } + const opening = rememberedView === 'editor' ? beginWorkspaceRestore() : null, epoch = configuration; + requestAnimationFrame(async () => { + const launch = waitForLauncherRestore(); if (launch) await launch; + if (epoch !== configuration) return; + if (!opening) { showStudio(node); return; } + try { + const handle = await openEditorView(node, {isCurrent: () => epoch === configuration, + onProjectLoaded: record => opening.setName(record.name), + onFailure: error => health({type: 'error', where: 'editor', error})}); + if (epoch !== configuration) return; + if (handle) { opening.cover(); await handle.ready; } + else { + const studio = showStudio(node), status = studio?.querySelector?.('.fv-status'); + if (status) { status.dataset.error = 'true'; status.textContent = text('Could not complete. Try again.', '没有完成，请重试'); } } - }); - } + } finally { if (epoch === configuration) opening.finish(); } + }); }, onNodeOutputsUpdated(outputs) { for (const [id, output] of Object.entries(outputs)) { diff --git a/web/health.js b/web/health.js index 071ada73..324ffcd5 100644 --- a/web/health.js +++ b/web/health.js @@ -18,7 +18,7 @@ export const MODULES = ['agent_activity.js', 'agent_connect.js', 'agent_session. 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_terms.js', 'request_assets.js', 'report_issue.js', 'studio_recent.js', 'result_actions.js', 'sampling_effort.js', 'setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'optional_inputs.js', 'studio_queue.js', 'studio_size.js', - 'toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'cast_library.js', 'media_upload.js']; + 'toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'workspace_restore.js', 'cast_library.js', 'media_upload.js']; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); let visibleAt = 0; async function waitForVisible() { @@ -368,6 +368,8 @@ function sheetLoaded(name) { function workspaceProblem(app) { // Scripts that never started are already named by the load check. if (!window.FreeVideoUI?.ready || !hasGenerate(app.graph)) return null; + const opening = document.querySelector('dialog.fv-workspace-opening[open]'); + if (opening && onScreen(opening)) return null; // The editor shares the studio shell; either view replaces the floating toolbar. const studio = document.querySelector('dialog.fv-editor[open]') || document.querySelector('dialog.fv-studio[open]'); if (studio) return onScreen(studio) ? null diff --git a/web/launcher.js b/web/launcher.js index 6b979607..2184a827 100644 --- a/web/launcher.js +++ b/web/launcher.js @@ -2,6 +2,7 @@ import { app } from '../../scripts/app.js'; import { api } from '../../scripts/api.js'; import { restorePromptDraft } from './prompt_draft.js'; import { takePortMarker } from './port_notice.js'; +import { launcherRestoreFinished } from './workspace_restore.js'; // health.js names what failed; files loaded before it queue their reports. const health = item => (window.__freevideoHealth ||= []).push(item); @@ -142,12 +143,14 @@ app.registerExtension({ } catch (error) { console.error('FreeVideo launcher:', error); health({type: 'launch', error}); + launcherRestoreFinished(); return; } const opened = cached ? undefined : app.extensionManager?.workflow?.activeWorkflow; - keepInFront(workflow, opened).then(() => health({type: 'launch'}), error => { + keepInFront(workflow, opened).then(() => { health({type: 'launch'}); launcherRestoreFinished(); }, error => { console.error('FreeVideo launcher:', error); health({type: 'launch', error}); + launcherRestoreFinished(); }); }, }); diff --git a/web/workspace_restore.js b/web/workspace_restore.js new file mode 100644 index 00000000..b3159280 --- /dev/null +++ b/web/workspace_restore.js @@ -0,0 +1,54 @@ +import { isLaunchVisit } from './launch_visit.js'; +// Small, eager startup surface: no editing-module or host imports to delay its first paint. +// Provisional copy keys for Iris. Keep project names as text, never markup. +export const WORKSPACE_OPENING_COPY = { + opening: ['Opening', '正在打开'], + openingNamed: ['Opening {name}', '正在打开 {name}'], +}; +let opening = null; +const query = new URLSearchParams(typeof location === 'undefined' ? '' : location.search); +const language = query.get('freevideo_lang'); +const cn = language === 'zh' || language !== 'en' && String(globalThis.navigator?.language || '').toLowerCase().startsWith('zh'); +let resolveLaunch; +const launchReady = query.get('freevideo') === 'launch' ? new Promise(resolve => { resolveLaunch = resolve; }) : null; +export const waitForLauncherRestore = () => launchReady; +export function launcherRestoreFinished() { resolveLaunch?.(); resolveLaunch = null; } + +function cachedName() { + try { + const value = JSON.parse(localStorage.getItem('fv-editor-opening')); + return value?.id === localStorage.getItem('fv-editor-last') ? value.name : ''; + } catch { return ''; } +} +export function beginWorkspaceRestore() { + if (opening) return opening; + const dialog = document.createElement('dialog'), label = document.createElement('span'), style = document.createElement('style'); + dialog.className = 'fv-workspace-opening'; + dialog.setAttribute('aria-busy', 'true'); label.setAttribute('role', 'status'); label.setAttribute('aria-live', 'polite'); + // Inline sizing works before the product styles arrive. Suppress cached/live update + // banners until the recovered workspace is ready, without delaying update checks. + style.textContent = '.fv-workspace-opening[open]{position:fixed;inset:0;box-sizing:border-box;width:100vw;height:100vh;max-width:none;max-height:none;margin:0;border:0;padding:24px;display:grid;place-items:center;background:#10151e;color:#e6edf6;font:16px/1.6 system-ui,sans-serif}.fv-workspace-opening::backdrop{background:#10151e}.fv-update-notice{display:none!important}'; + dialog.append(label); document.head.append(style); document.body.append(dialog); + const setName = name => { + const known = typeof name === 'string' ? name.trim() : ''; + const words = WORKSPACE_OPENING_COPY[known ? 'openingNamed' : 'opening']; + label.textContent = words[cn ? 1 : 0].replace('{name}', () => known); + }; + dialog.oncancel = event => event.preventDefault(); + const handle = {setName, cover() { + // openEditor creates a native modal; keep this status above its unfinished UI. + if (opening !== handle) return; + if (dialog.open) dialog.close(); + dialog.showModal(); + }, finish() { + if (opening !== handle) return; + opening = null; if (dialog.open) dialog.close(); dialog.remove(); style.remove(); + }}; + opening = handle; setName(cachedName()); dialog.showModal(); + return handle; +} +export function finishWorkspaceRestore() { opening?.finish(); } +// ComfyUI loads extension modules before setup/session restoration. Show words at +// this first opportunity, including ordinary refreshes after the launcher flag is gone. +// Only a launcher visit opens the workspace on its own (see afterConfigureGraph), so only it is covered here. +try { if (localStorage.getItem('freevideo.view') === 'editor' && isLaunchVisit(location.search, globalThis.sessionStorage)) beginWorkspaceRestore(); } catch { /* Storage may be unavailable. */ } From b93e6a6206cb909c885f4d990ecc84a8e4c12753 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 09/21] Run the small CPU table work of AdaLN loading on one thread so it no longer stalls on a busy host - adaln serial_ops: comment says the thread setting is process-wide --- freevideo_engine/adaln.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/freevideo_engine/adaln.py b/freevideo_engine/adaln.py index 4fbbea8b..b45a56f4 100644 --- a/freevideo_engine/adaln.py +++ b/freevideo_engine/adaln.py @@ -4,6 +4,7 @@ known before sampling, so retain the small modulation outputs instead of 26 GB of immutable projection weights. FL2VA has a separate keyframe timestep schedule. """ +import contextlib import hashlib import json from pathlib import Path @@ -12,6 +13,24 @@ import torch +@contextlib.contextmanager +def serial_ops(): + """Run the small CPU tensor work of a table on this thread alone. + + A table is a few hundred kilobytes. Spread over the full intra-op pool each + operation waits for its workers, and on a machine whose cores are busy that + wait is hundreds of times the work (about 0.3 s per block for a cat that + takes 0.2 ms alone). The thread count is a setting of the whole process, so + it is restored on exit and other threads running torch ops meanwhile see 1. + """ + previous = torch.get_num_threads() + torch.set_num_threads(1) + try: + yield + finally: + torch.set_num_threads(previous) + + def all_finite(tensor): """Whether every element is finite; reads half-precision bits instead of calling isfinite. @@ -50,8 +69,9 @@ def __init__(self, values, cursor): super().__init__() self.cursor = cursor self.steps = len(values) - for index, values_at_step in enumerate(values): - self.register_buffer(f'step_{index}', torch.cat(values_at_step, dim=-1)) + with serial_ops(): + for index, values_at_step in enumerate(values): + self.register_buffer(f'step_{index}', torch.cat(values_at_step, dim=-1)) # Steps differ in how many distinct timesteps they carry: the first H3 # step has video and audio at the same t, so its rows are fewer. The # fused pre/post kernels compile for static shapes and recompiled in the @@ -188,7 +208,9 @@ def load(self, index, steps): # These are small immutable constants; their producer's GPU/runtime is # deliberately irrelevant. Validate before transferring to this device. state = load_file(path, device=self.device) - if any(not all_finite(tensor) for tensor in state.values()): + with serial_ops(): + finite = all(all_finite(tensor) for tensor in state.values()) + if not finite: raise ValueError('Nonfinite AdaLN model asset') return [state[f'step_{i}'].chunk(6, dim=-1) for i in range(steps)] From fe5a71998bba88d233e6ccfcb97f027ce70e99e0 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 10/21] Reuse reference picture and clip latents when only the prompt changes - Reference latent cache: key on engine and diffusers versions --- freevideo_engine/encode_worker.py | 3 +- freevideo_engine/generate.py | 3 +- freevideo_engine/media_encoding.py | 88 ++++++++++++++++++++++++++++-- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index 7df26d07..67046393 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -645,7 +645,8 @@ def attempt(index, failure): add_vdn() from .media_encoding import encode_latents tick = time.perf_counter() - info = encode_latents(normalized, value, request['base'], request['geometry'], resident=resident) + info = encode_latents(normalized, value, request['base'], request['geometry'], resident=resident, + cache_root=request.get('reference_latent_cache')) keyframe_metrics.update(media_vae_encode_seconds=time.perf_counter() - tick, conditioning_info=info) task = info['task'] elif early_reference is not None: diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index d4fbaa1e..b4d0bc5d 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -752,7 +752,8 @@ def interrupted(signum, frame): 'metrics': str(destination.with_suffix('.encoding.json')), 'diagnostic_output': str(destination)} if media: - encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base')) + encoding.update(media=media, geometry=canvas, base=profile['engine'].get('base'), + reference_latent_cache=str(input_cache.root.parent.parent) if input_cache else None) # Lets the encoder child read the video model while it computes, on a slow drive. from .model_read_ahead import request_fields if getattr(args, 'cache', None): diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index 31da96f2..5014737a 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -217,7 +217,67 @@ def reference_options(normalized, options, step): return {'minimax_ref_items': items} if items else {} -def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda'): +def _latent_cache(root, base, device): + """Per-picture latent receipts; another encoder, recipe or device never matches.""" + import torch + from .input_cache import InputCache, digest + directory = Path(base) / 'vae' + files = sorted(directory.glob('*.safetensors')) + if not files: + return None + kind = torch.device(device).type + try: + from importlib.metadata import version + diffusers = version('diffusers') + except Exception: + diffusers = None + from . import __version__ + identity = {'recipe': 'reference-latent-v1', 'engine': __version__, 'diffusers': diffusers, + 'torch': torch.__version__, 'device': kind, + 'gpu': torch.cuda.get_device_name() if kind == 'cuda' else None, + 'config': digest(directory / 'config.json') if (directory / 'config.json').exists() else None, + 'weights': [[f.name, f.stat().st_size, f.stat().st_mtime_ns] for f in files]} + return InputCache(identity, root=root, kind='reference-latent') + + +def _pixels_key(mapped): + import hashlib + import numpy as np + value = hashlib.sha256(json.dumps([list(mapped.shape), str(mapped.dtype)]).encode()) + for begin in range(0, len(mapped), 8): + value.update(np.ascontiguousarray(mapped[begin:begin + 8]).reshape(-1).view('u1').data) + return value.hexdigest() + + +def _cached_latent(cache, key, directory): + import torch + target = Path(directory) / ('reference-latent-' + key[:16] + '.hit.pt') + try: + if cache.get(key, target) is None: + return None + latent = torch.load(target, map_location='cpu', weights_only=True) + if isinstance(latent, torch.Tensor) and latent.is_floating_point() and bool(torch.isfinite(latent).all()): + return latent + except (OSError, ValueError, RuntimeError, KeyError, TypeError, EOFError): + pass + finally: + target.unlink(missing_ok=True) + return None + + +def _remember_latent(cache, key, latent, directory): + import torch + path = Path(directory) / ('reference-latent-' + key[:16] + '.save.pt') + try: + torch.save(latent, path) + cache.put(key, path, {'success': True}) + except (OSError, ValueError, RuntimeError): + pass + finally: + path.unlink(missing_ok=True) + + +def encode_latents(normalized, value, base, canvas, *, resident=None, device='cuda', cache_root=None): import gc import numpy as np import torch @@ -228,7 +288,25 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu raise ValueError('Media encoding requires a CUDA or MPS device') empty_cache = torch.mps.empty_cache if torch.device(device).type == 'mps' else torch.cuda.empty_cache refs = [{key: row[key] for key in ('kind', 'early_steps') if key in row} for row in normalized] - if any(row.get('pixels') for row in normalized): + keys, cache, hits = {}, None, set() + if cache_root and any(row.get('pixels') for row in normalized): + # The same picture or clip with another prompt reuses its latent and may skip loading the encoder. + try: + cache = _latent_cache(cache_root, base, device) + for index, (row, ref) in enumerate(zip(normalized, refs)): + if cache is None or not row.get('pixels'): + continue + mapped = np.load(row['pixels'], mmap_mode='r', allow_pickle=False) + keys[index] = _pixels_key(mapped[:row['frames']]) + del mapped + latent = _cached_latent(cache, keys[index], Path(row['pixels']).parent) + if latent is not None: + ref['latent'] = latent + hits.add(index) + except (OSError, ValueError, RuntimeError, KeyError, TypeError): + cache = None + print(json.dumps({'event': 'reference_latent_cache', 'hits': len(hits), 'misses': len(keys) - len(hits)}), flush=True) + if any(row.get('pixels') for index, row in enumerate(normalized) if index not in hits): print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding image/video references'}), flush=True) # Conditioning never decodes: the 9.0 GiB decoder in the shared shards # is not read. The independent output decoder is loaded after sampling. @@ -239,8 +317,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu from .backends.mps_vae_encode import bounded_encoder lifetime = bounded_encoder(vae) with lifetime as memory: - for row, ref in zip(normalized, refs): - if row.get('pixels'): + for index, (row, ref) in enumerate(zip(normalized, refs)): + if row.get('pixels') and index not in hits: mapped = np.load(row['pixels'], mmap_mode='r', allow_pickle=False) if row['frames'] == 1: pixels = torch.from_numpy(np.array(mapped[:1])).to(device).permute(3, 0, 1, 2)[None] @@ -249,6 +327,8 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu else: ref['latent'] = encode_visual(vae, mapped[:row['frames']], device=device) del mapped + if cache is not None and index in keys: + _remember_latent(cache, keys[index], ref['latent'], Path(row['pixels']).parent) from .streamed_weights import release_file_pages release_file_pages([Path(row['pixels'])]) if memory is not None: From 0b77acd5e4656ecbc9e38ca8754125999a18b55f Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 11/21] Keep one steady line while a model group downloads; torch shows as Runtime components - Give the GPU acceleration build a visible share of the install progress - Install step line shows 'Runtime components' instead of package names - Install progress: the model line stays on one group while groups download side by side - Install progress: a build beside the download does not take the model line --- freevideo_engine/bootstrap.py | 4 +- freevideo_engine/launcher_session.py | 7 +++- freevideo_engine/setup_progress.py | 58 ++++++++++++++++++++++++++++ freevideo_engine/terminal_ui.py | 2 +- freevideo_engine/torch_download.py | 2 +- 5 files changed, 69 insertions(+), 4 deletions(-) diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index 784f6972..e16cc7ca 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -2093,7 +2093,9 @@ def built(fraction): # The GPU acceleration build is one task of the phase but takes minutes, so it # weighs in the same way: as a share of the phase that advances with its counter. -INSTALL_PHASE_BUILD_BYTES = 3 * 2**30 +# Measured on a fresh Linux install: the compile ran about twice as long as the +# 21 GiB model download, so it takes a share of the bar on that order. +INSTALL_PHASE_BUILD_BYTES = 12 * 2**30 def install_phase_units(tasks, finished, download_bytes, fetched_bytes, build_bytes=0, built=0.): diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index f33881e1..15362174 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -21,7 +21,7 @@ from .model_status import FAMILIES, NAMES from .model_guidance import (package_instructions, video_instructions, runtime_packages_supported, download_view, export_download_list, DownloadListError, DOWNLOAD_COPY) -from .setup_progress import progress_text +from .setup_progress import ModelLine, progress_text from .terminal_ui import clean, duration from .model_upgrade import TARGET from .runtime_alias import checked_real_path @@ -2060,6 +2060,11 @@ def snapshot(self): done=min(group['download_bytes'], group.get('downloaded_bytes') or 0), rate=group.get('bytes_per_second') or progress.get('rate')) file_text = progress_text(current, zh, decimal_sizes=sys.platform == 'darwin') + if not hasattr(self, 'model_line'): + self.model_line = ModelLine() + steady = self.model_line.line(progress, groups, display('Connecting to download server', zh), zh, + sys.platform == 'darwin') + file_text = steady if steady is not None else file_text if self.importer.busy and self.importer.state.get('activity'): imported = self.importer.state file_text = progress_text(dict(activity=imported['activity'], unit='bytes', diff --git a/freevideo_engine/setup_progress.py b/freevideo_engine/setup_progress.py index 0ef9e5e7..a882d203 100644 --- a/freevideo_engine/setup_progress.py +++ b/freevideo_engine/setup_progress.py @@ -158,3 +158,61 @@ def progress_text(event, zh=False, *, decimal_sizes=False): return ' · '.join([t('Downloading ', '正在下载' + gap) + name] + transfer_parts(event.get('done'), total or 0, event.get('rate'), decimal_sizes)) return '' + + +class ModelLine: + """One steady line while a model group downloads: its shards, connections and + checks must not take turns on the screen.""" + STALL_SECONDS, HOLD_SECONDS = 10., 3. + + def __init__(self): + self.group = None + self.done = self.rate = self.bytes_at = None + self.shown = self.shown_at = None + self.style = None + self.finished = set() + self.adopted_at = None + + def line(self, current, groups, connecting, zh=False, decimal_sizes=False, now=None): + """The stabilised text, or None when this is not a model download.""" + now = time.monotonic() if now is None else now + name = current.get('name') + if (zh, decimal_sizes) != self.style: + self.style, self.shown, self.shown_at = (zh, decimal_sizes), None, None + if self.group in groups and self.done is not None and self.done >= groups[self.group]['download_bytes']: + self.finished.add(self.group) + # Groups that download side by side must not take turns: the line stays on one + # group until it is complete or has gone quiet, and a finished group is not shown again. + since = self.bytes_at if self.bytes_at is not None else self.adopted_at + stalled = since is not None and now - since > self.STALL_SECONDS + if (current.get('activity') == 'download' and name in groups and name != self.group + and name not in self.finished + and (self.group is None or self.group in self.finished or stalled)): + self.group, self.done, self.rate, self.bytes_at = name, None, None, None + self.adopted_at = now + self.shown = self.shown_at = None + modelish = current.get('activity') in MODEL_ACTIVITIES or current.get('unit') == 'bytes' + # A build running beside the download does not take the line while the group is unfinished. + if self.group not in groups or not (modelish or self.group not in self.finished): + self.group = self.shown = self.shown_at = None + return None + group = groups[self.group] + total = group['download_bytes'] + done = min(total, group.get('downloaded_bytes') or 0) + if self.done is None or done > self.done: + self.bytes_at = now if done else self.bytes_at + self.done = done + if group.get('bytes_per_second'): + self.rate = group['bytes_per_second'] + shown = dict(name=self.group, total=total, done=done, rate=self.rate) + if done >= total: + text = progress_text(dict(shown, activity='verify'), zh) + elif self.bytes_at is None or now - self.bytes_at > self.STALL_SECONDS: + text = connecting + else: + text = progress_text(dict(shown, activity='download', unit='bytes'), zh, decimal_sizes=decimal_sizes) + if self.shown is not None and text != self.shown and now - self.shown_at < self.HOLD_SECONDS: + return self.shown + if text != self.shown: + self.shown, self.shown_at = text, now + return text diff --git a/freevideo_engine/terminal_ui.py b/freevideo_engine/terminal_ui.py index e6ef021d..c764a3ac 100644 --- a/freevideo_engine/terminal_ui.py +++ b/freevideo_engine/terminal_ui.py @@ -612,7 +612,7 @@ def update(**values): speed = event.get('network_bytes_per_second') shown_speed = speed if speed is not None else event['bytes_per_second'] update(done=done, total=total, scope=description + self.model_source, rate=event['bytes_per_second'], unit='bytes', - activity='download', name=event.get('name') or ('Python' if description.startswith('cpython-') else 'PyTorch' if description == 'torch' + activity='download', name=event.get('name') or ('Python' if description.startswith('cpython-') else 'Runtime components' if description == 'torch' else model_download_name(description).split(' · ')[0]), detail='%s%s%.1f MiB/s · %.1f / %.1f MiB · %s' % (source + ' · ' if source else '', diff --git a/freevideo_engine/torch_download.py b/freevideo_engine/torch_download.py index 77170c3d..5bd5bdd0 100644 --- a/freevideo_engine/torch_download.py +++ b/freevideo_engine/torch_download.py @@ -39,7 +39,7 @@ def fetch(row, root, family, networking, env, ui, check): names = list(row['sources']) candidates = [(name, row['sources'][name]) for name in names if name in row['sources']] key = 'download-' + row['filename'] - name = 'PyTorch' if row['package'] == 'torch' else row['package'] + name = 'Runtime components' source = '' ui.begin(key, 'Download PyTorch + CUDA', detail=row['filename']) success = False From fc5707fadb0c76211a13b0ad52da0439c1b87f4f Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 12/21] Download guidance: say what a download is and how long it takes before it starts --- freevideo_engine/download_settings.py | 36 ++++++++++ freevideo_engine/image_gen/service.py | 36 +++++++++- freevideo_engine/launcher/Main.qml | 85 ++++++++++++++++++---- freevideo_engine/launcher_session.py | 91 +++++++++++++++++++++--- freevideo_engine/launcher_settings.py | 2 +- freevideo_engine/modern_launcher.py | 8 +++ freevideo_engine/network.py | 4 ++ freevideo_engine/prompt_vlm/installer.py | 81 +++++++++++++++++++++ freevideo_engine/prompt_vlm/service.py | 4 +- freevideo_engine/request_assets.py | 4 +- web/autopilot.js | 15 +++- web/autopilot_chat.css | 2 + web/autopilot_chat.js | 7 +- web/editor.css | 2 + web/editor.js | 6 ++ web/editor_genbar.js | 7 +- web/image_editor.css | 1 + web/image_generation.js | 63 +++++++++++++--- web/request_assets.js | 23 ++++-- 19 files changed, 427 insertions(+), 50 deletions(-) create mode 100644 freevideo_engine/prompt_vlm/installer.py diff --git a/freevideo_engine/download_settings.py b/freevideo_engine/download_settings.py index 23402945..a06fb78e 100644 --- a/freevideo_engine/download_settings.py +++ b/freevideo_engine/download_settings.py @@ -39,6 +39,42 @@ def read(path): return value +SPEED_FILE = 'download-speed.json' +SPEED_MIN_BYTES = 32 * 2**20 + + +def record_speed(root, size, seconds): + """Remember how fast the last large download ran, to say how long the next one takes.""" + if not (size >= SPEED_MIN_BYTES and seconds >= 1.5): + return + path = Path(root) / SPEED_FILE + rate = size / seconds + try: + old = observed_speed(root) + save(path, dict(bytes_per_second=rate if old is None else (old + rate) / 2, updated_at=time.time())) + except OSError: + pass # A missing estimate only hides the time, never stops a download. + + +def observed_speed(root): + """Bytes per second of recent real downloads, else the best connected source of the last speed test.""" + try: + rate = json.loads(io_path(Path(root) / SPEED_FILE).read_text(encoding='utf-8')).get('bytes_per_second') + if isinstance(rate, (int, float)) and math.isfinite(rate) and rate > 0: + return float(rate) + except (OSError, ValueError, AttributeError): + pass + try: + rows = [r for family in read(Path(root) / 'download-settings.json').get('probe', {}).get('sources', {}).values() + for r in family if r.get('ok') and r.get('speed_measured') is not False] + rates = [r['bytes_per_second'] for r in rows + if isinstance(r.get('bytes_per_second'), (int, float)) and math.isfinite(r['bytes_per_second']) + and r['bytes_per_second'] > 0] + return float(max(rates)) if rates else None + except (OSError, ValueError, AttributeError, TypeError): + return None + + def preference(root, source=None, *, proxy_mode=None): if source is not None and source not in CHOICES: raise ValueError('Choose Auto, Hugging Face, HF Mirror or ModelScope') diff --git a/freevideo_engine/image_gen/service.py b/freevideo_engine/image_gen/service.py index c3ac98ca..a2d33bb1 100644 --- a/freevideo_engine/image_gen/service.py +++ b/freevideo_engine/image_gen/service.py @@ -14,6 +14,7 @@ import uuid from . import image_pack +from ..download_settings import observed_speed from .failure import Context, ERRORS, FailureError, MESSAGE_ZH, NO_REPORT, STAGES, classify_exception BODY_LIMIT = 64 * 1024 @@ -113,6 +114,7 @@ def __init__(self, installation, input_directory, *, video_busy=None, idle_secon self.queue_thread = None self.queue_wake = threading.Event() self.finished = OrderedDict() + self.pack_installer = None atexit.register(self.close) @staticmethod @@ -131,7 +133,37 @@ def info(self): checking=not ready and image_pack.checking(machine['model_root']), downloading=not ready and image_pack.downloading(machine['model_root'], engine_root), bytes=sum(row['bytes'] for row in spec['groups']['image']), busy=self.busy(), - max_images=MAX_IMAGES) + max_images=MAX_IMAGES, install=self.pack_state(ready), + bytes_per_second=None if ready else observed_speed(engine_root)) + + def pack_state(self, ready): + """Progress of the download this page started, in both languages for a failure.""" + installer = self.pack_installer + if ready or installer is None: + return None + from ..failure_details import image_pack_failure + value = installer.state() + row = dict(state=value['state'], done=value.get('done', 0), total=value.get('total', 0), + bytes_per_second=round(value.get('bytes_per_second', 0))) + if value['state'] == 'failed': + row.update(reason=value.get('reason'), message_en=image_pack_failure(value.get('reason'))['detail'], + message_zh=image_pack_failure(value.get('reason'), zh=True)['detail']) + return row + + def pack_action(self, action): + """Start or pause the image model download from the page; it yields to a running video.""" + engine_root, machine = self.installation() + root = machine['model_root'] + with self.mutex: + installer = self.pack_installer + if action == 'cancel': + if installer is not None: + installer.cancel() + elif not image_pack.ready(root) and not image_pack.downloading(root, engine_root): + if installer is None or installer.root != root or installer.engine_root != engine_root: + installer = self.pack_installer = image_pack.Installer(root, engine_root=engine_root, busy=self.video_busy) + installer.start('install', background=True) + return self.info() def request(self, value): if not isinstance(value, dict): @@ -859,6 +891,8 @@ async def action(request): result = service.status(value.get('job')) elif name == 'cancel': result = service.cancel(value.get('job')) + elif name in ('pack-install', 'pack-cancel'): + result = service.pack_action(name.split('-')[1]) else: result = service.start(name, value) return web.json_response(result) diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 622cfec7..dee2e399 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -66,6 +66,27 @@ ApplicationWindow { function sourceName(value) { return ({"auto": t("Automatic", "自动选择"), "official": "Hugging Face", "hf-mirror": t("HF Mirror", "HF 镜像"), "modelscope": t("ModelScope", "魔搭")})[value] || value } function number(n) { return typeof n === "number" && isFinite(n) } function fraction(row) { return row && number(row.total) && row.total > 0 && number(row.done) && row.done <= row.total ? row.done / row.total : -1 } + function gb(n) { return n >= 1e9 ? (n / 1e9).toFixed(1) + "\u00a0GB" : Math.max(1, Math.round(n / 1e6)) + "\u00a0MB" } + // "6.0 GB · about 12 min"; the time only when a real download speed has been measured. + function sizeAndTime(n, rate) { + var size = gb(n), speed = rate > 0 ? rate : s.bytes_per_second + if (!(speed > 0) || !(n > 0)) return size + var seconds = n / speed, minutes = Math.ceil(seconds / 60) + return size + " · " + (seconds >= 60 ? t("about " + minutes + " min", "约 " + minutes + " 分钟") : t("under 1 min", "不到 1 分钟")) + } + // The home-page line for the prompt model: the suggestion, then progress, a pause or the reason it stopped. + function promptModelLine() { + var m = s.prompt_model, total = gb(m.total_bytes), done = gb(m.downloaded_bytes) + if (m.busy) { + var rate = m.install.bytes_per_second, left = m.total_bytes - m.install.done + var more = rate > 0 || s.bytes_per_second > 0 ? sizeAndTime(left, rate).split(" · ")[1] : "" + return t("Downloading the prompt model · ", "正在下载提示词模型 · ") + gb(m.install.done) + " / " + total + (more ? " · " + more : "") + } + if (m.failure) return m.failure + if (m.install.state === "cancelled") return t("Prompt model download paused · ", "提示词模型下载已暂停 · ") + done + " / " + total + return t("You can also download the prompt model (", "可以再下载提示词模型（") + sizeAndTime(m.total_bytes).replace(" · ", t(", ", "，")) + + t(") to refine prompts before generating.", "），生成前会自动完善提示词。") + } function bytes(n) { // macOS counts in decimal units like Finder: 1.26 TB, 64.4 GB. if (s.decimal_sizes) { @@ -634,20 +655,37 @@ ApplicationWindow { FText { visible: !!s.offline.detail; text: s.offline.detail + (number(s.offline.total) ? " · " + bytes(s.offline.done) + " / " + bytes(s.offline.total) : ""); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } } FCard { - visible: !manualEnvironment && !offlineSelected; Layout.fillWidth: true; padding: 16 - FSwitch { - objectName: "samplingCaches"; Layout.fillWidth: true - text: t("Prepare all quality levels", "提前下载全部质量档位") - detail: t("Optional sampling caches · ", "可选采样缓存 · ") + bytes(s.sampling_cache_bytes) - checked: !!s.form.sampling_caches; enabled: !s.busy - onToggled: backend.edit("sampling_caches", checked) + objectName: "aheadCard" + visible: !manualEnvironment && !offlineSelected; Layout.fillWidth: true; padding: 16; spacing: 12 + FText { text: t("Download ahead", "提前下载"); font.weight: Font.DemiBold; Layout.fillWidth: true } + RowLayout { + Layout.fillWidth: true; spacing: 14 + FCheck { + objectName: "samplingCaches"; Layout.fillWidth: true + text: t("Sampling data for all quality levels", "全部质量档位的采样数据") + checked: !!s.form.sampling_caches; enabled: !s.busy + onToggled: backend.edit("sampling_caches", checked) + } + FText { objectName: "samplingCachesSize"; color: theme.muted; text: win.sizeAndTime(s.sampling_cache_bytes) } + } + RowLayout { + objectName: "promptModelRow" + visible: !!s.prompt_model && !s.prompt_model.ready + Layout.fillWidth: true; spacing: 14 + FCheck { + objectName: "promptModelSwitch"; Layout.fillWidth: true + text: t("Prompt model", "提示词模型") + checked: !!s.form.prompt_model && s.prompt_model.enough_disk; enabled: !s.busy && s.prompt_model.enough_disk + onToggled: backend.edit("prompt_model", checked) + } + FText { + objectName: "promptModelSize"; color: theme.muted + text: s.prompt_model.enough_disk ? win.sizeAndTime(s.prompt_model.total_bytes) : win.gb(s.prompt_model.total_bytes) + " · " + t("Not enough disk space", "磁盘空间不足") + } } - } - FCard { - objectName: "imageModelCard" - visible: Qt.platform.os !== "osx" && !!s.image_pack && s.image_pack.total_bytes > 0 && s.image_pack.state !== "ready" - Layout.fillWidth: true; padding: 16 RowLayout { + objectName: "imageModelCard" + visible: Qt.platform.os !== "osx" && !!s.image_pack && s.image_pack.total_bytes > 0 && s.image_pack.state !== "ready" Layout.fillWidth: true; spacing: 14 FCheck { objectName: "imageModelSwitch"; Layout.fillWidth: true @@ -657,7 +695,7 @@ ApplicationWindow { } FText { objectName: "imageModelSize"; color: theme.muted - text: s.image_pack.enough_disk ? bytes(s.image_pack.total_bytes).replace(" ", "\u00a0") : t("Not enough disk space", "磁盘空间不足") + text: s.image_pack.enough_disk ? win.sizeAndTime(s.image_pack.total_bytes) : win.gb(s.image_pack.total_bytes) + " · " + t("Not enough disk space", "磁盘空间不足") } } } @@ -922,6 +960,27 @@ ApplicationWindow { fraction: s.image_pack.install.total > 0 ? s.image_pack.install.done / s.image_pack.install.total : -1 } } + FCard { + objectName: "promptModelCard" + visible: !!s.prompt_model && !s.prompt_model.ready && (s.prompt_model.suggest || s.prompt_model.busy + || ["failed", "cancelled"].indexOf(s.prompt_model.install.state) >= 0) + Layout.fillWidth: true; padding: 16; spacing: 10 + FText { + objectName: "promptModelText"; Layout.fillWidth: true; wrapMode: Text.WordWrap + text: win.promptModelLine() + } + FMeter { + objectName: "promptModelProgress"; visible: s.prompt_model.busy; Layout.fillWidth: true; active: visible + fraction: s.prompt_model.install.total > 0 ? s.prompt_model.install.done / s.prompt_model.install.total : -1 + } + Flow { + Layout.fillWidth: true; spacing: 8 + FButton { objectName: "promptModelDownload"; visible: !s.prompt_model.busy + text: s.prompt_model.suggest ? t("Download", "下载") : t("Resume", "继续"); enabled: !s.busy; onClicked: backend.promptModelAction("install") } + FButton { objectName: "promptModelPause"; visible: s.prompt_model.busy; text: t("Pause", "暂停"); onClicked: backend.promptModelAction("cancel") } + FButton { objectName: "promptModelDismiss"; visible: s.prompt_model.suggest; text: t("Don't ask again", "不再提示"); flat: true; onClicked: backend.dismissPromptHint() } + } + } } Item { height: 18 } diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 15362174..378778c5 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -80,7 +80,7 @@ class Session: action_notice = '' pending_update = None pending_check = None - upgrade = image_installer = None + upgrade = image_installer = prompt_installer = None model_switch = None language = 'en' @@ -103,7 +103,7 @@ def __init__(self, source=None, *, controller=None, store=None, updater=None, sm url='http://127.0.0.1:8188', models='', model_dirs=[], model_method='auto', environment_method='auto', separate=False, repair=False, new_comfy=True, offline_runtime='', offline_models=[], imported_archives=[], kept_archives=[], sampling_caches=not saved.get('installation'), - image_model=False) + image_model=False, prompt_model=False, prompt_hint_dismissed=False) self.form['lan_access'] = False self.form.update({k: v for k, v in saved.items() if k in self.form}) if 'environment_method' not in saved and self.form['offline_runtime']: @@ -134,7 +134,7 @@ def __init__(self, source=None, *, controller=None, store=None, updater=None, sm self.tail = Tail() self.log_source = None self.model_groups = [] - self.image_installer = None + self.image_installer = self.prompt_installer = None self.download_export = self.download_error = self.download_error_key = '' self._download_view = None self.browser_attempted = False @@ -203,7 +203,7 @@ def task_busy(self): return bool(self.controller.busy or self.importer.busy or self.cleaner.busy or self.archives_busy or self.closing or self.upgrade and self.upgrade.active or self.model_switch or self.engine_updating - or self.image_installer and self.image_installer.busy) + or self.background_download_busy()) def can_manage(self): return not (self.task_busy() or self.update_intent) @@ -263,7 +263,7 @@ def edit(self, key, value): if self.controller.busy or self.importer.busy or self.cleaner.busy or self.archives_busy: self.explain_busy() return - if key in ('separate', 'repair', 'new_comfy', 'sampling_caches', 'lan_access', 'image_model'): + if key in ('separate', 'repair', 'new_comfy', 'sampling_caches', 'lan_access', 'image_model', 'prompt_model', 'prompt_hint_dismissed'): value = bool(value) if key == 'url': value = local_url(str(value)) @@ -480,7 +480,7 @@ def action(self, name, accepted=False, *, retry=None): return if self.cleaner.busy or self.archives_busy: return - if self.image_installer and self.image_installer.busy and ( + if self.background_download_busy() and ( name == 'setup' or name == 'update-engine' or name == 'retry' and self.retry_kind() != 'launch' or name == 'primary' and self.page != 'launcher'): @@ -625,7 +625,7 @@ def can_repair(self): return (bool(self.selected) and getattr(self.controller, 'fixed_environment', False) is not True and not (self.controller.busy or self.importer.busy or self.cleaner.busy or self.archives_busy or self.closing or self.engine_updating or self.upgrade.active or self.model_switch - or self.image_installer and self.image_installer.busy)) + or self.background_download_busy())) def repair_unavailable(self): """Why Repair is missing: it appears once a task ends, never in an offline @@ -781,6 +781,14 @@ def image_pack_report(self): def start_image_model_after_install(self): """The installer's "also install the image model": once ComfyUI opens, the download runs in the background.""" + if self.form['prompt_model'] and self.installation_engine() and self.controller.state.get('status') == 'open' \ + and not self.controller.busy and not (self.prompt_installer and self.prompt_installer.busy): + from .prompt_vlm import assets + if not assets.ready(self._image_location()[1]): + self.prompt_model_action('install') + if self.prompt_installer and self.prompt_installer.busy or assets.ready(self._image_location()[1]): + self.form['prompt_model'] = False + self.persist() if (not self.form['image_model'] or sys.platform == 'darwin' or not self.installation_engine() or self.controller.state.get('status') != 'open' or self.controller.busy): return @@ -791,6 +799,57 @@ def start_image_model_after_install(self): self.form['image_model'] = False self.persist() + def background_download_busy(self): + return bool(self.image_installer and self.image_installer.busy or self.prompt_installer and self.prompt_installer.busy) + + def prompt_model_action(self, action): + from .prompt_vlm import installer as prompt_installer + if action == 'cancel': + if self.prompt_installer: + self.prompt_installer.cancel() + return + if (self.controller.busy or self.importer.busy or self.cleaner.busy or self.archives_busy + or self.closing or self.engine_updating or self.update_intent or self.upgrade.active or self.model_switch + or self.updater and self.updater.busy or not self.installation_engine()): + return + engine = self._image_location()[1] + if self.prompt_installer and self.prompt_installer.busy: + return + if not (self.prompt_installer and self.prompt_installer.root == engine): + self.prompt_installer = prompt_installer.Installer(engine, busy=self.generating) + self.prompt_installer.start() + + def prompt_model_view(self): + """The prompt model as a row of the download card and as the suggestion on the home page.""" + from .prompt_vlm import assets + spec = assets.catalog() + total = sum(row['bytes'] for row in spec['files']) + engine = self._image_location()[1] if self.installation_engine() else None + ready = bool(engine) and assets.ready(engine) + installer = self.prompt_installer + active = installer.state() if installer and engine and installer.root == engine else {} + busy = bool(active and installer.busy) + try: + folder = Path(self.form['destination'] if self.form['new_comfy'] else self.form['comfy']) + if engine: + folder = engine + enough = disk_space.free_bytes(folder.expanduser()) >= total + 2 * 2**30 + except Exception: # A page reading the disk must never fail to draw. + enough = True + failure = '' + if active.get('state') == 'failed': + failure = {'disk_space': self.t('Not enough disk space.', '磁盘空间不足。'), + 'busy': self.t('Another download of this model is running.', '这个模型正在被另一处下载。')}.get( + active.get('reason'), self.t('The download stopped. Resume to continue.', '下载中断了，点继续可接着下载。')) + return dict(ready=ready, total_bytes=total, enough_disk=enough, busy=busy, install=active, failure=failure, + downloaded_bytes=active.get('done', 0), title=self.t(*NAMES['text']), + suggest=bool(engine) and not ready and not busy and enough and not self.form['prompt_hint_dismissed'] + and active.get('state') not in ('failed', 'cancelled')) + + def dismiss_prompt_hint(self): + self.form['prompt_hint_dismissed'] = True + self.persist() + def upgrade_model(self): """The user's one consent: download int8 beside the current model, switch, then release the old one. @@ -1142,12 +1201,15 @@ def update(self, token='', source='launcher'): return if self._open_release_page(): return - if (self.image_installer and self.image_installer.busy and not self.controller.busy + if (self.background_download_busy() and not self.controller.busy and not self.importer.busy and not self.cleaner.busy and not self.archives_busy and not self.closing and not self.upgrade.active and not self.model_switch): # An explicit update goes first. The image model download is the long, quiet kind and # keeps what it has; it carries on from there when the user continues it. - self.image_installer.cancel() + if self.image_installer: + self.image_installer.cancel() + if self.prompt_installer: + self.prompt_installer.cancel() if self.task_busy() or self.updater and self.updater.busy: self.pending_update = (token, source) self.action_notice = 'update_queued' @@ -1495,6 +1557,10 @@ def close(self): self.image_installer.cancel() if self.image_installer.thread: self.image_installer.thread.join(5) + if self.prompt_installer: + self.prompt_installer.cancel() + if self.prompt_installer.thread: + self.prompt_installer.thread.join(5) self.controller.close() from .launcher_bridge import remove remove(self.bridge) @@ -2045,6 +2111,12 @@ def snapshot(self): rate='%.1f MiB/s' % (item['bytes_per_second']/2**20) if item.get('bytes_per_second') else '')) image_pack = self.image_pack_view() image_pack['enough_disk'] = self.image_disk_fits(image_pack['total_bytes']) + prompt_model = self.prompt_model_view() + try: + from .download_settings import observed_speed + speed = observed_speed(self.engine_root()) or 0 + except Exception: + speed = 0 if image_pack['visible'] and 'image' not in by_id: models.append(dict(id='image', title=image_pack['title'], state=image_pack['state'], detail=image_pack['detail'], done=image_pack['downloaded_bytes'], @@ -2156,6 +2228,7 @@ def snapshot(self): if inspection_read_error: download_state.update(error=self.t(*DOWNLOAD_COPY['error_read']), needs_check=True) return plain_data(dict(download_list=download_state, sampling_cache_bytes=total_bytes(), version=__version__, image_pack=image_pack, + prompt_model=prompt_model, bytes_per_second=speed, disk=disk, decimal_sizes=sys.platform == 'darwin', zh=self.language.startswith('zh'), form=dict(self.form), page=self.page, status=row.get('status', 'idle'), busy=busy, working=working, action_notice=self.t(*ACTION_COPY[self.action_notice]) if self.action_notice else '', diff --git a/freevideo_engine/launcher_settings.py b/freevideo_engine/launcher_settings.py index 4b80a485..12314a70 100644 --- a/freevideo_engine/launcher_settings.py +++ b/freevideo_engine/launcher_settings.py @@ -38,7 +38,7 @@ def sanitize(value): extra = value.get('extra_model_dirs', []) if isinstance(value, dict) else [] value = plain_data(value) if isinstance(value, dict) else {} result = {k: value[k] for k in TEXT if isinstance(value.get(k), str)} - for key in ('separate', 'new_comfy', 'sampling_caches', 'lan_access', 'image_model'): + for key in ('separate', 'new_comfy', 'sampling_caches', 'lan_access', 'image_model', 'prompt_model', 'prompt_hint_dismissed'): if type(value.get(key)) is bool: result[key] = value[key] if value.get('model_method') in ('auto', 'manual', 'reuse'): diff --git a/freevideo_engine/modern_launcher.py b/freevideo_engine/modern_launcher.py index 67fc54c5..e3890af8 100644 --- a/freevideo_engine/modern_launcher.py +++ b/freevideo_engine/modern_launcher.py @@ -165,6 +165,14 @@ def upgradeModel(self): def imagePackAction(self, action): self.invoke(session.image_pack_action, action) + @Slot(str) + def promptModelAction(self, action): + self.invoke(session.prompt_model_action, action) + + @Slot() + def dismissPromptHint(self): + self.invoke(session.dismiss_prompt_hint) + @Slot() def cancelModelUpgrade(self): self.invoke(session.cancel_model_upgrade) diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index e69863a1..d6e81fc4 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -1018,6 +1018,10 @@ def remember_source(): if matches(partial, name): from .file_ops import publish publish(partial, path) + if network.get('download_settings_path'): + from .download_settings import record_speed + record_speed(Path(network['download_settings_path']).parent, + max(0, current_size - offset), time.monotonic() - started) source_health(network, category, name, True) route_health(network, category, name, route) event(network, category=category, source=name, route=route, file=path.name, action='verified', method='curl', bytes=path.stat().st_size) diff --git a/freevideo_engine/prompt_vlm/installer.py b/freevideo_engine/prompt_vlm/installer.py new file mode 100644 index 00000000..f9fd87b0 --- /dev/null +++ b/freevideo_engine/prompt_vlm/installer.py @@ -0,0 +1,81 @@ +"""The launcher's own download of the prompt model: one background thread, pausable, resumable.""" +from collections import deque +from pathlib import Path +import threading +import time + +from ..locking import runtime_lock +from . import assets + +SPEED_WINDOW_SECONDS = 20 + + +class Installer: + """state: idle, running, cancelled, failed (reason), done; done/total in bytes.""" + + def __init__(self, root, *, busy=None): + self.root = Path(root) + self.generation_busy = busy + self.lock = threading.Lock() + self.stop = threading.Event() + self.thread = None + self.samples = deque() + self.value = dict(state='idle', done=0, total=0, reason=None) + + @property + def busy(self): + return self.thread is not None and self.thread.is_alive() + + def state(self): + with self.lock: + value = dict(self.value) + now = time.monotonic() + while len(self.samples) > 1 and self.samples[1][0] <= now - SPEED_WINDOW_SECONDS: + self.samples.popleft() + rate = 0 + if value['state'] == 'running' and self.samples: + start, amount = self.samples[0] + if now > start: + rate = max(0, value['done'] - amount) / (now - start) + value['bytes_per_second'] = round(rate) + return value + + def start(self): + with self.lock: + if self.busy: + return + self.stop = threading.Event() + self.samples.clear() + self.value = dict(state='running', done=0, total=0, reason=None) + self.thread = threading.Thread(target=self._run, args=(self.stop,), daemon=True) + self.thread.start() + + def cancel(self): + self.stop.set() + + def _progress(self, row): + with self.lock: + if row.get('phase') == 'download': + self.value.update(done=row['done'], total=row['total']) + self.samples.append((time.monotonic(), row['done'])) + + def _run(self, stop): + def check(): + if stop.is_set(): + raise InterruptedError('cancelled') + try: + # The engine's own install and this one never write the same partial files. + with runtime_lock(self.root / 'prompt-vlm-install.lock', inherit=False): + assets.install(self.root, self._progress, check) + with self.lock: + self.value.update(state='done', reason=None) + except InterruptedError: + with self.lock: + self.value.update(state='cancelled', reason=None) + except BlockingIOError: + with self.lock: + self.value.update(state='failed', reason='busy') + except Exception as error: + known = {'disk_space', 'ram_space'} + with self.lock: + self.value.update(state='failed', reason=str(error) if str(error) in known else 'download_failed') diff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.py index cad2b225..d82ff367 100644 --- a/freevideo_engine/prompt_vlm/service.py +++ b/freevideo_engine/prompt_vlm/service.py @@ -384,7 +384,9 @@ def work(self, root, machine, action, value): with runtime_lock(Path(root) / 'engine.lock', inherit=False) as descriptor: self.check() if action == 'install': - assets.install(root, self.progress, self.check) + # The launcher's background download writes the same partial files under this lock. + with runtime_lock(Path(root) / 'prompt-vlm-install.lock', inherit=False): + assets.install(root, self.progress, self.check) self.check() self.progress(dict(phase='complete')) else: diff --git a/freevideo_engine/request_assets.py b/freevideo_engine/request_assets.py index 6c9893ac..624695b3 100644 --- a/freevideo_engine/request_assets.py +++ b/freevideo_engine/request_assets.py @@ -87,4 +87,6 @@ def inspect(root, machine, request, *, lora_names=None): if absent: items.append(dict(id='lora', kind='lora', files=len(absent), bytes=0, source='', action='manual', names=absent[:8])) - return dict(items=items, bytes=sum(item['bytes'] for item in items), ready=not items) + from .download_settings import observed_speed + return dict(items=items, bytes=sum(item['bytes'] for item in items), ready=not items, + bytes_per_second=observed_speed(root)) diff --git a/web/autopilot.js b/web/autopilot.js index d64dff0b..4994d7ae 100644 --- a/web/autopilot.js +++ b/web/autopilot.js @@ -2,6 +2,7 @@ import { api } from '../../scripts/api.js'; import { agentConnectCard, connectionLabel } from './agent_connect.js'; import { about, agentChat, jobName } from './autopilot_chat.js'; +import { createRequestAssets } from './request_assets.js'; import { openEditorView } from './editor_loader.js'; import { closeDialog } from './motion.js'; import { productVersion } from './updates.js'; @@ -46,7 +47,15 @@ export function openAutopilotView(node, {studio = null} = {}) { const card = agentConnectCard(t, api, () => inSight() && !card.element.hidden); // Leaving for another view waits until this page has closed, so the view underneath is never covered twice. const closed = () => dialog.open ? new Promise(done => { dialog.addEventListener('close', done, {once: true}); closeDialog(dialog); }) : null; - const chat = agentChat(t, api, {shown: inSight, productVersion, + // The next video at the studio's current settings; the agent may choose others, so this is only what is missing now. + const assets = createRequestAssets({t, api, request: () => { + const value = Object.fromEntries((node.widgets || []).filter(widget => widget.name).map(widget => [widget.name, widget.value])); + if (!(value.width > 0 && value.height > 0 && value.seconds > 0)) return null; + return new URLSearchParams({width: value.width, height: value.height, seconds: value.seconds, task: 't2va', + base_steps: value.base_steps ?? 8, refine_steps: value.refine_steps ?? 3, two_pass: value.two_pass === false ? '0' : '1', + enhance: node.properties?.freevideo_prompt_versions?.enabled ? '1' : '0'}); + }}); + const chat = agentChat(t, api, {shown: inSight, productVersion, hint: assets, // Creations continues the work on the editor timeline; this page gives way first. open: async job => (await import('./library.js')).openLibrary(t, {selectedWork: job.work, back: t('Back to chat', '返回对话'), extend: async (record, options) => { await closed(); return openExtendEditor(record, node, {studio, ...options}); }}), @@ -90,11 +99,11 @@ export function openAutopilotView(node, {studio = null} = {}) { dialog.append(header, body); dialog.addEventListener('cancel', event => { event.preventDefault(); closeDialog(dialog); }); dialog.onclose = () => { - card.stop(); chat.stop(); dialog.remove(); + card.stop(); chat.stop(); assets.dispose(); dialog.remove(); if (current === dialog) current = null; next(); }; - current = dialog; document.body.append(dialog); dialog.showModal(); viewChanged('autopilot', node); show(); + current = dialog; document.body.append(dialog); dialog.showModal(); viewChanged('autopilot', node); show(); assets.refresh(true); navigation.querySelector('[data-view=autopilot]').focus(); return dialog; } diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index dbdb1cf4..e7d99bdf 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -144,3 +144,5 @@ .fv-chat-me>.fv-chat-actions{margin-top:6px;flex-wrap:wrap;justify-content:flex-end} .fv-chat-me .fv-chat-error{flex:1;min-width:0} .fv-chat-keep{white-space:nowrap} +.fv-chat-main > .fv-request-assets{font-size:12px;line-height:1.5;margin:0 0 6px;padding:0 4px} +.fv-chat-main > .fv-request-assets[hidden]{display:none} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 5bbe650c..96f5191d 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -7,6 +7,7 @@ import { estimatedFraction, remainingAbove, remainingShown, createDiagnosticRepo import {createErrorReport} from './error_panel.js'; import {canDeleteConversation, conversationMenu, emptyConversations, hasConversations, mergeConversations, removeConversation, runningElsewhere, selectConversation} from './autopilot_conversations.js'; import {renderImageFailure} from './image_generation.js'; +import {about} from './request_assets.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./autopilot_chat.css', import.meta.url).href; document.head.append(css); @@ -41,7 +42,7 @@ function failureReason(value, t) { return t(error?.message || 'Did not finish', reason || GENERIC_FAILURE_ZH); } -export const about = (seconds, t) => seconds >= 60 ? t(`About ${Math.ceil(seconds / 60)} min`, `约 ${Math.ceil(seconds / 60)} 分钟`) : t('Under 1 min', '不到 1 分钟'); +export {about}; export function clock(seconds) { const s = Math.max(0, Math.round(seconds)); return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; } export const jobName = (job, t) => job.label || (job.kind === 'image' ? t('Picture', '图片') : t('Video', '视频')); export const jobFraction = job => Math.max(0, Math.min(1, Number(job.progress?.fraction) || 0)); @@ -232,7 +233,7 @@ function imageViewer(t, src, name, opener, failed) { // open(job) shows a finished video in Creations, edit(job) takes it to the editing view. shown() says whether the // user can see the conversation; the page calls sync() whenever that may have changed. changed(view) follows every // answer: {known, agent, current, active, any}. -export function agentChat(t, api, {open, edit, shown = () => true, changed = () => {}, interval = 1500, productVersion = () => null} = {}) { +export function agentChat(t, api, {open, edit, shown = () => true, changed = () => {}, interval = 1500, productVersion = () => null, hint = null} = {}) { const element = el('section', 'fv-chat'), main = el('div', 'fv-chat-main'), scroller = el('div', 'fv-chat-scroll'), log = el('ol', 'fv-chat-log'); log.setAttribute('role', 'log'); log.setAttribute('aria-label', t('Conversation', '对话')); const down = press('fv-chat-down', null, () => { scroller.scrollTo({top: scroller.scrollHeight, behavior: 'smooth'}); }); @@ -242,6 +243,8 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () sendButton.type = 'submit'; sendButton.setAttribute('aria-label', t('Send', '发送')); sendButton.append(glyph('M12 19V5M5 12l7-7 7 7')); form.append(input, sendButton); scroller.append(log); main.append(scroller, down, form); element.append(main); const notice = el('p', 'fv-chat-notice'); notice.hidden = true; notice.setAttribute('role', 'status'); main.insertBefore(notice, form); + // What the next video would download first, the studio's line, above the box the request is typed in. + if (hint) main.insertBefore(hint.element, form); // A quoted button name stays on one line, so “新对话” never wraps in the middle. function quoted(node, text) { const parts = String(text).split(/(“[^”]{1,12}”|"[^"]{1,24}")/); diff --git a/web/editor.css b/web/editor.css index 9b17abda..7d2cfb2e 100644 --- a/web/editor.css +++ b/web/editor.css @@ -75,6 +75,8 @@ .fv-ed-genbar[data-mode=running]>button:not(.fv-ed-generate){order:2} .fv-ed-gen-note{font-size:13px;color:var(--fv-ink);flex:0 1 auto;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} .fv-ed-gen-note[hidden]{display:none} +.fv-ed-genbar .fv-request-assets{order:-2;flex:1 1 100%;min-width:0;font-size:12px;line-height:1.5} +.fv-ed-genbar .fv-request-assets[hidden]{display:none} .fv-editor:has(.fv-ed-gen-failure:not([hidden])){--fv-ed-generation-row:max-content} .fv-ed-genbar:has(.fv-ed-gen-failure:not([hidden])){flex-wrap:wrap;padding-top:10px;padding-bottom:10px} .fv-editor .fv-ed-gen-failure{display:block;flex:1 0 100%;min-width:0} diff --git a/web/editor.js b/web/editor.js index dafaabe0..8329b0ae 100644 --- a/web/editor.js +++ b/web/editor.js @@ -804,6 +804,12 @@ export function openEditor({api, app, from = null, panel = {}, components: given references: panel.references ? {source: panel.references, notify, size: () => capability.generationSize || generationSize(state.record), visible: () => store.generationAvailable && actions.supportedSelection() ? partStart() : null} : null, setFirstFrame: value => { state.apply(t('First-frame reference', '首帧参考'), record => model.setFlag(record, 'first_frame_reference', value)); }, + // The sampling data and models the next generated part would still download first. + assets: {api, query: () => { + const size = capability.generationSize || generationSize(state.record), set = state.record.settings || {}; + return new URLSearchParams({width: size.width, height: size.height, seconds: 5, task: 't2va', enhance: '0', + base_steps: set.base_steps ?? 8, refine_steps: set.refine_steps ?? 3, two_pass: set.two_pass === false ? '0' : '1'}); + }}, previewRange: selection => genbar?.preview(selection), async estimate(selection) { const range = selectionRange(selection); if (!range) return null; diff --git a/web/editor_genbar.js b/web/editor_genbar.js index a953c54d..3802c844 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -2,6 +2,7 @@ import {createFirstFrameSwitch} from './editor_first_frame.js'; import {createEditorReferences} from './editor_references.js'; import {createExpandButton} from './editor_expand.js'; +import {createRequestAssets} from './request_assets.js'; const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60 ? t(`about ${Math.max(1, Math.ceil(seconds))} s`, `约 ${Math.max(1, Math.ceil(seconds))} 秒`) @@ -47,6 +48,9 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { input.dataset.fvGuide = 'editor-describe'; generate.dataset.fvGuide = 'editor-generate'; for (const button of [generate, cancel, resume]) button.type = 'button'; host.append(hint, note, description, generate, cancel, resume); + // What this part will download first, above the bar like the studio's line above Generate. + const assets = actions.assets ? createRequestAssets({t, api: actions.assets.api, request: () => actions.assets.query()}) : null; + if (assets) host.prepend(assets.element); const firstFrame = createFirstFrameSwitch(host, {state, t, actions}); description.after(firstFrame.element); const failure = el('div', 'fv-ed-gen-failure fv-result-links'), failureBar = el('div', 'fv-ed-failure-actions fv-actions'); @@ -149,6 +153,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { resizeDescription(); input.dispatchEvent(new Event('change')); generate.title = ''; + if (assets) { if (selected && !busy) assets.refresh(); else assets.element.hidden = true; } if (selected && !busy) { const key = JSON.stringify(selection); if (estimateCache?.key === key) generate.textContent += estimateCache.label; @@ -219,7 +224,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { }); render(); return {element: host, say, downloadFailure, getText() { expand.commit(); return input.value; }, update(value) { status = value; render(); }, preview(selection) { previewSelection = selection; render(); }, setText(value) { input.value = value || ''; state.setRangeText(input.value); render(); }, - dispose() { disposed = true; ++serial; clearTimeout(noteTimer); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); }}; + dispose() { disposed = true; ++serial; clearTimeout(noteTimer); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); assets?.dispose(); }}; } // A missing optional module leaves the editor's existing progress in place. diff --git a/web/image_editor.css b/web/image_editor.css index 6de00b22..40d9d1a1 100644 --- a/web/image_editor.css +++ b/web/image_editor.css @@ -270,6 +270,7 @@ button.fv-ie-size:hover{background:var(--fv-hover);color:var(--fv-ink)} .fv-ie-generation[data-running=true]>.fv-ie-button{grid-column:3} .fv-ie-generation[data-running=true]>.fv-ie-gen-note{grid-column:2;grid-row:2} .fv-ie-generation[data-message=true]>.fv-ie-gen-note{max-width:none} + .fv-ie-generation[data-pack=true]>.fv-ie-gen-note{text-align:right} .fv-ie-options.fv-ie-generation[data-empty=true]{grid-template-columns:minmax(0,1fr) auto} .fv-ie-generation[data-empty=true]>.fv-ie-seg{grid-column:1;grid-row:1} .fv-ie-generation[data-empty=true]>#fv-ie-gen-ratio{grid-column:2;grid-row:1} diff --git a/web/image_generation.js b/web/image_generation.js index aaafd256..96bb73aa 100644 --- a/web/image_generation.js +++ b/web/image_generation.js @@ -3,6 +3,7 @@ import * as commands from './image_commands.js'; import {orientMatrix, rasterKinds, scaleSelection} from './image_editor_engine.js'; import {saveReport, reportIssue, showReportTip} from './report_issue.js'; +import {about} from './request_assets.js'; export const IMAGE_GEN_PROTOCOL = Object.freeze({ root: '/freevideo/image-gen', header: 'X-FreeVideo-Image', upload: '/upload/image', @@ -14,6 +15,7 @@ export const IMAGE_GEN_PROTOCOL = Object.freeze({ }); export const IMAGE_GEN_POLL_MS = 500; export const IMAGE_GEN_INFO_POLL_MS = 10000; +export const IMAGE_PACK_POLL_MS = 2000; export const BLANK_CANVAS_RATIOS = Object.freeze(['16:9', '9:16', '1:1', '4:3', '3:4']); export function imageGenerationSupported(platform = globalThis.navigator?.userAgentData?.platform || globalThis.navigator?.platform || '') { return !/Mac|iPhone|iPad/i.test(platform); @@ -82,6 +84,9 @@ export function createImageGenClient(fetchImpl = globalThis.fetch.bind(globalThi ...(phase === 'cancelled' ? {error: 'cancelled'} : {})}; }, cancel: job => post('cancel', {[p.job]: job}), + // The image models are fetched by this page's own engine; the reply is the new info. + async packInstall() { const value = await post('pack-install', {}); return {...value, installed: value.installed ?? value[p.installed] ?? false}; }, + async packCancel() { const value = await post('pack-cancel', {}); return {...value, installed: value.installed ?? value[p.installed] ?? false}; }, // The prompts and sizes of a new character's five pictures (the Agent's cards use the same ones). castViews: async description => (await post('cast-views', {description})).views, async upload(file) { @@ -106,6 +111,28 @@ export function imagePackMessage(info, t) { .replace(/ (?=[KMGT]i?B$)/, '\u00a0'); return t(`Download "${name}" in the launcher first, ${size}.`, `先在启动器里下载“${name}”，${size}。`); } +// What the page offers while the image models are missing: one button, one muted line, and the bar once loading. +export function packView(info, t) { + const install = info?.install || {}, own = install.state === 'running' || install.state === 'paused_for_generation'; + // Another process (the launcher) is fetching or re-checking them: nothing to press here. + if (!own && (info?.[IMAGE_GEN_PROTOCOL.downloading] || info?.[IMAGE_GEN_PROTOCOL.checking])) return null; + const total = install.total || info?.[IMAGE_GEN_PROTOCOL.bytes] || 0; + const gb = bytes => (bytes / 1e9).toFixed(1); + const size = total > 0 ? gb(total) + '\u00a0GB' : '17.3\u00a0GB'; + const rate = install.bytes_per_second || info?.bytes_per_second || 0; + const left = Math.max(0, total - (install.done || 0)); + const time = rate > 0 && left > 0 ? ' · ' + about(left / rate, t).replace(/^About/, 'about').replaceAll(' ', '\u00a0') : ''; + if (own) { + const waiting = install.state === 'paused_for_generation'; + return {loading: true, button: t('Pause', '暂停'), fraction: total > 0 ? Math.min(1, (install.done || 0) / total) : 0, + note: waiting ? t('Paused while a video generates', '视频生成中，已暂停') + : t(`Downloading · ${gb(install.done || 0)}\u00a0GB / ${size}${time}`, `正在下载 · ${gb(install.done || 0)}\u00a0GB / ${size}${time}`)}; + } + const resume = install.state === 'failed' || install.state === 'cancelled' || install.done > 0; + const reason = install.state === 'failed' ? t(install.message_en, install.message_zh) : ''; + return {loading: false, fraction: null, reason: !!reason, button: resume ? t('Resume download', '继续下载') : t('Download image generation models', '下载图像生成模型'), + note: reason || (resume ? t(`${gb(install.done || 0)}\u00a0GB / ${size}${time}`, `${gb(install.done || 0)}\u00a0GB / ${size}${time}`) : size + time)}; +} export function imageGenError(code, info, t) { const reasons = { no_memory: ['Not enough GPU memory, even at the smallest size. Close other apps using the GPU, then try again.', @@ -319,10 +346,11 @@ export function createImageGenSession({client, prepare, apply, onChange = () => function scheduleInfo() { unschedule(infoTimer); infoTimer = null; if (disposed || state.info?.installed) return; + const loading = ['running', 'paused_for_generation'].includes(state.info?.install?.state); infoTimer = schedule(() => { infoTimer = null; if (visible()) void check(); else scheduleInfo(); - }, IMAGE_GEN_INFO_POLL_MS); + }, loading ? IMAGE_PACK_POLL_MS : IMAGE_GEN_INFO_POLL_MS); } async function check() { if (checking || disposed) return checking; @@ -401,7 +429,13 @@ export function createImageGenSession({client, prepare, apply, onChange = () => emit({running: false, job: null, stage: '', fraction: 0, error: '', report_id: null, waiting: '', fitted: null}); return job ? cancelJob(job) : Promise.resolve(); } - return {check, start, resume, cancel, clearError: () => emit({error: '', report_id: null}), state: () => state, + // Start or pause the download of the image models; the reply is the new state, then polling follows it. + async function pack(action) { + try { const info = await (action === 'cancel' ? client.packCancel() : client.packInstall()); if (!disposed) emit({info, error: '', report_id: null}); } + catch (error) { if (!disposed) emit(failed(error)); } + finally { if (!disposed) scheduleInfo(); } + } + return {check, start, resume, cancel, pack, clearError: () => emit({error: '', report_id: null}), state: () => state, dispose() { disposed = true; unschedule(infoTimer); infoTimer = null; return cancel(); }}; } @@ -555,11 +589,12 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection renderReferences(refs, disabled); input.placeholder = mode === 'edit' ? t('What to change', '要改什么') : t('Describe the picture', '描述画面'); input.setAttribute('aria-label', input.placeholder); - action.textContent = state.running ? t('Stop', '停止') : t('Generate', '生成'); + const pack = missing ? packView(state.info, t) : null; + action.textContent = pack ? pack.button : state.running ? t('Stop', '停止') : t('Generate', '生成'); action.classList.toggle('fv-ie-primary', !state.running); - action.setAttribute('aria-disabled', String(!state.running && (disabled || !prompt.trim()))); - action.title = !state.running && !prompt.trim() ? t('Write a prompt first', '请先写提示词') : ''; - action.hidden = !!missing; + action.setAttribute('aria-disabled', String(!pack && !state.running && (disabled || !prompt.trim()))); + action.title = !pack && !state.running && !prompt.trim() ? t('Enter a prompt first', '请先填写提示词') : ''; + action.hidden = !!missing && !pack; const percent = Math.floor(state.fraction * 100) + '%'; if (state.error && !promptRequired) { const failureKey = `${state.error}:${state.report_id || ''}:${state.stage || ''}`; @@ -569,8 +604,9 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection } } else { mounted.failureKey = null; - note.textContent = missing ? imagePackUnavailableMessage(state.info, t) - : promptRequired ? t('Write a prompt first.', '请先写提示词。') + note.textContent = pack ? pack.note + : missing ? imagePackUnavailableMessage(state.info, t) + : promptRequired ? t('Enter a prompt first.', '请先填写提示词。') : state.running && state.waiting ? t(...(QUEUE_LABELS[state.waiting] || QUEUE_LABELS.image)) : state.running && state.fitted ? t(`Not enough GPU memory. Retrying at ${state.fitted.width}\u00a0×\u00a0${state.fitted.height}`, `显存不足，改用 ${state.fitted.width}\u00a0×\u00a0${state.fitted.height} 重试`) + ' · ' + percent @@ -581,8 +617,11 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection bar.dataset.running = String(state.running); bar.dataset.message = String(!!missing || promptRequired || !!state.error || !!note.textContent); bar.dataset.failed = String(!!state.error && !promptRequired); - meter.hidden = !state.running || !!state.waiting; meter.setAttribute('aria-valuenow', String(Math.floor(state.fraction * 100))); - fill.style.width = state.fraction * 100 + '%'; + bar.dataset.pack = String(!!pack && !pack.reason && !state.error); + const shown = pack ? pack.fraction : state.fraction; + meter.hidden = pack ? pack.fraction == null : !state.running || !!state.waiting; + meter.setAttribute('aria-valuenow', String(Math.floor(shown * 100))); + fill.style.width = shown * 100 + '%'; } function mount(bar) { bar.classList.add('fv-ie-generation'); @@ -608,7 +647,9 @@ export function createImageGeneration({client, renderer, t, getDoc, getSelection box.append(input); field.append(refs, box); const note = el('div', '', 'fv-ie-opt-note fv-ie-gen-note'); note.setAttribute('role', 'status'); const action = button(t('Generate', '生成'), null, () => { - if (session.state().running) void session.cancel(); else void start(); + const now = session.state(), pack = !now.running && now.info && !now.info.installed ? packView(now.info, t) : null; + if (pack) void session.pack(pack.loading ? 'cancel' : 'install'); + else if (now.running) void session.cancel(); else void start(); }, 'fv-ie-button fv-ie-primary'); const meter = el('div', null, 'fv-ie-gen-meter'), fill = el('i'); meter.append(fill); meter.setAttribute('role', 'progressbar'); meter.setAttribute('aria-label', t('Generate with AI', 'AI 生成')); diff --git a/web/request_assets.js b/web/request_assets.js index 17689f5b..31eec2c4 100644 --- a/web/request_assets.js +++ b/web/request_assets.js @@ -1,4 +1,11 @@ // One muted line above Generate: what this request will download first. +export const about = (seconds, t) => seconds >= 60 ? t(`About ${Math.ceil(seconds / 60)} min`, `约 ${Math.ceil(seconds / 60)} 分钟`) : t('Under 1 min', '不到 1 分钟'); +// ", about 5 min" after a size, only when a real download speed is known. +export function timeAfter(bytes, rate, t) { + if (!(bytes > 0) || !(rate > 0)) return ''; + const text = about(bytes / rate, t).replaceAll(' ', '\u00a0'); // "about 2 min" never breaks across lines + return t(', ' + text.charAt(0).toLowerCase() + text.slice(1), '，' + text); +} // LoRA file names this request would load, through the whole `previous` chain. export function enabledLoras(start, {upstream, widget, value}) { const names = []; @@ -33,8 +40,8 @@ export function createRequestAssets({t, api, request, onError = () => {}}) { element.hidden = true; element.setAttribute('aria-live', 'polite'); let timer = 0, controller = null, key = '', disposed = false; - const size = bytes => bytes >= 2 ** 30 ? (bytes / 2 ** 30).toFixed(1) + ' GB' : Math.max(1, Math.round(bytes / 2 ** 20)) + ' MB'; - function lines(items) { + const size = bytes => bytes >= 1e9 ? (bytes / 1e9).toFixed(1) + ' GB' : Math.max(1, Math.round(bytes / 1e6)) + ' MB'; + function lines(items, rate) { const rows = []; // One line names what Generate will fetch first: the two-pass model, then the sampling data. const names = {upscaler: t('two-pass model', '二次采样模型'), sampling: t('sampling data', '采样数据')}; @@ -46,10 +53,12 @@ export function createRequestAssets({t, api, request, onError = () => {}}) { } const parts = [['upscaler', names.upscaler], ['sampling', names.sampling], ['other', t('other model files', '其他模型文件')]] .filter(([kind]) => bytes[kind]).map(([kind, name]) => `${name} ${size(bytes[kind])}`); - if (parts.length) rows.push(t(`Downloads before generating: ${parts.join(', ')}`, `生成前会先下载：${parts.join('、')}`)); + const wait = timeAfter(Object.values(bytes).reduce((sum, n) => sum + n, 0), rate, t); + if (parts.length) rows.push(t(`Downloads before generating: ${parts.join(', ')}${wait}`, `生成前会先下载：${parts.join('、')}${wait}`)); for (const item of items) { if (item.kind === 'prompt-vlm') { - rows.push(t(`First prompt rewrite downloads ${size(item.bytes)}`, `首次改写提示词会先下载 ${size(item.bytes)}`)); + const wait = timeAfter(item.bytes, rate, t); + rows.push(t(`First prompt rewrite downloads ${size(item.bytes)}${wait}`, `首次改写提示词会先下载 ${size(item.bytes)}${wait}`)); } else if (item.kind === 'lora') { const names = (item.names || []).map(short); rows.push(t(`LoRA not found: ${names.join(', ')}`, `找不到 LoRA：${names.join('、')}`)); @@ -57,8 +66,8 @@ export function createRequestAssets({t, api, request, onError = () => {}}) { } return rows; } - function show(items) { - const rows = lines(items); + function show(items, rate = 0) { + const rows = lines(items, rate); element.replaceChildren(...rows.map(text => Object.assign(document.createElement('div'), {textContent: text}))); element.hidden = rows.length === 0; } @@ -76,7 +85,7 @@ export function createRequestAssets({t, api, request, onError = () => {}}) { if (!reply.ok) return; const result = await reply.json(); if (disposed || current.signal.aborted || key !== next) return; - show(Array.isArray(result.items) ? result.items : []); + show(Array.isArray(result.items) ? result.items : [], Number(result.bytes_per_second) || 0); } catch (error) { if (error?.name !== 'AbortError') onError(error); } // Advisory; never blocks Generate. }, 300); } From 8c559317d4d3afbac11908e7a7fed21d3b4ac106 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Sat, 17 Oct 2026 12:28:14 +0000 Subject: [PATCH 13/21] Guide reference images and longer videos in the creation panel - Guide: settle wording, state part limits per way of making longer videos - Guide: one 5-part limit for both ways of making longer videos; fix reference marker name - Long videos: a device that cannot continue a video refuses lengths over one block up front - Long videos: the no-parts line has the same spacing as the hint it replaces --- ...77\347\224\250\350\257\264\346\230\216.md" | 53 +++++++++++++++++++ freevideo_engine/comfy_chain.py | 8 ++- web/first_run_guide.js | 17 +++++- web/setup.css | 1 + web/studio.css | 1 + web/studio.js | 21 ++++++-- web/studio_length.js | 16 +++++- 7 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 "docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" diff --git "a/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 00000000..3e00fb3d --- /dev/null +++ "b/docs/zh-CN/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,53 @@ +# 使用说明 + +这一页说明创作面板里三件常被问到的事：用参考图生成视频、首帧尾帧和参考图的区别、做更长的视频。 + +## 参考图生成视频 + +1. 打开创作面板，展开“素材”。 +2. 点“添加”，选择图片。上传的每张图片可以指定用途：首帧、尾帧或参考。要让画面里出现这张图里的人物、物品或场景，选“参考”。 +3. 在描述里输入 `@`，从列表里选一张素材，描述里会出现 `` 这样的标记。视频和音频素材对应 `