diff --git a/MANIFEST.in b/MANIFEST.in index c430037e..f4dd048c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -68,5 +68,11 @@ include web/assets/tour/c9-s1.webp include web/assets/tour/c10-person.webp include web/assets/tour/c10-s1.mp4 include web/assets/tour/c10-s1.webp +include web/assets/tour/c11-s1.mp4 +include web/assets/tour/c11-s1.webp +include web/assets/tour/c12-s1.mp4 +include web/assets/tour/c12-s1.webp +include web/assets/tour/c13-s1.mp4 +include web/assets/tour/c13-s1.webp include web/assets/tour/c10-source.webp include web/assets/tour/fox-sheet.webp diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 64906f63..647e47b8 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -17,6 +17,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th | [FreeToken](https://github.com/FlashML-org/FreeToken) prompt VLM operators, [Qwen3-VL-4B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) optional weights | [Apache-2.0](freevideo_engine/licenses/FreeToken-Apache-2.0.txt) | | [Qwen-Image-2.1](https://huggingface.co/Comfy-Org/Qwen-Image-2.1) Turbo image generation models, including Qwen3-VL-8B and VAE (optional download; [notice](freevideo_engine/image_gen/licenses/NOTICE.txt) installed with the models), © 2026 Hangzhou Tongyi Laboratory Technology Co., Ltd. All Rights Reserved. | [Qwen RESEARCH LICENSE AGREEMENT](https://huggingface.co/Comfy-Org/Qwen-Image-2.1/blob/main/LICENSE) | | [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) 1.13.8 speech runtime, built by FreeVideo without text-to-speech, ALSA and PortAudio (modified build; `__init__.py` edited; source snapshot 11afbd00 kept by FreeVideo; optional download for automatic subtitles, [build note](freevideo_engine/licenses/sherpa-onnx-build-NOTE.txt)). Includes statically linked kaldi-native-fbank, kaldi-decoder, kaldifst, OpenFst, simple-sentencepiece (Apache-2.0), KissFFT (BSD-3-Clause), pybind11 (BSD-3-Clause), nlohmann/json (MIT), hclust-cpp and Darts-clone (BSD-2-Clause), ThreadPool (Zlib). The Windows build also contains msvcp140.dll and msvcp140_1.dll (Microsoft Visual C++ 2022 runtime 14.44.35211, © Microsoft Corporation), unmodified and redistributed as permitted by Microsoft for its redistributable files. | Apache-2.0 | +| [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) code adapted for dubbing and [Fun-CosyVoice3-0.5B-2512](https://modelscope.cn/models/FunAudioLLM/Fun-CosyVoice3-0.5B-2512) weights, © 2024 Alibaba Inc (optional download); includes MIT-licensed parts of Matcha-TTS, HiFi-GAN, x_transformers and Snake | [Apache-2.0](freevideo_engine/licenses/CosyVoice-Apache-2.0.txt); [MIT notices](freevideo_engine/licenses/CosyVoice-NOTICE.txt) | | [ONNX Runtime](https://github.com/microsoft/onnxruntime) 1.28.2 (CPU), © Microsoft Corporation; its third-party notices are shipped inside the sherpa-onnx wheel | MIT | | [Eigen](https://gitlab.com/libeigen/eigen) 5.0.1, compiled into sherpa-onnx and ONNX Runtime; source: https://gitlab.com/libeigen/eigen/-/tree/5.0.1 | MPL-2.0 | | [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download with subtitles, license file downloaded with the model; the onnx-community export is downloaded with the music model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) | diff --git a/__init__.py b/__init__.py index 3349199c..b6214c5a 100644 --- a/__init__.py +++ b/__init__.py @@ -31,6 +31,12 @@ async def comfy_entrypoint(): except Exception as error: import logging logging.warning('[FreeVideo] Image generation routes not registered: %s', error) + try: + from .freevideo_engine.dubbing.service import register as dubbing_routes + dubbing_routes() + except Exception as error: + import logging + logging.warning('[FreeVideo] Dubbing routes not registered: %s', error) from .freevideo_engine.comfy_launcher_api import register as launcher_routes launcher_routes() from .freevideo_engine.comfy_updates import register as update_routes diff --git a/constraints/unified.txt b/constraints/unified.txt index 882a9d69..b2814919 100644 --- a/constraints/unified.txt +++ b/constraints/unified.txt @@ -17,6 +17,7 @@ cuda-toolkit==13.0.3.0 cryptography==50.0.1 einops==0.8.2 filelock==3.32.3 +flatbuffers==25.12.19 fsspec==2026.7.0 h11==0.16.0 hf-xet==1.6.1a0 @@ -50,10 +51,12 @@ nvidia-nvjitlink==13.3.33 nvidia-nvshmem-cu13==3.4.5 nvidia-nvtx==13.0.85 omegaconf==2.3.1 +onnxruntime==1.31.1 packaging==26.3 peft==0.20.0 pillow==12.3.0 pip==26.2.1 +protobuf==7.36.2 psutil==7.2.2 pygments==2.21.0 pycparser==3.0 diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index 8f11a97f..1df7d39b 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -125,6 +125,14 @@ 'Tell the user the model needs a network connection, then submit the job again with a new ref.'), 'stems_failed': (409, True, 'Splitting the sound did not finish.', 'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'), + 'voice_failed': (409, True, 'Changing the voice did not finish.', + 'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'), + 'voice_model_missing': (409, False, 'The dubbing model is not ready on this machine.', + 'Tell the user to install the dubbing model first; do not retry.'), + 'no_speech': (409, False, 'No speech was found in the selection.', + 'Do not retry; choose a stretch of sound with someone speaking.'), + 'shorten_unavailable': (409, False, 'Lines cannot be shortened here yet.', + 'Write the shorter line yourself and send it with the other lines.'), 'clip_source_unusable': (409, False, 'A clip\'s source is missing, or cannot be read as far as the clip uses it.', 'Read the project and check each clip\'s source: it must exist and play, and in + frames must stay ' 'within its real length. Correct the project, then export the new revision.'), @@ -239,7 +247,7 @@ def recovery(code, *, kind=None, stage='request', resumable=True): return 'Tell the user to free disk space, then submit the picture again with a new ref.' if kind == 'music' and code == 'out_of_memory': return 'Close other GPU applications, then submit the music job again with a new ref.' - if kind == 'stems' and code == 'out_of_memory': + if kind in ('stems', 'voice', 'voice_lines') and code == 'out_of_memory': return 'Close other GPU applications, then submit the job again with a new ref.' if kind == 'image' and code == 'no_memory': return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.' diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 81726d2d..cd6ce81c 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -145,6 +145,7 @@ 'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.', 'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.', 'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.', + 'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice.', 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.', 'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.', 'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.', @@ -380,6 +381,10 @@ '剪辑拼好后先读取 revision，并删去该轨道上原有的配乐；首次使用会下载音乐模型（约 20 GB）', ('POST', '/projects/{id}/stems'): '把一个声音片段的声音分成人声、音乐和环境声，放在其下方的 a4、a5、a6 三条轨道上，原片段静音。' '先读取 revision 与片段 ID，等待返回的任务完成，结果 parts 给出各轨道的名称、起止帧；仅在 features 含 projects.stems 时可用', + ('POST', '/projects/{id}/voice/lines'): '把选中声音片段里说的话逐句写下来，每句带起止秒数；先读取 revision 与片段 ID，等待返回的任务完成，结果 lines 给出各句；仅在 features 含 projects.voice 时可用', + ('POST', '/projects/{id}/voice/replace'): '把改过文字的句子用原说话人的声音重新说一遍，放回原句的位置，没改的句子保持原声；提交 voice/lines 给出的全部句子，只改 text；结果 lines 给出每句新声的起止，放不进下一句之前的句子保留原声', + ('POST', '/projects/{id}/voice/shorten'): '按目标秒数改短一句的文字；只返回文字，不保存也不配音', + ('POST', '/projects/{id}/voice/estimate'): '估算改过的句子配出来的秒数；立即返回', ('POST', '/projects/{id}/stems/export'): '把已拆开片段的人声、音乐、环境声和原混音保存为四个 wav 文件；先读取 revision 与被拆片段的 ID，返回立即给出文件夹和文件名', ('POST', '/images'): '像图片编辑页面一样打开图片；提交 source，会话会保留源图片副本和图层文档', ('GET', '/images/{id}'): '读取图片编辑会话的图层文档、选区和 revision；编辑前先读取 revision 和图层标识', diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 53d5b346..906d04a3 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -32,7 +32,7 @@ ACTIVE = ('waiting', 'queued', 'running', 'cancelling') CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.' DONE = ('succeeded', 'failed', 'cancelled', 'interrupted') -GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems') +GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'voice') IMAGE_ERRORS = errors.IMAGE_ERRORS IMAGE_WAIT_ERRORS = ('busy', 'busy_video') IMAGE_WAIT_NOTICES = { @@ -532,7 +532,7 @@ def load(self): def readable(row): return (isinstance(row, dict) and specs.JOB_ID.fullmatch(str(row.get('id'))) - and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'image_edit') + GPU_KINDS + and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'voice_lines', 'image_edit') + GPU_KINDS and row.get('status') in ACTIVE + DONE and isinstance(row.get('spec'), dict) and type(row.get('created_at')) in (int, float) and math.isfinite(row['created_at']) and (self._receipts is None or (type(row.get('_receipt_revision', 0)) is int @@ -601,7 +601,7 @@ def readable(row): records = list(latest.values()) for row in records: if readable(row): - if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'image_edit') + GPU_KINDS: + if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'voice_lines', 'image_edit') + GPU_KINDS: # This process restarted while the work ran in it; nothing continues it. self.finish(row, 'failed', error=dict(code='interrupted', message='ComfyUI restarted while this job ran; submit it again'), save=False) @@ -1488,6 +1488,12 @@ def run(): elif row['kind'] == 'stems': from .stems import run_stems result = run_stems(self, row) + elif row['kind'] == 'voice_lines': + from .voice import run_voice_lines + result = run_voice_lines(self, row) + elif row['kind'] == 'voice': + from .voice import run_voice_replace + result = run_voice_replace(self, row) else: result = self.run_assemble(row) return result diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index 619935cb..803cd886 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -74,6 +74,11 @@ def view(self): return value +def _as_expected(project, clip_id, expect): + from .voice import signature + return signature(_clips(project).get(clip_id)) == expect + + def _invalid(path, expected, received=None): return spec.SpecError([schema.diagnostic(path, expected, received)]) @@ -460,11 +465,75 @@ def stems_export(self, identity, value, *, ref=None, client=None): self._remember(ref, digest, result, client) return result - def lay_stems(self, identity, clip_id, sources): - """Put the three parts on sub-tracks under the clip and silence the clip; returns the save and the new clip IDs.""" + def _voice_job(self, name, kind, identity, value, *, jobs, ref, client): + """Queue a voice job for the revision the caller read; the selected clip must be a sound clip with media.""" + schema.validate(schema.project_schema(name), value) + with self.listen_lock: + digest, replay = self._receipt('projects:' + name, dict(id=identity, body=value), ref, client) + if replay is not None: + return replay + stored = self._load(identity)['project'] + if stored['revision'] != value['revision']: + raise ProjectError('project_changed', project=stored) + selection = value['selection'] + clip = _clips(stored).get(selection['clip_id']) + if clip is None or not clip.get('source'): + raise _invalid('/selection/clip_id', 'a sound clip with media', selection['clip_id']) + if 'end' in selection and selection['end'] <= selection.get('start', 0): + raise _invalid('/selection/end', 'a time after the start', selection['end']) + if kind == 'voice': + ids = [line['id'] for line in value['lines']] + if len(set(ids)) != len(ids): + raise _invalid('/lines', 'lines with different ids', ids[:3]) + for i, line in enumerate(value['lines']): + if line['end'] <= line['start']: + raise _invalid('/lines/%d/end' % i, 'a time after the start', line['end']) + from . import voice + if not value.get('recognition'): + raise _invalid('/recognition', 'the recognition voice_lines gave for these lines; run voice_lines first') + wrong = voice.check(value['recognition'], identity, selection['clip_id'], value['lines']) + if wrong: + raise _invalid(*wrong) + reference = (value.get('voice') or {}).get('reference') + if reference and reference['clip_id'] not in _clips(stored): + raise _invalid('/voice/reference/clip_id', 'a clip of this project', reference['clip_id']) + job = jobs.submit(dict(kind=kind, editor_project=identity, + **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0] + result = dict(job=jobs.view(job), jobs=[jobs.view(job)]) + self._remember(ref, digest, result, client) + return result + + def voice_lines(self, identity, value, *, jobs, ref=None, client=None): + """Queue the writing down of a selection's lines (what is said, and when).""" + return self._voice_job('voice_lines', 'voice_lines', identity, value, jobs=jobs, ref=ref, client=client) + + def voice_replace(self, identity, value, *, jobs, ref=None, client=None): + """Queue the new voice for the lines the caller changed.""" + return self._voice_job('voice_replace', 'voice', identity, value, jobs=jobs, ref=ref, client=client) + + def voice_estimate(self, identity, value, *, ref=None, client=None): + """Seconds the changed lines take to make; asked at once, nothing is stored.""" + from . import voice + schema.validate(schema.project_schema('voice_estimate'), value) + self._load(identity) + return voice.estimate(value['lines'], value.get('limit')) + + def voice_shorten(self, identity, value, *, ref=None, client=None): + """Rewriting a line to a length needs a text model; none is wired in this version.""" + schema.validate(schema.project_schema('voice_shorten'), value) + self._load(identity) + raise JobError('shorten_unavailable', 'No text model is available to shorten the line') + + def lay_stems(self, identity, clip_id, sources, *, expect=None): + """Put the three parts on sub-tracks under the clip and silence the clip; returns the save and the new clip IDs. + + `expect` (voice.signature of the clip) makes it a no-op that raises project_changed when the clip is not as it was.""" before, refused = {}, {} def change(project): before['ids'] = set(_clips(project)) + if expect is not None and not _as_expected(project, clip_id, expect): + refused['stale'] = True + return dict(ok=True, project=project) reply = editor_runtime.runtime().call('placeStems', project, [clip_id, { name: dict(kind='generated', id=source.split(':', 1)[1]) for name, source in sources.items()}], module='model', serialize=True)[0] @@ -473,11 +542,36 @@ def change(project): return dict(ok=True, project=project) return dict(ok=True, project=reply['value']) saved = self._update(identity, change) + if refused.get('stale'): + raise JobError('project_changed', 'The clip changed while the voice was made') if refused: code = STATE_CODES.get(refused.get('code'), 'stems_failed') raise JobError(code, 'The separated sound cannot be laid under the clip (%s)' % refused.get('code')) return saved, sorted(set(_clips(saved)) - before['ids']) + def replace_voice_source(self, identity, clip_id, source, *, expect=None): + """The dialogue clip takes the file with the replaced voice; its place and range stay. Returns the save. + + `expect` (voice.signature of the clip) makes it a no-op that raises project_changed when the clip is not as it was.""" + refused = {} + def change(project): + if expect is not None and not _as_expected(project, clip_id, expect): + refused['stale'] = True + return dict(ok=True, project=project) + reply = editor_runtime.runtime().call('replaceSource', project, [clip_id, dict( + kind='generated', id=source.split(':', 1)[1])], module='model', serialize=True)[0] + if not reply['ok']: + refused.update(reply['error']) + return dict(ok=True, project=project) + return dict(ok=True, project=reply['value']) + saved = self._update(identity, change) + if refused.get('stale'): + raise JobError('project_changed', 'The clip changed while the voice was made') + if refused: + code = STATE_CODES.get(refused.get('code'), 'voice_failed') + raise JobError(code, 'The replaced voice cannot be put on the sound track (%s)' % refused.get('code')) + return saved + def lay_music(self, identity, source): """Put a finished score on a2 from frame 0 of the project as it is now; returns the save and the new clip IDs.""" found = comfy_editor.sources(self.root, [source])[source] diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 79c82b5c..7ef65d92 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -172,13 +172,27 @@ def submit_batch(jobs, body, client): return jobs.submit_request(body, client=client)[0] +JOB_OPERATIONS = ('generate', 'export', 'listen', 'music', 'stems', 'voice_lines', 'voice_replace') + + def stems_supported(machine): """Sound separation is offered on NVIDIA machines with enough video memory, never on a Mac.""" from ..stems.service import supported return supported(machine.get('vram_gib'), uuid=machine.get('gpu_uuid')) and machine.get('device_backend', 'cuda') == 'cuda' -def features(continuation, vlm, stems=False): +def voice_ready(port): + """Whether the dubbing model that changes a voice is on this machine.""" + service = getattr(port.server, '_freevideo_dubbing', None) + if service is None: + return False + try: + return bool(service.info(wait=False).get('ready')) + except (OSError, ValueError): + return False + + +def features(continuation, vlm, stems=False, voice=True): """What an agent can use here (ready says whether the engine can run now). A missing name is not offered.""" from .editor_runtime import available rules = available() @@ -190,7 +204,7 @@ def features(continuation, vlm, stems=False): 'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules, 'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True, **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music')}, - 'projects.stems': bool(rules and stems)} + 'projects.stems': bool(rules and stems), 'projects.voice': bool(rules and stems and voice)} def whitebox_support(jobs): @@ -228,7 +242,7 @@ def capabilities(port, jobs): return dict( api_version=API_VERSION, freevideo_version=release or __version__, build=__version__, ready=ready, problem=problem, continuation=continuation, - features=features(continuation, vlm, stems=stems_supported(machine)), + features=features(continuation, vlm, stems=stems_supported(machine), voice=voice_ready(port)), whitebox=whitebox_support(jobs), device=dict(backend=machine.get('device_backend', 'cuda') if machine else None, vram_limit_gib=machine.get('vram_gib'), ram_limit_gib=machine.get('ram_gib')), @@ -241,6 +255,8 @@ def capabilities(port, jobs): 'listen': 'Listen to the sound of an editing project for subtitle lines and times', 'music': 'Compose music for an editing project and lay it on its second sound track', 'stems': 'Split the sound of a clip into dialogue, music and ambience on sub-tracks', + 'voice_lines': 'Write down the spoken lines of a sound clip with their start and end', + 'voice': 'Say edited lines again in the original voice and put them back where the originals were', 'image_edit': 'Render a saved revision of an image session to a picture'}, limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS], seconds_tested=[4, 15], @@ -2225,7 +2241,7 @@ async def handler(request): raise JobError('idempotency_conflict', 'Conflicting keys') args = [] if operation == 'create' else [identity] extra = dict(client=request.headers.get('X-FreeVideo-Client', '')[:80] or None) - if operation in ('generate', 'export', 'listen', 'music', 'stems'): + if operation in JOB_OPERATIONS: extra['jobs'] = jobs if operation == 'generate': def capability(size): @@ -2235,9 +2251,9 @@ def capability(size): return None extra['capability'] = capability result = await asyncio.to_thread(getattr(projects, operation), *args, value, ref=key or ref, **extra) - if operation in ('generate', 'export', 'listen', 'music', 'stems'): + if operation in JOB_OPERATIONS: await jobs.tick() - return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen', 'music', 'stems') else 200) + return reply(result, 201 if operation == 'create' or operation in JOB_OPERATIONS else 200) return guarded(handler) for operation, (method, suffix, _) in contract.PROJECT_OPERATIONS.items(): diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index ecbef87a..ff7bab3a 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -351,6 +351,58 @@ def stems(*, local=False): **{'x-job-kind': 'stems'}) +VOICE_SELECTION = obj(dict(clip_id=string(64), start=number(0, 36000, description='Seconds from the clip start; default 0.'), + end=number(0, 36000, description='Seconds from the clip start; default the clip end.')), ('clip_id',), + description='The sound clip, or part of it, whose voice to change: the original sound, or its dialogue sub-track.') +VOICE_LANGUAGE = string(8, pattern=r'^(auto|[a-z]{2,3})$', default='auto') +VOICE_LINE = obj(dict(id=string(16, pattern=r'^l[0-9]{3,5}$'), start=number(0, 36000), end=number(0, 36000), + text=string(2000, description='The line as the user wants it said.'), + original_text=string(2000, description='The line as it was recognised.'), + rate=number(.1, 60, description='Units spoken per second, from voice_lines.')), + ('id', 'start', 'end', 'text', 'original_text')) +VOICE_REFERENCE = obj(dict(clip_id=string(64), start=number(0, 36000), end=number(0, 36000), text=string(1000)), + ('clip_id', 'start', 'text'), description='A stretch of a sound clip whose voice every line takes.') +VOICE_LINES_FIELDS = dict( + selection=VOICE_SELECTION, language=VOICE_LANGUAGE, + separate_first={'type': 'boolean', 'default': False, + 'description': 'Separate the sound first and recognise on the dialogue alone.'}) +VOICE_FIELDS = dict( + selection=VOICE_SELECTION, language=VOICE_LANGUAGE, + lines=array(VOICE_LINE, 1, 200, description='The lines voice_lines gave, with the edited ones changed; only those are made again.'), + voice=obj(dict(preset=string(64), reference=VOICE_REFERENCE), + description='Omit for each line to keep its own speaker\'s voice.'), + recognition=string(64, description='The recognition voice_lines gave (required); the lines are checked against it. When it is no longer kept, run voice_lines again.'), + seed=number(0, 2**31 - 1, integer=True, description='Same seed and lines, same voice.'), + lay={'type': 'boolean', 'default': True, + 'description': 'False leaves the new voice unplaced: result.apply says how the page puts it on the timeline.'}) +VOICE_SHORTEN_FIELDS = dict( + line=obj(dict(id=string(16, pattern=r'^l[0-9]{3,5}$'), text=string(2000), seconds=number(.1, 600)), ('id', 'text', 'seconds')), + target_seconds=number(.1, 600), language=VOICE_LANGUAGE, + context=array(string(2000), 0, 6, description='The lines before and after, for sense.')) +VOICE_ESTIMATE_FIELDS = dict( + lines=array(obj(dict(id=string(16, pattern=r'^l[0-9]{3,5}$'), text=string(2000), original_text=string(2000), + rate=number(.1, 60), start=number(0, 36000), end=number(0, 36000)), + ('id', 'text', 'original_text')), 1, 200, + description='Every line of the selection; with start and end on the lines, each changed line also gets the room it has.'), + limit=number(0, 36000, description='The end of the selection in seconds, where the last line may reach; with the lines\' start and end.')) + + +def voice_lines(*, local=False): + return obj(dict(kind={'const': 'voice_lines', 'type': 'string'}, + editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$'), revision=number(1, 2**53 - 1, integer=True), + **VOICE_LINES_FIELDS, **common()), ('kind', 'editor_project', 'revision', 'selection'), + description='Write down the lines of a selection with the local speech model; nothing is saved.', + **{'x-job-kind': 'voice_lines'}) + + +def voice(*, local=False): + return obj(dict(kind={'const': 'voice', 'type': 'string'}, + editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$'), revision=number(1, 2**53 - 1, integer=True), + **VOICE_FIELDS, **common()), ('kind', 'editor_project', 'revision', 'selection', 'lines'), + description='Say the edited lines again in the voice of the original and put them where the originals were.', + **{'x-job-kind': 'voice'}) + + def image_edit(*, local=False): return obj(dict(kind={'const': 'image_edit', 'type': 'string'}, image=dict(IMAGE, description='The image session to render (POST /images).'), @@ -361,12 +413,12 @@ def image_edit(*, local=False): KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, image=image, - export=export, listen=listen, music=music, stems=stems, image_edit=image_edit) + export=export, listen=listen, music=music, stems=stems, voice_lines=voice_lines, voice=voice, image_edit=image_edit) def job_schema(*, local=False, refs=False, batch=False): # Music scores a joined cut, so it is not a storyboard row. - value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems'))], 'x-discriminator': 'kind', + value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems', 'voice_lines', 'voice'))], 'x-discriminator': 'kind', 'description': 'x-discriminator selects field diagnostics by kind; all accepted branches use standard oneOf.'} return batch_references(value) if refs else value @@ -483,7 +535,7 @@ def route(summary, body=None, query=None, **kw): if method == 'GET': extra['query'] = shape else: - extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems') else {})) + extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems', 'voice_lines', 'voice_replace') else {})) routes[method, suffix] = route(summary, **extra) for name, (method, suffix, summary) in WHITEBOX_OPERATIONS.items(): extra = dict(path=obj({'id': WHITEBOX}, ('id',))) if '{id}' in suffix else {} @@ -874,13 +926,19 @@ def image_schema(name): 'listen': ('POST', '/projects/{id}/listen', 'Listen to the timeline sound for subtitles: mode align times lines you give, transcribe writes down what is said. First get the revision; wait for the returned job, then add the lines with the subtitles edit. Nothing is saved; with no speech model for the language the call fails with no_speech_model, so write the lines by hand.'), 'stems': ('POST', '/projects/{id}/stems', 'Split one clip\'s sound into dialogue, music and ambience on tracks a4-a6. First get the revision and clip ID; wait for the job, then read result.parts.'), 'stems_export': ('POST', '/projects/{id}/stems/export', 'Save the separated dialogue, music and ambience of a split clip, with its original mix, as four wav files in the saved sounds. First get the revision and the split clip\'s ID; the answer lists the files at once.'), + 'voice_lines': ('POST', '/projects/{id}/voice/lines', 'Write down the lines of a selected sound clip, each with its start and end. First get the revision and clip ID; wait for the job, then read result.lines. Nothing is saved.'), + 'voice_replace': ('POST', '/projects/{id}/voice/replace', 'Say the lines you changed again in the original voice and put them where the originals were; unchanged lines keep their sound. Send the lines voice_lines gave, with text changed; wait for the job, then read result.lines for where each new line starts and ends.'), + 'voice_shorten': ('POST', '/projects/{id}/voice/shorten', 'Shorten one line to fit a number of seconds. The answer is a text; nothing is saved or spoken.'), + 'voice_estimate': ('POST', '/projects/{id}/voice/estimate', 'Estimate the seconds the changed lines take to make. The answer comes at once.'), 'music': ('POST', '/projects/{id}/music', 'Compose music as long as the joined cut and lay it on sound track a2 at frame 0, under the video sound. First get the revision; a2 must be empty. Wait for the returned job; the first use downloads the model (20 GB).'), } PROJECT_TOOLS = dict(list='freevideo_projects', get='freevideo_project_read', create='freevideo_project_create', write='freevideo_project_write', operations='freevideo_editor_operations', generate='freevideo_project_generate', export='freevideo_export', listen='freevideo_project_listen', music='freevideo_project_music', - stems='freevideo_project_stems', stems_export='freevideo_project_stems_export') + stems='freevideo_project_stems', stems_export='freevideo_project_stems_export', + voice_lines='freevideo_project_voice_lines', voice_replace='freevideo_project_voice_replace', + voice_shorten='freevideo_project_voice_shorten', voice_estimate='freevideo_project_voice_estimate') SESSION_OPERATIONS = { 'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'), @@ -1003,6 +1061,14 @@ def project_schema(name): return obj(dict(revision=revision, **MUSIC_FIELDS, ref=REF), ('revision', 'caption')) if name in ('stems', 'stems_export'): return obj(dict(revision=revision, **STEMS_FIELDS, **(STEMS_LAY if name == 'stems' else {}), ref=REF), ('revision', 'clip_id')) + if name == 'voice_lines': + return obj(dict(revision=revision, **VOICE_LINES_FIELDS, ref=REF), ('revision', 'selection')) + if name == 'voice_replace': + return obj(dict(revision=revision, **VOICE_FIELDS, ref=REF), ('revision', 'selection', 'lines', 'recognition')) + if name == 'voice_shorten': + return obj(dict(revision=revision, **VOICE_SHORTEN_FIELDS, ref=REF), ('revision', 'line', 'target_seconds')) + if name == 'voice_estimate': + return obj(dict(revision=revision, **VOICE_ESTIMATE_FIELDS, ref=REF), ('lines',)) if name != 'operations': return EMPTY fields = { diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index f19581f8..ef3200a1 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -300,6 +300,14 @@ def normalize(value, *, port=None): elif kind == 'stems': result.update(editor_project=value['editor_project'], revision=value['revision'], clip_id=value['clip_id'], lay=value.get('lay', True)) + elif kind == 'voice_lines': + result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']), + language=value.get('language', 'auto'), separate_first=bool(value.get('separate_first', False))) + elif kind == 'voice': + result.update(editor_project=value['editor_project'], revision=value['revision'], selection=dict(value['selection']), + lines=[dict(line) for line in value['lines']], language=value.get('language', 'auto'), + voice=dict(value.get('voice') or {}), lay=value.get('lay', True), + **{k: value[k] for k in ('seed',) if k in value}) elif kind == 'image_edit': result.update(image=value['image'], revision=value['revision']) else: diff --git a/freevideo_engine/agent/voice.py b/freevideo_engine/agent/voice.py new file mode 100644 index 00000000..ffd31741 --- /dev/null +++ b/freevideo_engine/agent/voice.py @@ -0,0 +1,721 @@ +"""Voice change jobs: the lines of a selection, and new voice for the lines the user edited. + +The lines come from the separated voice (where each one starts and ends) and the recogniser (what it says). +A replacement keeps every unedited sample of the separated voice and puts each new line into the window of the +original one; a line that cannot fit before the next original line keeps its original voice. +""" +import hashlib +import json +import re +import secrets +import shutil +import threading +import time +from collections import OrderedDict +from math import gcd +from pathlib import Path + +import numpy as np + +from .jobs import JobError +from .stems import FOLDER as STEMS_FOLDER, _busy, run_stems + +FOLDER = 'freevideo-voice' +FPS = 24 +RATE = 44100 +GAP_BEFORE = .05 # quiet kept before the first sound of a line, at most (half the gap to the line before) +GAP_AFTER = .1 # quiet kept after the last sound of a line, at most (half the gap to the next line) +CLEARANCE = .1 # a new voice stops this long before the next original line starts +MIN_WINDOW = .3 +MARGIN = .1 # extra file read around a line when its sound is measured +PREEMPTIONS = 8 +BUSY_WAITS = 8 +SPEED = dict(zh=4.5, en=2.8) # units per second of a voice, when the line gave no rate of its own +DUB_CODES = {'out_of_memory': 'out_of_memory', 'model_missing': 'voice_model_missing', + 'invalid_request': 'invalid_request', 'disk_space': 'storage_unavailable'} +SELECTION_MESSAGE = 'The selection must be a sound clip played at normal speed, forwards' + + +def units(text): + """Spoken units of a text: every Chinese, Japanese or Korean character, and every other word.""" + text = re.sub(r'[　-〿＀-／]', ' ', text) + wide = re.findall(r'[぀-ヿ㐀-鿿가-힯]', text) + rest = re.sub(r'[぀-ヿ㐀-鿿가-힯]', ' ', text) + return len(wide) + len(re.findall(r"[^\W_]+(?:['’-][^\W_]+)*", rest)) + + +def edited(lines): + """The lines whose text differs from the original after trimming.""" + return [line for line in lines if line['text'].strip() != line['original_text'].strip()] + + +def _room(lines, limit): + """{id: seconds a new voice may take from the line's start}, for lines that came with their times. + + The same rule as the placing: the new voice may use the quiet after the line up to CLEARANCE before the next line + (or the end of the selection); the line's recognised start and end stand in for the measured onset and offset.""" + timed = sorted((line for line in lines if line.get('start') is not None and line.get('end') is not None), + key=lambda line: line['start']) + room = {} + for position, line in enumerate(timed): + start, end = line['start'], line['end'] + if position + 1 < len(timed): + after = timed[position + 1]['start'] + elif limit is not None: + after = limit + CLEARANCE + else: + after = end + 2 * GAP_AFTER + trail = min(GAP_AFTER, max(0., (after - end) / 2)) + last = max(end + trail, start + MIN_WINDOW) + room[line['id']] = max(last, after - CLEARANCE) - start + return room + + +def estimate(lines, limit=None): + """Seconds each edited line takes to say: its units over the rate of the original line (or a default). + + A line that came with its times also gets `available` (seconds from its start to the latest end of its new voice) and + `overrun` (seconds by which the new voice, at the fastest tempo, still passes that end; 0 when it fits).""" + from ..dubbing import align + room = _room(lines, limit) + result = [] + for line in edited(lines): + text = line['text'].strip() + rate = line.get('rate') + if not rate or rate <= 0: + rate = SPEED['zh'] if re.search(r'[぀-ヿ㐀-鿿가-힯]', text) else SPEED['en'] + seconds = round(max(units(text), 1) / rate, 2) + available = room.get(line['id']) + result.append(dict(id=line['id'], seconds=seconds, + available=None if available is None else round(available, 2), + overrun=None if available is None else round(max(0., seconds / align.MAX_STRETCH - available), 2))) + return dict(seconds=round(sum(row['seconds'] for row in result), 2), lines=result) + + +# ---------------------------------------------------------------------------------------------- what was recognised + +_RECOGNIZED = OrderedDict() # token -> dict(project, clip, lines={id: (start, end, text)}, at) +_RECOGNIZED_LOCK = threading.Lock() +RECOGNIZED_KEEP = 64 +RECOGNIZED_SECONDS = 3600 +LINE_TOLERANCE = .011 # the lines are sent back rounded to milliseconds + + +def remember(spec, ctx, language, lines): + """Keep what voice_lines recognised under a token: a hash of the project, the sound it read and every line. + + The ids are positions in this one recognition, so a new recognition (or any change of the sound) gives a new token.""" + rows = {line['id']: (line['start'], line['end'], line['text']) for line in lines} + source = ctx['source'] + text = json.dumps([spec['editor_project'], ctx['clip']['id'], source['kind'], str(source['id']), ctx['base'], + ctx['start'], ctx['end'], language, sorted(rows.items())], ensure_ascii=False) + token = hashlib.sha256(text.encode()).hexdigest()[:16] + with _RECOGNIZED_LOCK: + _RECOGNIZED[token] = dict(project=spec['editor_project'], clip=ctx['clip']['id'], lines=rows, at=time.monotonic()) + _RECOGNIZED.move_to_end(token) + while len(_RECOGNIZED) > RECOGNIZED_KEEP: + _RECOGNIZED.popitem(last=False) + return token + + +def check(token, project, clip_id, lines): + """None when the lines agree with the recognition behind the token, else (path, expected, received) of the first + disagreement: a token that is unknown or too old (the service restarted, or other recognitions pushed it out), an id the + recognition never had, a line of the recognition that is not sent, another start or end, or another original text. Edited text may be anything; punctuation and width changes count as edits.""" + with _RECOGNIZED_LOCK: + found = _RECOGNIZED.get(token) + if found is not None and time.monotonic() - found['at'] > RECOGNIZED_SECONDS: + del _RECOGNIZED[token] + found = None + if found is None: + return '/recognition', 'a recognition that is still kept; run voice_lines again for this clip', token + if found['project'] != project or found['clip'] != clip_id: + return '/recognition', 'a recognition of this clip', token + for index, line in enumerate(lines): + known = found['lines'].get(line['id']) + if known is None: + return '/lines/%d/id' % index, 'a line id from the recognition', line['id'] + if abs(line['start'] - known[0]) > LINE_TOLERANCE: + return '/lines/%d/start' % index, 'the recognised start %s' % known[0], line['start'] + if abs(line['end'] - known[1]) > LINE_TOLERANCE: + return '/lines/%d/end' % index, 'the recognised end %s' % known[1], line['end'] + if line['original_text'].strip() != known[2].strip(): + return '/lines/%d/original_text' % index, 'the recognised text', None + sent = {line['id'] for line in lines} + left = [name for name in found['lines'] if name not in sent] + if left: + # a new voice that runs on past a changed line must not land on a line that was left out + return '/lines', 'every line of the recognition (%d lines); missing %s' % (len(found['lines']), ', '.join(left[:3])), len(lines) + return None + + +def signature(clip): + """What a result placed on a clip depends on: where it is, which sound it plays and how. None when it is gone.""" + if not clip: + return None + source = clip.get('source') or {} + return dict(start=clip['start'], frames=clip['frames'], at=clip.get('in', 0), source=[source.get('kind'), source.get('id')], + speed=clip.get('speed') or 1, reverse=bool(clip.get('reverse'))) + + +# ---------------------------------------------------------------------------------------------- the selection + +def _tracks(project): + return [track for track in project['tracks'] if track['kind'] == 'audio'] + + +def context(jobs, spec): + """Where the sound of a selection is: {clip, parent, dialogue, separated, base, start, end, source, wav}.""" + from ..editor_export import resolve_source + try: + project = jobs.projects.get(spec['editor_project'])['project'] + except FileNotFoundError as error: + raise JobError('not_found', 'The project does not exist') from error + found = {} + for track in _tracks(project): + for clip in track['clips']: + found[clip['id']] = (track, clip) + selection = spec['selection'] + if selection['clip_id'] not in found: + raise JobError('invalid_request', 'The selection names a clip that is not in this project') + track, clip = found[selection['clip_id']] + if track.get('stem') in ('music', 'ambience'): + raise JobError('invalid_request', 'The music and ambience parts have no voice to change') + if not clip.get('source'): + raise JobError('invalid_request', 'The clip is not a sound clip with media') + if (clip.get('speed') or 1) != 1 or clip.get('reverse'): + raise JobError('invalid_request', SELECTION_MESSAGE) + if clip['source']['kind'] == 'generated' and str(clip['source']['id']).startswith('freevideo-dubbing/'): + raise JobError('invalid_request', 'A written line is dubbed again from its text, not by changing its voice') + parent, dialogue = clip, None + if track.get('stem') == 'dialogue': + dialogue = clip + if clip.get('stem_of') not in found: + raise JobError('invalid_request', 'The voice part has lost the clip it was split from') + parent = found[clip['stem_of']][1] + else: + for other, candidate in found.values(): + if (other.get('stem') == 'dialogue' and candidate.get('stem_of') == clip['id'] + and candidate['start'] <= clip['start'] and candidate['start'] + candidate['frames'] >= clip['start'] + clip['frames']): + dialogue = candidate + break + length = clip['frames'] / FPS + start = float(selection.get('start', 0.)) + end = float(selection.get('end', length)) + if not 0 <= start < end <= length + 1e-6: + raise JobError('invalid_request', 'The selection must lie inside the clip') + carrier = dialogue or clip + try: + found_source = resolve_source('%s:%s' % (carrier['source']['kind'], carrier['source']['id'])) + except (OSError, ValueError) as error: + raise JobError('voice_failed', 'The clip media is not readable') from error + path = found_source['audio'] + if path is None: + raise JobError('voice_failed', 'The clip has no sound') + wav = Path(path).with_suffix('.wav') + return dict(clip=clip, parent=parent, dialogue=dialogue, separated=dialogue is not None, + base=carrier.get('in', 0) / FPS, start=start, end=min(end, length), + source=carrier['source'], path=wav if wav.is_file() else Path(path)) + + +def original_of(ctx, port): + """The wav of the voice before any change (a second change takes its references from it), and its source.""" + if not ctx['separated']: + return ctx['path'], None + note = ctx['path'].parent / 'voice.json' + if ctx['source']['kind'] == 'generated' and note.is_file(): + try: + original = json.loads(note.read_text(encoding='utf-8'))['original'] + from ..editor_export import resolve_source + path = Path(resolve_source(original)['audio']) + wav = path.with_suffix('.wav') + return (wav if wav.is_file() else path), original + except (OSError, ValueError, KeyError, TypeError): + pass + return ctx['path'], '%s:%s' % (ctx['source']['kind'], ctx['source']['id']) + + +# ---------------------------------------------------------------------------------------------- helpers + +class _Row: + """The real job row, with the request spec swapped: lets the stem job run inside another job without laying.""" + def __init__(self, row, spec): + self._row, self.spec = row, spec + + def __getitem__(self, key): + return self.spec if key == 'spec' else self._row[key] + + def __setitem__(self, key, value): + self._row[key] = value + + def get(self, key, default=None): + return self.spec if key == 'spec' else self._row.get(key, default) + + +def _cancelled(row): + if row.get('_cancel'): + raise JobError('cancelled', 'Cancelled') + + +def _phase(row, name, **detail): + row['_phase'] = name + row['_detail'] = json.dumps(detail) if detail else None + + +def _separate(jobs, row, ctx, poll=.5): + """Split the sound of the parent clip without laying the parts; returns the sources and the dialogue wav.""" + from ..editor_export import resolve_source + spec = dict(row['spec'], clip_id=ctx['parent']['id'], lay=False) + _phase(row, 'separating') + result = run_stems(jobs, _Row(row, spec), poll=poll) + sources = result['sources'] + mp4 = Path(resolve_source(sources['dialogue'])['audio']) + return sources, (mp4.with_suffix('.wav') if mp4.with_suffix('.wav').is_file() else mp4) + + +def _drop_folder(jobs, sources): + if not sources: + return + first = next(iter(sources.values())) + relative = first.split(':', 1)[1].rsplit('/', 1)[0] + if relative.startswith(STEMS_FOLDER + '/') and relative.count('/') == 1: + shutil.rmtree(jobs.port.output_path(relative + '/x.mp4', create=False).parent, ignore_errors=True) + + +def _read(path, first, last): + """Mono float32 samples at 44.1 kHz of [first, last) seconds of a file, and the rate.""" + from ..stems.separate import read_audio + audio, _ = read_audio(path) + low, high = max(0, int(first * RATE)), min(audio.shape[1], int(last * RATE)) + return audio[:, low:high].mean(axis=0) + + +def _resample(samples, rate): + if rate == RATE: + return np.asarray(samples, dtype=np.float32) + from scipy.signal import resample_poly + divisor = gcd(RATE, int(rate)) + return resample_poly(np.asarray(samples, dtype=np.float32), RATE // divisor, int(rate) // divisor).astype(np.float32) + + +# ---------------------------------------------------------------------------------------------- voice_lines + +def run_voice_lines(jobs, row): + """The lines of a selection: where each one is (from the voice) and what it says (from the recogniser).""" + from .. import subtitle_ears + from .. import subtitle_sherpa # noqa: F401 registers the recogniser + from ..voice import sentences + spec = row['spec'] + ctx = context(jobs, spec) + sources = None + path = ctx['path'] + try: + if spec.get('separate_first') and not ctx['separated']: + sources, path = _separate(jobs, row, ctx) + _cancelled(row) + try: + engine = subtitle_ears.engine_for('transcribe') + except subtitle_ears.EarsError as error: + raise JobError('no_speech_model', 'No speech model is installed on this machine') from error + _phase(row, 'recognizing', fraction=0.) + + def cancelled(): + return bool(row.get('_cancel')) + + def progress(fraction): + _phase(row, 'recognizing', fraction=round(float(fraction), 3)) + try: + whole = subtitle_ears.decode_mono(path, cancelled=cancelled) + rate = subtitle_ears.RATE + low, high = int((ctx['base'] + ctx['start']) * rate), int((ctx['base'] + ctx['end']) * rate) + audio = np.ascontiguousarray(whole[low:high]) + heard = engine.transcribe(audio, spec.get('language', 'auto'), progress, cancelled) if len(audio) > rate * .2 \ + else dict(language=spec.get('language', 'auto'), lines=[]) + except subtitle_ears.Cancelled as error: + raise JobError('cancelled', 'Cancelled') from error + except subtitle_ears.EarsError as error: + raise JobError(('invalid_request' if error.reason in ('no_sound', 'too_long') else 'voice_failed'), + 'The selection could not be recognised (%s)' % error.reason) from error + words = [word for line in heard['lines'] for word in line['words']] + rows = sentences.lines(words, audio, rate) + lines = [] + for index, line in enumerate(rows, 1): + lines.append(dict(id='l%03d' % index, start=round(line['start'], 3), end=round(line['end'], 3), text=line['text'], + speaker=0, words=line['words'], + rate=round(max(len(line['words']), 1) / max(line['end'] - line['start'], .1), 2))) + kind = 'dialogue' if ctx['separated'] or sources else 'a1' + return dict(editor_project=spec['editor_project'], revision=spec['revision'], separated=ctx['separated'], + language=heard.get('language'), speech=bool(lines), lines=lines, source=kind, + recognition=remember(spec, ctx, spec.get('language', 'auto'), lines) if lines else None) + finally: + _drop_folder(jobs, sources) + + +# ---------------------------------------------------------------------------------------------- voice_replace + + + +def _seed(spec, lines): + if spec.get('seed') is not None: + return spec['seed'] + text = '|'.join([spec['editor_project']] + ['%s=%s' % (line['id'], line['text']) for line in lines]) + return int.from_bytes(hashlib.sha256(text.encode()).digest()[:4], 'big') % (2 ** 30) + + +class Pace: + """Remaining seconds of the job: the average of the lines done so far, or the estimate before the first.""" + def __init__(self, lines): + self.guess = {row['id']: row['seconds'] for row in estimate(lines)['lines']} + self.order = [line['id'] for line in lines] + self.times, self.mark_time = [], time.monotonic() + + def mark(self): + now = time.monotonic() + self.times.append(now - self.mark_time) + self.mark_time = now + + def left(self, done): + if self.times: + return round(sum(self.times) / len(self.times) * max(0, len(self.order) - done), 1) + return round(sum(self.guess.get(key, 0.) for key in self.order[done:]), 1) + + +def _round(jobs, row, items, reference, seed, pace, base, count, done, poll=.25): + """One dubbing job for `items` ([{key, text, seconds, onset, offset, reference?}]). + + Returns ({key: line row with 'data' (mono float32, 44.1 kHz) when a file was made}, cancelled).""" + from ..dubbing import wavio + service = getattr(jobs.port.server, '_freevideo_dubbing', None) + if service is None: + raise JobError('voice_model_missing', 'The dubbing model is not available on this machine') + names = {'d%d' % index: item['key'] for index, item in enumerate(items)} + lines = [dict({k: v for k, v in item.items() if k != 'key'}, id=name) for name, item in zip(names, items)] + value = dict(lines=lines, reference=reference, seed=seed) + waits, began = 0, None + while True: + _cancelled(row) + try: + key = service.start('dub', value, trusted=True, keep=True)['job'] + began = None + except ValueError as error: + if str(error) == 'busy': + began = began or time.monotonic() + _phase(row, 'waiting', waited_seconds=int(time.monotonic() - began)) + time.sleep(poll * 4) + continue + raise JobError(DUB_CODES.get(str(error), 'voice_failed'), 'The voice could not be made (%s)' % error) from None + cancelled, seen = False, 0 + while True: + if row.get('_cancel') and not cancelled: + service.cancel(key) + cancelled = True + try: + state = service.status(key) + except ValueError: + raise JobError('voice_failed', 'The dubbing service lost the job') from None + phase = state.get('phase') + folder = service.folder + made = state.get('lines') or [] + while seen < len(made): + pace.mark() + seen += 1 + if phase in ('complete', 'failed', 'cancelled'): + break + if not cancelled: + if phase in ('starting', 'preparing'): + _phase(row, 'preparing') + elif phase in ('loading', 'loaded'): + _phase(row, 'loading') + else: + index = min(count, base + seen + 1) + _phase(row, 'dubbing', index=index, count=count, remaining_seconds=pace.left(base + seen), + line=pace.order[index - 1], done=done + [names.get(item['id']) for item in made]) + time.sleep(poll) + waited = time.monotonic() + while service.busy() and time.monotonic() - waited < 60: + time.sleep(.1) + try: + if phase == 'failed': + if state.get('error') == 'busy' and waits < BUSY_WAITS: + waits += 1 + waited = began = time.monotonic() + while _busy(jobs) and time.monotonic() - waited < 600: + _cancelled(row) + _phase(row, 'waiting', waited_seconds=int(time.monotonic() - began)) + time.sleep(poll * 4) + began = None + continue + error = state.get('error') + raise JobError(DUB_CODES.get(error, 'voice_failed'), 'The voice could not be made (%s)' % error) + rows = {} + root = Path(service.input_directory()) + for item in made: + target = dict(item, key=names[item['id']]) + if item.get('file') and not item.get('code'): + samples, rate = wavio.read(root / item['file']) + target['data'] = _resample(samples[:, 0], rate) + rows[target['key']] = target + return rows, cancelled + finally: + if folder: + shutil.rmtree(folder, ignore_errors=True) + + +def _measure(audio, first, last): + """(onset, offset) in file seconds of the sound in [first, last) of a mono 44.1 kHz file, or None.""" + from ..dubbing import align + low, high = max(0, int(first * RATE)), min(len(audio), int(last * RATE)) + span = align.energy_span(audio[low:high], RATE) + if span is None: + return None + return (low + span[0]) / RATE, (low + span[1]) / RATE + + +def _windows(every, todo, spans, base, limit): + """Where each changed line's new voice goes: the original line plus a little quiet each side, and how far it may reach.""" + windows, failures = {}, {} + for position, line in enumerate(every): + if line not in todo: + continue + onset, offset = spans[line['id']] + before = spans[every[position - 1]['id']][1] if position else None + after = spans[every[position + 1]['id']][0] if position + 1 < len(every) else limit + CLEARANCE + lead = min(GAP_BEFORE, max(0., (onset - before) / 2)) if before is not None else GAP_BEFORE + trail = min(GAP_AFTER, max(0., (after - offset) / 2)) + first, last = max(0., onset - lead), offset + trail + if offset - onset < MIN_WINDOW / 2: + failures[line['id']] = 'failed' + continue + # nothing of this window reaches the quiet kept before the next line (where that one's own window starts) + hard = after if position + 1 == len(every) else after - min(GAP_BEFORE, max(0., (after - offset) / 2)) + last = min(max(last, first + MIN_WINDOW), hard) + windows[line['id']] = dict(first=first, last=last, onset=onset, offset=offset, hard=hard, + deadline=max(last, after - CLEARANCE)) + return windows, failures + + +def _reference(jobs, spec, voice, every, original_wav, base): + """The job's reference (one that is always there), and whether lines keep their own.""" + from ..editor_export import resolve_source + ref = voice.get('reference') + if ref: + project = jobs.projects.get(spec['editor_project'])['project'] + other = next((clip for track in _tracks(project) for clip in track['clips'] if clip['id'] == ref['clip_id']), None) + if other is None or not other.get('source'): + raise JobError('invalid_request', 'The reference clip is not in this project') + try: + path = resolve_source('%s:%s' % (other['source']['kind'], other['source']['id']))['audio'] + except (OSError, ValueError): + path = None + if path is None: + raise JobError('invalid_request', 'The reference clip has no sound') + picked = dict(file=str(path), text=ref.get('text', ''), start=other.get('in', 0) / FPS + float(ref.get('start', 0.))) + if ref.get('end') is not None: + picked['end'] = other.get('in', 0) / FPS + float(ref['end']) + return picked, False + longest = max(every, key=lambda line: line['end'] - line['start']) + return dict(file=str(original_wav), text=longest['original_text'].strip(), start=base + longest['start'], + end=base + longest['end']), True + + +def run_voice_replace(jobs, row, *, poll=.25): + from ..dubbing import align + from ..stems.separate import read_audio, write_mp4, write_wav + from ..voice import assemble + started = time.monotonic() + spec, port = row['spec'], jobs.port + voice = spec.get('voice') or {} + if voice.get('preset'): + raise JobError('invalid_request', 'Preset voices are not offered yet') + ctx = context(jobs, spec) + every = sorted(spec['lines'], key=lambda line: line['start']) + todo = edited(every) + if not todo: + raise JobError('invalid_request', 'No line was changed') + sources, separate_seconds, out = None, None, None + try: + separated_now = not ctx['separated'] + expect = signature(ctx['parent'] if separated_now else ctx['dialogue']) + if separated_now: + began = time.monotonic() + sources, dialogue_wav = _separate(jobs, row, ctx) + separate_seconds = round(time.monotonic() - began, 1) + original_wav, original_source = dialogue_wav, sources['dialogue'] + else: + dialogue_wav = ctx['path'] + original_wav, original_source = original_of(ctx, port) + _cancelled(row) + _phase(row, 'preparing') + audio, total = read_audio(dialogue_wav) + mono = audio.mean(axis=0) + base = ctx['base'] + spans = {} + for position, line in enumerate(every): + first, last = base + line['start'], base + line['end'] + low, high = first - MARGIN, last + MARGIN + # a neighbour closer than the margin is not this line's sound: stop halfway between the two lines + if position and base + every[position - 1]['end'] > low: + low = min(first, (base + every[position - 1]['end'] + first) / 2) + if position + 1 < len(every) and base + every[position + 1]['start'] < high: + high = max(last, (last + base + every[position + 1]['start']) / 2) + spans[line['id']] = _measure(mono, low, high) or (first, last) + windows, failures = _windows(every, todo, spans, base, min(base + ctx['end'], total)) + work = [line for line in todo if line['id'] in windows] + if not work: + raise JobError('voice_failed', 'None of the changed lines can be dubbed') + reference, own = _reference(jobs, spec, voice, every, original_wav, base) + seed = _seed(spec, todo) + pace = Pace(work) + + def item(line, length, offset): + window = windows[line['id']] + made = dict(key=line['id'], text=line['text'].strip(), seconds=round(length, 3), + onset=round(window['onset'] - window['first'], 3), offset=round(offset - window['first'], 3)) + if own and line['end'] - line['start'] >= 1. and line['original_text'].strip(): + made['reference'] = dict(file=str(original_wav), text=line['original_text'].strip(), + start=base + line['start'], end=base + line['end']) + return made + + first_round, cancelled = _round(jobs, row, [item(line, windows[line['id']]['last'] - windows[line['id']]['first'], + windows[line['id']]['offset']) for line in work], + reference, seed, pace, 0, len(work), [], poll) + # a line that is too long for its own window may reach into the quiet before the next original line + second_round, retry = {}, [] + if not cancelled: + for line in work: + made, window = first_round.get(line['id']), windows[line['id']] + if made and 'data' in made and made.get('warning') == 'too_long' and window['deadline'] > window['last'] + .02: + shortest = window['offset'] - window['onset'] + float(made.get('overrun') or 0.) + room = window['deadline'] - window['onset'] + if room >= shortest * .99: + natural = shortest * float(made.get('stretch') or 1.) * float(made.get('speed') or 1.) + retry.append((line, min(natural, room))) + if retry: + count = len(work) + len(retry) + pace.order = [line['id'] for line in work] + [line['id'] for line, _ in retry] + second_round, cancelled = _round( + jobs, row, [item(line, windows[line['id']]['deadline'] - windows[line['id']]['first'], + windows[line['id']]['onset'] + span) for line, span in retry], + reference, seed, pace, len(work), count, list(first_round), poll) + unfinished = {line['id'] for line, _ in retry} - set(second_round) if cancelled else set() + # what was made goes into the separated voice, and every changed line gets its measured result + _phase(row, 'placing') + edits, results = [], [] + for line in every: + if line not in todo: + continue + result = dict(id=line['id'], text=line['text'].strip(), start=line['start'], end=line['end'], + replaced=False, overrun=0., fitted=False, warning=None) + results.append(result) + if line['id'] in failures: + result['code'] = failures[line['id']] + continue + window = windows[line['id']] + made = second_round.get(line['id']) or first_round.get(line['id']) + if made is None: + result['code'] = 'cancelled' + continue + result.update(available=round((window['deadline'] if line['id'] in second_round else window['last']) - window['onset'], 3), + speed=made.get('speed'), stretch=made.get('stretch'), gain_db=made.get('gain_db'), + overrun=round(float(made.get('overrun') or 0.), 3), warning=made.get('warning')) + if made.get('code') or 'data' not in made: + result['code'] = made.get('code') or 'failed' + continue + if made.get('overrun') or made.get('warning') == 'too_long': + result['warning'] = 'too_long' + continue + position = int(round(window['first'] * RATE)) + data = made['data'][:max(0, int(window['hard'] * RATE) - position)] + span = align.energy_span(data, RATE) + if span is None: + result['code'] = 'failed' + continue + first, last = (position + span[0]) / RATE, (position + span[1]) / RATE + onset_error, offset_error = first - window['onset'], last - window['offset'] + # the fades stay in the quiet: the original sound is before its onset and after its offset + fades = (min(assemble.CROSSFADE, max(0., window['onset'] - window['first'])), + min(assemble.CROSSFADE, max(0., (position + len(data)) / RATE - window['offset']))) + edits.append((position, data, fades)) + result.update(replaced=True, onset=round(first - base, 3), offset=round(last - base, 3), + onset_error=round(onset_error, 3), offset_error=round(offset_error, 3), + fitted=abs(onset_error) <= align.FIT_SECONDS and abs(offset_error) <= align.FIT_SECONDS) + if not result['fitted'] and not result['warning'] and offset_error < -align.FIT_SECONDS: + # the line as placed ends before its slot does, whatever the dubbing measured before the cut + result['warning'] = 'too_short' + replaced = [item for item in results if item['replaced']] + if cancelled and not replaced: + raise JobError('cancelled', 'Cancelled') + if not replaced and not any(item['warning'] == 'too_long' for item in results): + code = next((item['code'] for item in results if item.get('code')), 'failed') + raise JobError(DUB_CODES.get(code, 'voice_failed'), 'No changed line could be dubbed (%s)' % code) + if not replaced: + # every changed line was too long: the voice would be the original again, so nothing is written or placed, + # and a separation made for this job goes too (shorten the lines, then change again) + _drop_folder(jobs, sources) + seconds = round(time.monotonic() - started, 1) + return dict(editor_project=spec['editor_project'], revision=spec['revision'], + clip_id=ctx['dialogue']['id'] if ctx['dialogue'] else None, + sources=dict(new=None, original=None if separated_now else original_source), separated_now=False, parts=[], + placed=False, reason='too_long', cancelled=bool(cancelled), + partial=any(item.get('code') == 'cancelled' for item in results) or bool(unfinished), + kept=len(every) - len(todo), failed=sum(1 for item in results if item.get('code') and item['code'] != 'cancelled'), + lines=results, seed=seed, apply=None, seconds=seconds, separate_seconds=separate_seconds, + dub_seconds=round(seconds - (separate_seconds or 0.), 1)) + stitched = assemble.stitch(audio.T, edits, RATE).T + folder = '%s/%s-%s' % (FOLDER, row['id'], secrets.token_hex(2)) + out = port.output_path(folder + '/dialogue.mp4', create=True).parent + write_wav(out / 'dialogue.wav', stitched, exact=True) + write_mp4(out / 'dialogue.mp4', stitched) + new_source = 'generated:%s/dialogue.mp4' % folder + (out / 'voice.json').write_text(json.dumps(dict(version=1, original=original_source, base=base, seed=seed, + lines=results), ensure_ascii=False), encoding='utf-8') + revision, parts = spec['revision'], [] + clip_id = ctx['dialogue']['id'] if ctx['dialogue'] else None + laid = dict(sources, dialogue=new_source) if separated_now else None + apply = dict(op='placeStems', clip_id=ctx['parent']['id'], sources=laid) if separated_now \ + else dict(op='replaceSource', clip_id=clip_id, source=new_source) + placed, reason = False, None + if spec.get('lay', True): + # the project may have moved on while the voice was made: it is placed only if the clip it goes on is as it was + try: + if separated_now: + saved, clip_ids = jobs.projects.lay_stems(spec['editor_project'], ctx['parent']['id'], laid, expect=expect) + fresh = set(clip_ids) + parts = [dict(track=track['id'], name=track['stem'], clip_id=clip['id'], start=clip['start'], + frames=clip['frames'], end=clip['start'] + clip['frames']) + for track in saved['tracks'] if track.get('stem') for clip in track['clips'] if clip['id'] in fresh] + clip_id = next((part['clip_id'] for part in parts if part['name'] == 'dialogue'), None) + else: + saved = jobs.projects.replace_voice_source(spec['editor_project'], clip_id, new_source, expect=expect) + revision, placed, apply = saved['revision'], True, None + except JobError as error: + if error.code != 'project_changed': + raise + # nothing was changed: the files stay, and the result says how to put them on the project as it is now + try: + revision = jobs.projects.get(spec['editor_project'])['project']['revision'] + except FileNotFoundError as missing: + raise JobError('not_found', 'The project does not exist') from missing + reason = 'project_changed' + else: + # the page places it itself: the same check, read only, tells it whether the clip is still as it was + from .projects import _as_expected + try: + current = jobs.projects.get(spec['editor_project'])['project'] + except FileNotFoundError as missing: + raise JobError('not_found', 'The project does not exist') from missing + if not _as_expected(current, ctx['parent']['id'] if separated_now else ctx['dialogue']['id'], expect): + revision, reason = current['revision'], 'project_changed' + seconds = round(time.monotonic() - started, 1) + return dict(editor_project=spec['editor_project'], revision=revision, clip_id=clip_id, + sources=dict(new=new_source, original=original_source), separated_now=separated_now, parts=parts, + placed=placed, reason=reason, cancelled=bool(cancelled), + partial=any(item.get('code') == 'cancelled' for item in results) or bool(unfinished), kept=len(every) - len(todo), + failed=sum(1 for item in results if item.get('code') and item['code'] != 'cancelled'), + lines=results, seed=seed, apply=apply, seconds=seconds, separate_seconds=separate_seconds, + dub_seconds=round(seconds - (separate_seconds or 0.), 1)) + except BaseException: + if out is not None: + shutil.rmtree(out, ignore_errors=True) + _drop_folder(jobs, sources) + raise diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index 7db24c3d..767d5987 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -869,7 +869,7 @@ def list_imports(root, unreadable=None): value = _read_import(root, child.name) stamp = _import_path(root, child.name).stat().st_mtime_ns fields = ('id', 'name', 'kind', 'state', 'fraction', 'reason', 'frames', 'fps', 'width', 'height', - 'preview_width', 'preview_height', 'has_audio') + 'preview_width', 'preview_height', 'has_audio', 'detail') row = {k: value.get(k) for k in fields} row['started'] = bool(value.get('started')) for side in ('width', 'height'): @@ -984,7 +984,7 @@ def transcode(root, identity): aout.codec_context.thread_count = 1 resampler = av.AudioResampler(format='fltp', layout='stereo', rate=32000) next_video, previous, previous_time, previous_end = 0, None, 0., 0. - audio_pos, decoded_audio = None, 0 + audio_pos, decoded_audio, damaged = None, 0, 0 last_update, fraction = 0., 0. def emit_video(until): @@ -1011,8 +1011,22 @@ def emit_audio(frame): for packet in aout.encode(frame): writer.mux(packet) - for packet in reader.demux([s for s in (vin, ain) if s is not None]): - for frame in packet.decode(): + packets = reader.demux([s for s in (vin, ain) if s is not None]) + while True: + try: + packet = next(packets) + except StopIteration: + break + except av.error.InvalidDataError: + # A file cut off or damaged part-way keeps what was read before the damage. + damaged += 1 + break + try: + frames = packet.decode() + except av.error.InvalidDataError: + damaged += 1 # Skip a packet the decoder rejects; the rest of the file still plays. + continue + for frame in frames: frame_time = Fraction(frame.pts) * frame.time_base - origin if frame.pts is not None else 0. if packet.stream == vin: picture = display_frame(frame) @@ -1064,6 +1078,8 @@ def emit_audio(frame): width=original_info['width'], height=original_info['height'], original_size=True) if kind == 'video': info['frames'] = next_video + if damaged: + _LOG.warning('Editor import kept the readable part of a damaged file (%d unreadable packets).', damaged) _import_update(root, identity, **info, state='ready', fraction=1., reason=None) finally: partial.unlink(missing_ok=True) @@ -1089,8 +1105,12 @@ def prepare(self, identity): except Exception as error: # A full disk is not the file's fault, so the page says so instead of listing formats. reason = _disk_full(error).get('reason', 'unreadable') + if reason == 'unreadable': + _LOG.warning('Editor import could not be read (%s).', type(error).__name__) # The message may carry a path. try: - _import_update(self.root, identity, state='failed', reason=reason) + # The class of the error (never its message, which can name a path) goes with the page's report button. + _import_update(self.root, identity, state='failed', reason=reason, + detail=type(error).__name__ if reason == 'unreadable' else None) except (OSError, ValueError): pass # The terminal snapshot is retained and retried by the worker/reads. diff --git a/freevideo_engine/comfy_setup.py b/freevideo_engine/comfy_setup.py index e71834c7..edfaa55c 100644 --- a/freevideo_engine/comfy_setup.py +++ b/freevideo_engine/comfy_setup.py @@ -69,6 +69,26 @@ def discover_libraries(folder_paths, environ=None): 'on wet stone.') +def inspection_failed(value): + """The first line of a failed inspection: what ran, how it ended and how much the child printed. + + Only integers, fixed words and an exception class name, so a report keeps it whole however the rest is cut.""" + code, failed = value.get('returncode'), value.get('failed_with') + parts = [value.get('action') if value.get('action') in ('plan', 'setup') else 'other'] + if isinstance(code, int): + # Windows reports a crash as a large unsigned code; its hex form is the one people search for + parts.append('exit %d' % code + (' (0x%08X)' % code if 0xFFFF < code <= 0xFFFFFFFF else '')) + else: + parts.append('no exit code') + parts.append('%d output lines' % int(value.get('lines') or 0)) + # A cancelled run never gets here: only a complete or failed plan is read, so cancelling is not a failure to report. + if failed in ('TimeoutExpired', 'TimeoutError'): + parts.append('timed out') + if isinstance(failed, str) and failed.isidentifier() and len(failed) <= 60: + parts.append(failed) + return 'Setup inspection failed (%s)' % ', '.join(parts) + + class SetupRunner(Runner): token = '' @@ -433,7 +453,7 @@ def status(self): self.plan = preflight_json(value['output']) self.state.update(plan=self.plan, plan_id=uuid.uuid4().hex) except ValueError: - self.state.update(status='failed', error='Setup inspection failed\n' + + self.state.update(status='failed', error=inspection_failed(value) + '\n' + (self.state.get('error') or value.get('output', ''))) elif value['action'] == 'setup' and value['status'] == 'complete': try: diff --git a/freevideo_engine/desktop_runtime.py b/freevideo_engine/desktop_runtime.py index 32e6ba02..b5de434e 100644 --- a/freevideo_engine/desktop_runtime.py +++ b/freevideo_engine/desktop_runtime.py @@ -308,6 +308,8 @@ def prepare(self, action, root, env, progress): def _run_awake(self, action, root, arguments): run = io_path(self.logs) / (time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) + '-' + uuid.uuid4().hex[:8]) code, output, error = None, '', None + # what a report can say when the child left no text: how much it printed, and what stopped the run + lines, failed_with = 0, None started = time.monotonic() try: run.mkdir(parents=True) @@ -360,6 +362,7 @@ def progress(message): if self.cancelled.is_set(): processes.stop(process) for line in process.stdout: + lines += 1 line = launch_view.line_back(plain_text(line), view) log.write(line) output = (output + line)[-2 * 1024 * 1024:] @@ -373,6 +376,7 @@ def progress(message): code = process.wait(timeout=15) process.stdout.close() except BaseException as failure: + failed_with = type(failure).__name__ error = plain_text(repr(failure)) detail = plain_text('\n' + traceback.format_exc()) output += detail @@ -386,7 +390,8 @@ def progress(message): processes.stop(self.process) finally: result = dict(action=action, status='cancelled' if self.cancelled.is_set() else 'complete' if code == 0 else 'failed', - returncode=code, error=error, wall_seconds=time.monotonic()-started, root=str(root), log=plain(run)) + returncode=code, error=error, wall_seconds=time.monotonic()-started, root=str(root), log=plain(run), + lines=lines, failed_with=failed_with) try: save(run / 'status.json', result) except OSError as failure: diff --git a/freevideo_engine/dubbing/__init__.py b/freevideo_engine/dubbing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/align.py b/freevideo_engine/dubbing/align.py new file mode 100644 index 00000000..5c3ec0d2 --- /dev/null +++ b/freevideo_engine/dubbing/align.py @@ -0,0 +1,238 @@ +"""Fit a synthesized line to a target duration, whatever backend made it. + +A backend exposes `synth(speed)` -> (mono float32 samples, sample rate); `speed` +above 1 makes the line shorter. The first pass is natural speed, the second +pass uses the measured ratio, and a third corrects what is still outside the +tolerance. The voiced part (first to last loud 5 ms frame) is then closed to the wanted length by a +pitch-preserving tempo change within `max_stretch`, and placed at the wanted onset in a file of exactly +the target number of samples: silence fills what is short. Voice that cannot fit is never cut: the file +is then longer than the target and reports `overrun` seconds (`fitted` false). A few milliseconds of the quiet +edge around the voiced part are kept, so soft consonants and the decay of the last sound survive. +""" +from fractions import Fraction + +import numpy as np + +TOLERANCE = 0.02 +FIT_SECONDS = 0.04 +FADE = 0.003 +EDGE = 0.02 +MEASURE_FRAME = 0.005 +MAX_STRETCH = 1.25 +SPEED_RANGE = (0.6, 1.8) + + +LEAD_KEEP = 0.05 +TRAIL_KEEP = 0.12 +SILENCE_LEVEL = 0.02 +SPAN_SHARE = 0.02 # a line starts and ends where the smoothed energy crosses this share of its peak +NOISE_MARGIN = 4. # the voiced level is at least this many times the noise floor (10th percentile of the frames) +NOISE_CAP = 0.1 # ... but never above this share of the loudest frame + + +def seconds(samples, rate): + return len(samples) / rate + + +def trim_silence(samples, rate, lead_keep=LEAD_KEEP, trail_keep=TRAIL_KEEP, level=SILENCE_LEVEL): + """Cut the quiet ends a model leaves around a line, keeping a short margin of each. + + The language model sometimes opens with seconds of silence; measuring that as speech would make + every pass miss the target. Quiet is below `level` times the loudest 10 ms frame. + """ + samples = np.asarray(samples, dtype=np.float32) + frame = max(1, rate // 100) + count = len(samples) // frame + if count < 2: + return samples + rms = np.sqrt((samples[:count * frame].reshape(count, frame) ** 2).mean(axis=1)) + loud = np.nonzero(rms > level * rms.max())[0] + if not len(loud): + return samples + start = max(0, int(loud[0] * frame - lead_keep * rate)) + end = min(len(samples), int((loud[-1] + 1) * frame + trail_keep * rate)) + return samples[start:end] + + +def stretch(samples, rate, ratio): + """Return `samples` played `ratio` times faster (ratio > 1 shortens), pitch kept.""" + import av + graph = av.filter.Graph() + source = graph.add_abuffer(format='flt', sample_rate=rate, layout='mono', time_base=Fraction(1, rate)) + tempo = graph.add('atempo', '%.6f' % ratio) + sink = graph.add('abuffersink') + source.link_to(tempo) + tempo.link_to(sink) + graph.configure() + frame = av.AudioFrame.from_ndarray(np.ascontiguousarray(samples, dtype=np.float32)[None, :], + format='flt', layout='mono') + frame.sample_rate, frame.pts = rate, 0 + graph.push(frame) + graph.push(None) + parts = [] + while True: + try: + parts.append(sink.pull().to_ndarray()[0]) + except (av.error.EOFError, av.error.BlockingIOError): + break + return np.concatenate(parts).astype(np.float32) if parts else np.zeros(0, np.float32) + + +def voiced_span(samples, rate, level=SILENCE_LEVEL): + """(first, end) sample of the voiced part: from the first to the last loud 5 ms frame, or None.""" + samples = np.asarray(samples, dtype=np.float32) + frame = max(1, int(rate * MEASURE_FRAME)) + count = len(samples) // frame + if count < 1: + return None + rms = np.sqrt((samples[:count * frame].reshape(count, frame) ** 2).mean(axis=1)) + if not rms.max() > 0: + return None + # A recording with music or room noise comes back with that noise floor: the level rises above it + # (never past NOISE_CAP of the loudest frame), and a lone loud frame is not voice. + threshold = min(max(level * rms.max(), NOISE_MARGIN * np.percentile(rms, 10)), NOISE_CAP * rms.max()) + above = rms > threshold + joined = above & (np.concatenate([above[1:], [False]]) | np.concatenate([[False], above[:-1]])) + loud = np.nonzero(joined if joined.any() else above)[0] + if not len(loud): + return None + return int(loud[0] * frame), int((loud[-1] + 1) * frame) + + +def energy_span(samples, rate, share=SPAN_SHARE): + """(first, end) sample where the 5 ms smoothed energy stays above `share` of its peak: the definition + the timeline of an original line is taken with. None for silence.""" + data = np.asarray(samples, dtype=np.float64) + window = max(1, int(rate * MEASURE_FRAME)) + if len(data) < window: + return None + total = np.concatenate([[0.], np.cumsum(data * data)]) + half = window // 2 + centre = np.arange(len(data)) + low, high = np.clip(centre - half, 0, len(data)), np.clip(centre - half + window, 0, len(data)) + level = np.sqrt((total[high] - total[low]) / window) + if not level.max() > 0: + return None + loud = np.nonzero(level > share * level.max())[0] + return int(loud[0]), int(loud[-1]) + 1 + + +def place(voice, rate, total, onset, lead=0, trail=0): + """`voice` (`lead` quiet samples, the voiced part, `trail` quiet samples) laid into silence of + round(total*rate) samples so that the voiced part starts at `onset`; returns (samples, overrun). + + `overrun` is the number of voiced samples past the end of the line. The file is then longer by that + much and nothing is cut; otherwise the quiet tail is trimmed to the exact length. Also returns the + voiced part's first and end sample in the result: the placement is known, so it is not measured again + (a line that fills its slot has no silence left, and the measure's noise floor would then be speech).""" + size = int(round(total * rate)) + voiced = len(voice) - lead - trail + start = int(round(onset * rate)) - lead + take = np.array(voice, dtype=np.float32) + if start < 0: + take, start, lead = take[-start:], 0, lead + start + overrun = max(0, start + lead + voiced - size) + if overrun <= int(MEASURE_FRAME * rate): # under the resolution of the measurement + overrun = 0 + take = take[:size - start] + length = max(size, start + len(take)) + first = start + max(lead, 0) + last = min(first + voiced, start + len(take)) + for width, edge in ((int(FADE * rate), 'in'), (int(FADE * rate), 'out')): + width = min(width, len(take)) + if width > 1: + ramp = np.linspace(0, 1, width, dtype=np.float32) + if edge == 'in': + take[:width] *= ramp + else: + take[-width:] *= ramp[::-1] + out = np.zeros(length, np.float32) + out[start:start + len(take)] = take + return out, overrun, first, last + + +def fit(synth, target, *, onset=0., offset=None, tolerance=TOLERANCE, max_stretch=MAX_STRETCH, + speed_range=SPEED_RANGE, check=lambda: None): + """Run the passes; return dict(samples, rate, seconds, onset, offset, error, onset_error, + offset_error, speed, passes, stretch, + fitted, warning, overrun, natural). `samples` has exactly round(target*rate) samples unless the voice overruns; the voiced part is + wanted from `onset` to `offset` (default: the whole line).""" + if not target > 0: + raise ValueError('Target duration must be positive') + offset = target if offset is None else offset + wanted = offset - onset + if not 0 <= onset < offset <= target + 1e-9: + raise ValueError('The wanted voiced part must lie inside the line') + low, high = speed_range + tried = [] + + def run(speed): + check() + samples, rate = synth(speed) + span = voiced_span(samples, rate) + if span is None: + raise ValueError('The dubbing backend returned no audio') + samples = np.asarray(samples, dtype=np.float32) + edge = int(EDGE * rate) + voice = samples[span[0]:span[1]] + # quiet samples around the voiced part, kept as they are (not stretched) + edges = (samples[max(0, span[0] - edge):span[0]], samples[span[1]:span[1] + edge]) + tried.append((speed, voice, rate, seconds(voice, rate), edges)) + return tried[-1] + + def off(entry): + return abs(entry[3] - wanted) / wanted + + speed = 1.0 + entry = run(speed) + while off(entry) > tolerance and len(tried) < 3: + # the length scales with 1/speed, so the next speed is the measured ratio + speed = min(max(speed * entry[3] / wanted, low), high) + if speed == tried[-1][0]: + break + entry = run(speed) + speed, voice, rate, length, edges = min(tried, key=off) + padded_edges = edges + + def compose(start, width): + """The voiced part closed to `width` seconds and placed from `start`; (out, overrun, first, last, ratio).""" + source, ratio, current, length_now = voice, 1.0, voice, length + for _ in range(2): + want = min(max(ratio * length_now / width, 1 / max_stretch), max_stretch) + if abs(want - ratio) < 0.001 and ratio != 1.0 or abs(length_now - width) <= 0.004 and ratio == 1.0: + break + ratio = want + current = stretch(source, rate, ratio) + length_now = seconds(current, rate) + padded = np.concatenate([padded_edges[0], current, padded_edges[1]]) + out, overrun, first, last = place(padded, rate, target, start, lead=len(padded_edges[0]), + trail=len(padded_edges[1])) + return out, overrun, first, last, ratio + + # The placed file is measured the way the originals' timeline is (smoothed energy above a share of its + # peak): the tempo change and the quiet edges spread a little energy past the voiced part, so the first + # placement is corrected by what is measured, twice at most. + start, width = onset, wanted + best = None + for attempt in range(3): + out, overrun, first, last, ratio = compose(start, width) + span = energy_span(out, rate) + first, last = (span[0], span[1]) if span else (first, last) + first, last = first / rate, last / rate + miss = max(abs(first - onset), abs(last - offset)) + (overrun > 0) + if best is None or miss < best[0]: + best = (miss, out, overrun, first, last, ratio) + if miss <= 0.004 or attempt == 2: + break + start = max(0., start + (onset - first)) + width = max(0.25 * wanted, width + (wanted - (last - first))) + _, out, overrun, first, last, ratio = best + error = (last - first) - wanted + onset_error, offset_error = first - onset, last - offset + overrun = overrun / rate + fitted = abs(onset_error) <= FIT_SECONDS and abs(offset_error) <= FIT_SECONDS and not overrun + warning = None if fitted else 'too_long' if (overrun or offset_error > 0) else 'too_short' + return dict(samples=out, rate=rate, seconds=len(out) / rate, onset=first, offset=last, error=error, + onset_error=onset_error, offset_error=offset_error, + speed=speed, passes=len(tried), stretch=ratio, fitted=fitted, warning=warning, overrun=overrun, + natural=tried[0][3]) diff --git a/freevideo_engine/dubbing/backends.py b/freevideo_engine/dubbing/backends.py new file mode 100644 index 00000000..3ceed521 --- /dev/null +++ b/freevideo_engine/dubbing/backends.py @@ -0,0 +1,63 @@ +"""Dubbing backends behind one interface, so a model can be replaced without touching the service. + +A backend is built from its model folder, says a line in the voice of one reference recording and +frees its memory in `close`. `speed` above 1 makes the line shorter; `align.fit` uses it to reach a +target duration. +""" +import importlib + +REGISTRY = { + 'cosyvoice3': 'freevideo_engine.dubbing.backends:CosyVoice3Backend', +} +DEFAULT = 'cosyvoice3' +MAX_REFERENCE_SECONDS = 29.0 + + +class Backend: + name = '' + sample_rate = 24000 + languages = () + # Whether the reference needs a transcript of what it says + needs_reference_text = True + supports_instruction = False + + def __init__(self, model_dir, device='cuda'): + raise NotImplementedError + + def synth(self, text, reference, reference_text, speed=1.0, instruction=None): + """Return (mono float32 samples, sample rate).""" + raise NotImplementedError + + def close(self): + pass + + +class CosyVoice3Backend(Backend): + name = 'cosyvoice3' + sample_rate = 24000 + languages = ('zh', 'en', 'ja', 'ko', 'de', 'es', 'fr', 'it', 'ru') + supports_instruction = True + + def __init__(self, model_dir, device='cuda'): + from .cosyvoice3.build import CosyVoice3 + self.model = CosyVoice3(model_dir, device=device) + + def synth(self, text, reference, reference_text, speed=1.0, instruction=None): + if instruction: + samples = self.model.instruct(text, instruction, reference, speed=speed) + else: + samples = self.model.zero_shot(text, reference_text, reference, speed=speed) + return samples, self.sample_rate + + def close(self): + self.model = None + + +def names(): + return sorted(REGISTRY) + + +def create(name, model_dir, device='cuda'): + """Build the named backend; an unknown name raises KeyError.""" + module, _, cls = REGISTRY[name].partition(':') + return getattr(importlib.import_module(module), cls)(model_dir, device=device) diff --git a/freevideo_engine/dubbing/cosyvoice3/__init__.py b/freevideo_engine/dubbing/cosyvoice3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/build.py b/freevideo_engine/dubbing/cosyvoice3/build.py new file mode 100644 index 00000000..6e4c6a2a --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/build.py @@ -0,0 +1,115 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Build Fun-CosyVoice3-0.5B from its published weights without hyperpyyaml. + +The parameters below are those of the model's cosyvoice3.yaml. +""" +import os +from functools import partial + +import torch +from omegaconf import DictConfig + +from freevideo_engine.dubbing.cosyvoice3.cli.frontend import CosyVoiceFrontEnd +from freevideo_engine.dubbing.cosyvoice3.cli.model import CosyVoice3Model +from freevideo_engine.dubbing.cosyvoice3.flow.DiT.dit import DiT +from freevideo_engine.dubbing.cosyvoice3.flow.flow import CausalMaskedDiffWithDiT +from freevideo_engine.dubbing.cosyvoice3.flow.flow_matching import CausalConditionalCFM +from freevideo_engine.dubbing.cosyvoice3.hifigan.f0_predictor import CausalConvRNNF0Predictor +from freevideo_engine.dubbing.cosyvoice3.hifigan.generator import CausalHiFTGenerator +from freevideo_engine.dubbing.cosyvoice3.llm.llm import CosyVoice3LM, Qwen2Encoder +from freevideo_engine.dubbing.cosyvoice3.tokenizer.tokenizer import get_qwen_tokenizer +from freevideo_engine.dubbing.cosyvoice3.transformer.upsample_encoder import PreLookaheadLayer +from freevideo_engine.dubbing.cosyvoice3.utils.common import ras_sampling +from freevideo_engine.dubbing.cosyvoice3.utils.mel import mel_spectrogram + +SAMPLE_RATE = 24000 +QWEN_DIR = 'CosyVoice-BlankEN' +PROMPT_PREFIX = 'You are a helpful assistant.<|endofprompt|>' + + +def build_networks(qwen_dir): + llm = CosyVoice3LM( + llm_input_size=896, llm_output_size=896, speech_token_size=6561, + llm=Qwen2Encoder(qwen_dir), + sampling=partial(ras_sampling, top_p=0.8, top_k=25, win_size=10, tau_r=0.1), + mix_ratio=[5, 15]) + flow = CausalMaskedDiffWithDiT( + input_size=80, output_size=80, spk_embed_dim=192, output_type='mel', vocab_size=6561, + input_frame_rate=25, only_mask_loss=True, token_mel_ratio=2, pre_lookahead_len=3, + pre_lookahead_layer=PreLookaheadLayer(in_channels=80, channels=1024, pre_lookahead_len=3), + decoder=CausalConditionalCFM( + in_channels=240, n_spks=1, spk_emb_dim=80, + cfm_params=DictConfig({'sigma_min': 1e-06, 'solver': 'euler', 't_scheduler': 'cosine', + 'training_cfg_rate': 0.2, 'inference_cfg_rate': 0.7, 'reg_loss_type': 'l1'}), + estimator=DiT(dim=1024, depth=22, heads=16, dim_head=64, ff_mult=2, mel_dim=80, mu_dim=80, + spk_dim=80, out_channels=80, static_chunk_size=50, num_decoding_left_chunks=-1))) + hift = CausalHiFTGenerator( + in_channels=80, base_channels=512, nb_harmonics=8, sampling_rate=SAMPLE_RATE, + nsf_alpha=0.1, nsf_sigma=0.003, nsf_voiced_threshold=10, + upsample_rates=[8, 5, 3], upsample_kernel_sizes=[16, 11, 7], + istft_params={'n_fft': 16, 'hop_len': 4}, + resblock_kernel_sizes=[3, 7, 11], resblock_dilation_sizes=[[1, 3, 5]] * 3, + source_resblock_kernel_sizes=[7, 7, 11], source_resblock_dilation_sizes=[[1, 3, 5]] * 3, + lrelu_slope=0.1, audio_limit=0.99, conv_pre_look_right=4, + f0_predictor=CausalConvRNNF0Predictor(num_class=1, in_channels=80, cond_channels=512)) + return llm, flow, hift + + +def _feat_extractor(): + return partial(mel_spectrogram, n_fft=1920, num_mels=80, sampling_rate=SAMPLE_RATE, hop_size=480, + win_size=1920, fmin=0, fmax=None, center=False) + + +class CosyVoice3: + """Zero-shot voice cloning with Fun-CosyVoice3-0.5B; every call returns one float32 numpy waveform.""" + + sample_rate = SAMPLE_RATE + + def __init__(self, model_dir, device='cuda', fp16=False): + qwen_dir = os.path.join(model_dir, QWEN_DIR) + self.frontend = CosyVoiceFrontEnd( + lambda: get_qwen_tokenizer(token_path=qwen_dir, skip_special_tokens=True, version='cosyvoice3'), + _feat_extractor(), + os.path.join(model_dir, 'campplus.onnx'), + os.path.join(model_dir, 'speech_tokenizer_v3.onnx'), + device=device) + llm, flow, hift = build_networks(qwen_dir) + self.model = CosyVoice3Model(llm, flow, hift, device, fp16) + self.model.load(os.path.join(model_dir, 'llm.pt'), os.path.join(model_dir, 'flow.pt'), + os.path.join(model_dir, 'hift.pt')) + + def _run(self, texts, make_input, speed): + import numpy as np + parts = [] + for piece in self.frontend.text_normalize(texts, split=True, text_frontend=True): + inp = make_input(piece) + speech = self.model.tts( + text=inp['text'], flow_embedding=inp['flow_embedding'], llm_embedding=inp['llm_embedding'], + prompt_text=inp['prompt_text'], + llm_prompt_speech_token=inp.get('llm_prompt_speech_token', torch.zeros(1, 0, dtype=torch.int32)), + flow_prompt_speech_token=inp['flow_prompt_speech_token'], + prompt_speech_feat=inp['prompt_speech_feat'], speed=speed) + parts.append(speech.squeeze(0).float().cpu().numpy()) + return np.concatenate(parts).astype('float32') + + def zero_shot(self, text, prompt_text, prompt_wav, speed=1.0): + """Say `text` in the voice of `prompt_wav`, whose transcript is `prompt_text`.""" + prompt = PROMPT_PREFIX + prompt_text + return self._run(text, lambda t: self.frontend.frontend_zero_shot(t, prompt, prompt_wav, SAMPLE_RATE), speed) + + def instruct(self, text, instruction, prompt_wav, speed=1.0): + """Say `text` in the voice of `prompt_wav` in the manner `instruction` (a Chinese sentence) describes.""" + prompt = f'You are a helpful assistant. {instruction}<|endofprompt|>' + return self._run(text, lambda t: self.frontend.frontend_instruct2(t, prompt, prompt_wav, SAMPLE_RATE), speed) diff --git a/freevideo_engine/dubbing/cosyvoice3/cli/__init__.py b/freevideo_engine/dubbing/cosyvoice3/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/cli/frontend.py b/freevideo_engine/dubbing/cosyvoice3/cli/frontend.py new file mode 100644 index 00000000..3781db47 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/cli/frontend.py @@ -0,0 +1,153 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified by FreeVideo: reduced to the zero-shot and instruct2 inputs; onnxruntime, wetext and +# inflect are optional so the package imports without them. +import logging +import re +from functools import partial +from typing import Callable + +import numpy as np +import torch +import torchaudio.compliance.kaldi as kaldi + +from freevideo_engine.dubbing.cosyvoice3.utils.file_utils import load_wav +from freevideo_engine.dubbing.cosyvoice3.utils.frontend_utils import ( + contains_chinese, is_only_punctuation, remove_bracket, replace_blank, replace_corner_mark, + spell_out_number, split_paragraph) +from freevideo_engine.dubbing.cosyvoice3.utils.mel import log_mel_spectrogram + +logger = logging.getLogger(__name__) + + +class CosyVoiceFrontEnd: + + def __init__(self, + get_tokenizer: Callable, + feat_extractor: Callable, + campplus_model: str, + speech_tokenizer_model: str, + device='cpu'): + import onnxruntime + self.tokenizer = get_tokenizer() + self.feat_extractor = feat_extractor + self.device = torch.device(device) + option = onnxruntime.SessionOptions() + option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + option.intra_op_num_threads = 1 + # both small networks run on the CPU so they take no video memory next to the three generators + self.campplus_session = onnxruntime.InferenceSession( + campplus_model, sess_options=option, providers=['CPUExecutionProvider']) + self.speech_tokenizer_session = onnxruntime.InferenceSession( + speech_tokenizer_model, sess_options=option, providers=['CPUExecutionProvider']) + self.inflect_parser = None + try: + import inflect + self.inflect_parser = inflect.engine() + except Exception: + logger.info('inflect is not available, digits in English text are kept as written') + self.zh_tn_model = self.en_tn_model = None + try: + from wetext import Normalizer + self.zh_tn_model = Normalizer(remove_erhua=False) + self.en_tn_model = Normalizer() + except Exception: + logger.info('wetext is not available, text is not normalized') + + def _extract_text_token(self, text): + text_token = self.tokenizer.encode(text) + text_token = torch.tensor([text_token], dtype=torch.int32).to(self.device) + text_token_len = torch.tensor([text_token.shape[1]], dtype=torch.int32).to(self.device) + return text_token, text_token_len + + def _extract_speech_token(self, prompt_wav): + speech = load_wav(prompt_wav, 16000) + assert speech.shape[1] / 16000 <= 30, 'do not support extract speech token for audio longer than 30s' + feat = log_mel_spectrogram(speech, n_mels=128) + speech_token = self.speech_tokenizer_session.run( + None, {self.speech_tokenizer_session.get_inputs()[0].name: feat.detach().cpu().numpy(), + self.speech_tokenizer_session.get_inputs()[1].name: np.array([feat.shape[2]], dtype=np.int32)} + )[0].flatten().tolist() + speech_token = torch.tensor([speech_token], dtype=torch.int32).to(self.device) + speech_token_len = torch.tensor([speech_token.shape[1]], dtype=torch.int32).to(self.device) + return speech_token, speech_token_len + + def _extract_spk_embedding(self, prompt_wav): + speech = load_wav(prompt_wav, 16000) + feat = kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000) + feat = feat - feat.mean(dim=0, keepdim=True) + embedding = self.campplus_session.run( + None, {self.campplus_session.get_inputs()[0].name: feat.unsqueeze(dim=0).cpu().numpy()} + )[0].flatten().tolist() + return torch.tensor([embedding]).to(self.device) + + def _extract_speech_feat(self, prompt_wav): + speech = load_wav(prompt_wav, 24000) + speech_feat = self.feat_extractor(speech).squeeze(dim=0).transpose(0, 1).to(self.device) + speech_feat = speech_feat.unsqueeze(dim=0) + speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device) + return speech_feat, speech_feat_len + + def text_normalize(self, text, split=True, text_frontend=True): + # NOTE skip the text frontend when a control symbol is in the text + if '<|' in text and '|>' in text: + text_frontend = False + if text_frontend is False or text == '': + return [text] if split is True else text + text = text.strip() + if contains_chinese(text): + if self.zh_tn_model is not None: + text = self.zh_tn_model.normalize(text) + text = text.replace("\n", "") + text = replace_blank(text) + text = replace_corner_mark(text) + text = text.replace(".", "。") + text = text.replace(" - ", "，") + text = remove_bracket(text) + text = re.sub(r'[，,、]+$', '。', text) + lang = 'zh' + else: + if self.en_tn_model is not None: + text = self.en_tn_model.normalize(text) + if self.inflect_parser is not None: + text = spell_out_number(text, self.inflect_parser) + lang = 'en' + texts = list(split_paragraph(text, self.tokenizer.encode, lang, token_max_n=80, + token_min_n=60, merge_len=20, comma_split=False)) + texts = [i for i in texts if not is_only_punctuation(i)] + return texts if split is True else text + + def frontend_zero_shot(self, tts_text, prompt_text, prompt_wav, resample_rate=24000): + tts_text_token, tts_text_token_len = self._extract_text_token(tts_text) + prompt_text_token, prompt_text_token_len = self._extract_text_token(prompt_text) + speech_feat, speech_feat_len = self._extract_speech_feat(prompt_wav) + speech_token, speech_token_len = self._extract_speech_token(prompt_wav) + if resample_rate == 24000: + # force speech_feat % speech_token = 2 + token_len = min(int(speech_feat.shape[1] / 2), speech_token.shape[1]) + speech_feat, speech_feat_len[:] = speech_feat[:, :2 * token_len], 2 * token_len + speech_token, speech_token_len[:] = speech_token[:, :token_len], token_len + embedding = self._extract_spk_embedding(prompt_wav) + return {'prompt_text': prompt_text_token, 'prompt_text_len': prompt_text_token_len, + 'llm_prompt_speech_token': speech_token, 'llm_prompt_speech_token_len': speech_token_len, + 'flow_prompt_speech_token': speech_token, 'flow_prompt_speech_token_len': speech_token_len, + 'prompt_speech_feat': speech_feat, 'prompt_speech_feat_len': speech_feat_len, + 'llm_embedding': embedding, 'flow_embedding': embedding, + 'text': tts_text_token, 'text_len': tts_text_token_len} + + def frontend_instruct2(self, tts_text, instruct_text, prompt_wav, resample_rate=24000): + model_input = self.frontend_zero_shot(tts_text, instruct_text, prompt_wav, resample_rate) + del model_input['llm_prompt_speech_token'] + del model_input['llm_prompt_speech_token_len'] + return model_input diff --git a/freevideo_engine/dubbing/cosyvoice3/cli/model.py b/freevideo_engine/dubbing/cosyvoice3/cli/model.py new file mode 100644 index 00000000..b3ee8870 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/cli/model.py @@ -0,0 +1,88 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) +# 2025 Alibaba Inc (authors: Xiang Lyu, Bofan Zhou) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by FreeVideo: reduced to the non-streaming CosyVoice3 inference path +# (no streaming, TensorRT, vLLM or JIT variants); the language model runs on the +# calling thread so errors propagate. +import torch +from torch.nn import functional as F + + +class CosyVoice3Model: + + def __init__(self, llm: torch.nn.Module, flow: torch.nn.Module, hift: torch.nn.Module, device, fp16: bool = False): + self.device = torch.device(device) + self.llm = llm + self.flow = flow + self.hift = hift + self.fp16 = fp16 + # FSQ silent and breath token + self.silent_tokens = [1, 2, 28, 29, 55, 248, 494, 2241, 2242, 2322, 2323] + + def load(self, llm_model, flow_model, hift_model): + """Load the three state dicts onto the CPU first and then move each part to the device.""" + for part, path in ((self.llm, llm_model), (self.flow, flow_model)): + part.load_state_dict(torch.load(path, map_location='cpu', weights_only=True), strict=True) + hift_state = torch.load(hift_model, map_location='cpu', weights_only=True) + self.hift.load_state_dict({k.replace('generator.', ''): v for k, v in hift_state.items()}, strict=True) + for part in (self.llm, self.flow, self.hift): + part.to(self.device).eval() + + def speech_tokens(self, text, prompt_text, llm_prompt_speech_token, llm_embedding): + """Run the language model and return the generated speech tokens as a list.""" + device = self.device + tokens, silent_run, max_silent_run = [], 0, 5 + generator = self.llm.inference( + text=text.to(device), + text_len=torch.tensor([text.shape[1]], dtype=torch.int32).to(device), + prompt_text=prompt_text.to(device), + prompt_text_len=torch.tensor([prompt_text.shape[1]], dtype=torch.int32).to(device), + prompt_speech_token=llm_prompt_speech_token.to(device), + prompt_speech_token_len=torch.tensor([llm_prompt_speech_token.shape[1]], dtype=torch.int32).to(device), + embedding=llm_embedding.to(device), + uuid='freevideo') + for token in generator: + if token in self.silent_tokens: + silent_run += 1 + if silent_run > max_silent_run: + continue + else: + silent_run = 0 + tokens.append(token) + return tokens + + def token2wav(self, token, prompt_token, prompt_feat, embedding, speed=1.0): + device = self.device + with torch.autocast(device_type=device.type, enabled=bool(self.fp16) and device.type == 'cuda'): + mel, _ = self.flow.inference(token=token.to(device, dtype=torch.int32), + token_len=torch.tensor([token.shape[1]], dtype=torch.int32).to(device), + prompt_token=prompt_token.to(device), + prompt_token_len=torch.tensor([prompt_token.shape[1]], dtype=torch.int32).to(device), + prompt_feat=prompt_feat.to(device), + prompt_feat_len=torch.tensor([prompt_feat.shape[1]], dtype=torch.int32).to(device), + embedding=embedding.to(device), + streaming=False, + finalize=True) + if speed != 1.0: + mel = F.interpolate(mel, size=int(mel.shape[2] / speed), mode='linear') + speech, _ = self.hift.inference(speech_feat=mel, finalize=True) + return speech + + def tts(self, text, flow_embedding, llm_embedding, prompt_text, llm_prompt_speech_token, + flow_prompt_speech_token, prompt_speech_feat, speed=1.0): + tokens = self.speech_tokens(text, prompt_text, llm_prompt_speech_token, llm_embedding) + speech = self.token2wav(torch.tensor(tokens).unsqueeze(dim=0), flow_prompt_speech_token, + prompt_speech_feat, flow_embedding, speed=speed) + return speech.cpu() diff --git a/freevideo_engine/dubbing/cosyvoice3/flow/DiT/__init__.py b/freevideo_engine/dubbing/cosyvoice3/flow/DiT/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/flow/DiT/dit.py b/freevideo_engine/dubbing/cosyvoice3/flow/DiT/dit.py new file mode 100644 index 00000000..e12fceef --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/flow/DiT/dit.py @@ -0,0 +1,176 @@ + +""" +ein notation: +b - batch +n - sequence +nt - text sequence +nw - raw wave length +d - dimension +""" + +from __future__ import annotations + +import torch +from torch import nn +import torch.nn.functional as F +from einops import repeat +from freevideo_engine.dubbing.cosyvoice3.utils.mask import add_optional_chunk_mask +from freevideo_engine.dubbing.cosyvoice3.flow.DiT.modules import ( + RotaryEmbedding, + TimestepEmbedding, + ConvNeXtV2Block, + CausalConvPositionEmbedding, + DiTBlock, + AdaLayerNormZero_Final, + precompute_freqs_cis, + get_pos_embed_indices, +) + + +# Text embedding + + +class TextEmbedding(nn.Module): + def __init__(self, text_num_embeds, text_dim, conv_layers=0, conv_mult=2): + super().__init__() + self.text_embed = nn.Embedding(text_num_embeds + 1, text_dim) # use 0 as filler token + + if conv_layers > 0: + self.extra_modeling = True + self.precompute_max_pos = 4096 # ~44s of 24khz audio + self.register_buffer("freqs_cis", precompute_freqs_cis(text_dim, self.precompute_max_pos), persistent=False) + self.text_blocks = nn.Sequential( + *[ConvNeXtV2Block(text_dim, text_dim * conv_mult) for _ in range(conv_layers)] + ) + else: + self.extra_modeling = False + + def forward(self, text: int["b nt"], seq_len, drop_text=False): # noqa: F722 + batch, text_len = text.shape[0], text.shape[1] + text = text + 1 # use 0 as filler token. preprocess of batch pad -1, see list_str_to_idx() + text = text[:, :seq_len] # curtail if character tokens are more than the mel spec tokens + text = F.pad(text, (0, seq_len - text_len), value=0) + + if drop_text: # cfg for text + text = torch.zeros_like(text) + + text = self.text_embed(text) # b n -> b n d + + # possible extra modeling + if self.extra_modeling: + # sinus pos emb + batch_start = torch.zeros((batch,), dtype=torch.long) + pos_idx = get_pos_embed_indices(batch_start, seq_len, max_pos=self.precompute_max_pos) + text_pos_embed = self.freqs_cis[pos_idx] + text = text + text_pos_embed + + # convnextv2 blocks + text = self.text_blocks(text) + + return text + + +# noised input audio and context mixing embedding + + +class InputEmbedding(nn.Module): + def __init__(self, mel_dim, text_dim, out_dim, spk_dim=None): + super().__init__() + spk_dim = 0 if spk_dim is None else spk_dim + self.spk_dim = spk_dim + self.proj = nn.Linear(mel_dim * 2 + text_dim + spk_dim, out_dim) + self.conv_pos_embed = CausalConvPositionEmbedding(dim=out_dim) + + def forward( + self, + x: float["b n d"], + cond: float["b n d"], + text_embed: float["b n d"], + spks: float["b d"], + ): + to_cat = [x, cond, text_embed] + if self.spk_dim > 0: + spks = repeat(spks, "b c -> b t c", t=x.shape[1]) + to_cat.append(spks) + + x = self.proj(torch.cat(to_cat, dim=-1)) + x = self.conv_pos_embed(x) + x + return x + + +# Transformer backbone using DiT blocks + + +class DiT(nn.Module): + def __init__( + self, + *, + dim, + depth=8, + heads=8, + dim_head=64, + dropout=0.1, + ff_mult=4, + mel_dim=80, + mu_dim=None, + long_skip_connection=False, + spk_dim=None, + out_channels=None, + static_chunk_size=50, + num_decoding_left_chunks=2 + ): + super().__init__() + + self.time_embed = TimestepEmbedding(dim) + if mu_dim is None: + mu_dim = mel_dim + self.input_embed = InputEmbedding(mel_dim, mu_dim, dim, spk_dim) + + self.rotary_embed = RotaryEmbedding(dim_head) + + self.dim = dim + self.depth = depth + + self.transformer_blocks = nn.ModuleList( + [DiTBlock(dim=dim, heads=heads, dim_head=dim_head, ff_mult=ff_mult, dropout=dropout) for _ in range(depth)] + ) + self.long_skip_connection = nn.Linear(dim * 2, dim, bias=False) if long_skip_connection else None + + self.norm_out = AdaLayerNormZero_Final(dim) # final modulation + self.proj_out = nn.Linear(dim, mel_dim) + self.out_channels = out_channels + self.static_chunk_size = static_chunk_size + self.num_decoding_left_chunks = num_decoding_left_chunks + + def forward(self, x, mask, mu, t, spks=None, cond=None, streaming=False): + x = x.transpose(1, 2) + mu = mu.transpose(1, 2) + cond = cond.transpose(1, 2) + spks = spks.unsqueeze(dim=1) + batch, seq_len = x.shape[0], x.shape[1] + if t.ndim == 0: + t = t.repeat(batch) + + # t: conditioning time, c: context (text + masked cond audio), x: noised input audio + t = self.time_embed(t) + x = self.input_embed(x, cond, mu, spks.squeeze(1)) + + rope = self.rotary_embed.forward_from_seq_len(seq_len) + + if self.long_skip_connection is not None: + residual = x + + if streaming is True: + attn_mask = add_optional_chunk_mask(x, mask.bool(), False, False, 0, self.static_chunk_size, -1).unsqueeze(dim=1) + else: + attn_mask = add_optional_chunk_mask(x, mask.bool(), False, False, 0, 0, -1).repeat(1, x.size(1), 1).unsqueeze(dim=1) + + for block in self.transformer_blocks: + x = block(x, t, mask=attn_mask.bool(), rope=rope) + + if self.long_skip_connection is not None: + x = self.long_skip_connection(torch.cat((x, residual), dim=-1)) + + x = self.norm_out(x, t) + output = self.proj_out(x).transpose(1, 2) + return output diff --git a/freevideo_engine/dubbing/cosyvoice3/flow/DiT/modules.py b/freevideo_engine/dubbing/cosyvoice3/flow/DiT/modules.py new file mode 100644 index 00000000..1b1cd865 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/flow/DiT/modules.py @@ -0,0 +1,653 @@ + +""" +ein notation: +b - batch +n - sequence +nt - text sequence +nw - raw wave length +d - dimension +""" + +from __future__ import annotations +from typing import Optional +import math + +import torch +from torch import nn +import torch.nn.functional as F +import torchaudio + + + +# raw wav to mel spec +class MelSpec(nn.Module): + def __init__( + self, + filter_length=1024, + hop_length=256, + win_length=1024, + n_mel_channels=100, + target_sample_rate=24_000, + normalize=False, + power=1, + norm=None, + center=True, + ): + super().__init__() + self.n_mel_channels = n_mel_channels + + self.mel_stft = torchaudio.transforms.MelSpectrogram( + sample_rate=target_sample_rate, + n_fft=filter_length, + win_length=win_length, + hop_length=hop_length, + n_mels=n_mel_channels, + power=power, + center=center, + normalized=normalize, + norm=norm, + ) + + self.register_buffer("dummy", torch.tensor(0), persistent=False) + + def forward(self, inp): + if len(inp.shape) == 3: + inp = inp.squeeze(1) # 'b 1 nw -> b nw' + + assert len(inp.shape) == 2 + + if self.dummy.device != inp.device: + self.to(inp.device) + + mel = self.mel_stft(inp) + mel = mel.clamp(min=1e-5).log() + return mel + + +# sinusoidal position embedding + + +class SinusPositionEmbedding(nn.Module): + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x, scale=1000): + device = x.device + half_dim = self.dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, device=device).float() * -emb) + emb = scale * x.unsqueeze(1) * emb.unsqueeze(0) + emb = torch.cat((emb.sin(), emb.cos()), dim=-1) + return emb + + +# convolutional position embedding + + +class ConvPositionEmbedding(nn.Module): + def __init__(self, dim, kernel_size=31, groups=16): + super().__init__() + assert kernel_size % 2 != 0 + self.conv1d = nn.Sequential( + nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=kernel_size // 2), + nn.Mish(), + nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=kernel_size // 2), + nn.Mish(), + ) + + def forward(self, x: float["b n d"], mask: bool["b n"] | None = None): # noqa: F722 + if mask is not None: + mask = mask[..., None] + x = x.masked_fill(~mask, 0.0) + + x = x.permute(0, 2, 1) + x = self.conv1d(x) + out = x.permute(0, 2, 1) + + if mask is not None: + out = out.masked_fill(~mask, 0.0) + + return out + + +class CausalConvPositionEmbedding(nn.Module): + def __init__(self, dim, kernel_size=31, groups=16): + super().__init__() + assert kernel_size % 2 != 0 + self.kernel_size = kernel_size + self.conv1 = nn.Sequential( + nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=0), + nn.Mish(), + ) + self.conv2 = nn.Sequential( + nn.Conv1d(dim, dim, kernel_size, groups=groups, padding=0), + nn.Mish(), + ) + + def forward(self, x: float["b n d"], mask: bool["b n"] | None = None): # noqa: F722 + if mask is not None: + mask = mask[..., None] + x = x.masked_fill(~mask, 0.0) + + x = x.permute(0, 2, 1) + x = F.pad(x, (self.kernel_size - 1, 0, 0, 0)) + x = self.conv1(x) + x = F.pad(x, (self.kernel_size - 1, 0, 0, 0)) + x = self.conv2(x) + out = x.permute(0, 2, 1) + + if mask is not None: + out = out.masked_fill(~mask, 0.0) + + return out + + +# rotary positional embedding, re-implemented from x_transformers (MIT) so the package is not a dependency + + +class RotaryEmbedding(nn.Module): + def __init__(self, dim, base=10000): + super().__init__() + inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim)) + self.register_buffer("inv_freq", inv_freq) + + def forward_from_seq_len(self, seq_len): + t = torch.arange(seq_len, device=self.inv_freq.device) + with torch.autocast(t.device.type, enabled=False): + t = t.unsqueeze(0) + freqs = torch.einsum("b i , j -> b i j", t.type_as(self.inv_freq), self.inv_freq) + freqs = torch.stack((freqs, freqs), dim=-1) + freqs = freqs.reshape(*freqs.shape[:-2], -1) + return freqs, 1.0 + + +def rotate_half(x): + x = x.reshape(*x.shape[:-1], -1, 2) + x1, x2 = x.unbind(dim=-1) + x = torch.stack((-x2, x1), dim=-1) + return x.reshape(*x.shape[:-2], -1) + + +def apply_rotary_pos_emb(t, freqs, scale=1): + rot_dim, seq_len, orig_dtype = freqs.shape[-1], t.shape[-2], t.dtype + with torch.autocast(t.device.type, enabled=False): + freqs = freqs[:, -seq_len:, :] + if t.ndim == 4 and freqs.ndim == 3: + freqs = freqs.unsqueeze(1) + t, t_unrotated = t[..., :rot_dim], t[..., rot_dim:] + t = (t * freqs.cos() * scale) + (rotate_half(t) * freqs.sin() * scale) + out = torch.cat((t, t_unrotated), dim=-1) + return out.type(orig_dtype) + + +# rotary positional embedding related + + +def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0, theta_rescale_factor=1.0): + # proposed by reddit user bloc97, to rescale rotary embeddings to longer sequence length without fine-tuning + # has some connection to NTK literature + # https://www.reddit.com/r/LocalLLaMA/comments/14lz7j5/ntkaware_scaled_rope_allows_llama_models_to_have/ + # https://github.com/lucidrains/rotary-embedding-torch/blob/main/rotary_embedding_torch/rotary_embedding_torch.py + theta *= theta_rescale_factor ** (dim / (dim - 2)) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) + t = torch.arange(end, device=freqs.device) # type: ignore + freqs = torch.outer(t, freqs).float() # type: ignore + freqs_cos = torch.cos(freqs) # real part + freqs_sin = torch.sin(freqs) # imaginary part + return torch.cat([freqs_cos, freqs_sin], dim=-1) + + +def get_pos_embed_indices(start, length, max_pos, scale=1.0): + # length = length if isinstance(length, int) else length.max() + scale = scale * torch.ones_like(start, dtype=torch.float32) # in case scale is a scalar + pos = ( + start.unsqueeze(1) + + (torch.arange(length, device=start.device, dtype=torch.float32).unsqueeze(0) * scale.unsqueeze(1)).long() + ) + # avoid extra long error. + pos = torch.where(pos < max_pos, pos, max_pos - 1) + return pos + + +# Global Response Normalization layer (Instance Normalization ?) + + +class GRN(nn.Module): + def __init__(self, dim): + super().__init__() + self.gamma = nn.Parameter(torch.zeros(1, 1, dim)) + self.beta = nn.Parameter(torch.zeros(1, 1, dim)) + + def forward(self, x): + Gx = torch.norm(x, p=2, dim=1, keepdim=True) + Nx = Gx / (Gx.mean(dim=-1, keepdim=True) + 1e-6) + return self.gamma * (x * Nx) + self.beta + x + + +# ConvNeXt-V2 Block https://github.com/facebookresearch/ConvNeXt-V2/blob/main/models/convnextv2.py +# ref: https://github.com/bfs18/e2_tts/blob/main/rfwave/modules.py#L108 + + +class ConvNeXtV2Block(nn.Module): + def __init__( + self, + dim: int, + intermediate_dim: int, + dilation: int = 1, + ): + super().__init__() + padding = (dilation * (7 - 1)) // 2 + self.dwconv = nn.Conv1d( + dim, dim, kernel_size=7, padding=padding, groups=dim, dilation=dilation + ) # depthwise conv + self.norm = nn.LayerNorm(dim, eps=1e-6) + self.pwconv1 = nn.Linear(dim, intermediate_dim) # pointwise/1x1 convs, implemented with linear layers + self.act = nn.GELU() + self.grn = GRN(intermediate_dim) + self.pwconv2 = nn.Linear(intermediate_dim, dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + residual = x + x = x.transpose(1, 2) # b n d -> b d n + x = self.dwconv(x) + x = x.transpose(1, 2) # b d n -> b n d + x = self.norm(x) + x = self.pwconv1(x) + x = self.act(x) + x = self.grn(x) + x = self.pwconv2(x) + return residual + x + + +# AdaLayerNormZero +# return with modulated x for attn input, and params for later mlp modulation + + +class AdaLayerNormZero(nn.Module): + def __init__(self, dim): + super().__init__() + + self.silu = nn.SiLU() + self.linear = nn.Linear(dim, dim * 6) + + self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + + def forward(self, x, emb=None): + emb = self.linear(self.silu(emb)) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = torch.chunk(emb, 6, dim=1) + + x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] + return x, gate_msa, shift_mlp, scale_mlp, gate_mlp + + +# AdaLayerNormZero for final layer +# return only with modulated x for attn input, cuz no more mlp modulation + + +class AdaLayerNormZero_Final(nn.Module): + def __init__(self, dim): + super().__init__() + + self.silu = nn.SiLU() + self.linear = nn.Linear(dim, dim * 2) + + self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + + def forward(self, x, emb): + emb = self.linear(self.silu(emb)) + scale, shift = torch.chunk(emb, 2, dim=1) + + x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :] + return x + + +# FeedForward + + +class FeedForward(nn.Module): + def __init__(self, dim, dim_out=None, mult=4, dropout=0.0, approximate: str = "none"): + super().__init__() + inner_dim = int(dim * mult) + dim_out = dim_out if dim_out is not None else dim + + activation = nn.GELU(approximate=approximate) + project_in = nn.Sequential(nn.Linear(dim, inner_dim), activation) + self.ff = nn.Sequential(project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out)) + + def forward(self, x): + return self.ff(x) + + +# Attention with possible joint part +# modified from diffusers/src/diffusers/models/attention_processor.py + + +class Attention(nn.Module): + def __init__( + self, + processor: JointAttnProcessor | AttnProcessor, + dim: int, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + context_dim: Optional[int] = None, # if not None -> joint attention + context_pre_only=None, + ): + super().__init__() + + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("Attention equires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + + self.processor = processor + + self.dim = dim + self.heads = heads + self.inner_dim = dim_head * heads + self.dropout = dropout + + self.context_dim = context_dim + self.context_pre_only = context_pre_only + + self.to_q = nn.Linear(dim, self.inner_dim) + self.to_k = nn.Linear(dim, self.inner_dim) + self.to_v = nn.Linear(dim, self.inner_dim) + + if self.context_dim is not None: + self.to_k_c = nn.Linear(context_dim, self.inner_dim) + self.to_v_c = nn.Linear(context_dim, self.inner_dim) + if self.context_pre_only is not None: + self.to_q_c = nn.Linear(context_dim, self.inner_dim) + + self.to_out = nn.ModuleList([]) + self.to_out.append(nn.Linear(self.inner_dim, dim)) + self.to_out.append(nn.Dropout(dropout)) + + if self.context_pre_only is not None and not self.context_pre_only: + self.to_out_c = nn.Linear(self.inner_dim, dim) + + def forward( + self, + x: float["b n d"], # noised input x # noqa: F722 + c: float["b n d"] = None, # context c # noqa: F722 + mask: bool["b n"] | None = None, # noqa: F722 + rope=None, # rotary position embedding for x + c_rope=None, # rotary position embedding for c + ) -> torch.Tensor: + if c is not None: + return self.processor(self, x, c=c, mask=mask, rope=rope, c_rope=c_rope) + else: + return self.processor(self, x, mask=mask, rope=rope) + + +# Attention processor + + +class AttnProcessor: + def __init__(self): + pass + + def __call__( + self, + attn: Attention, + x: float["b n d"], # noised input x # noqa: F722 + mask: bool["b n"] | None = None, # noqa: F722 + rope=None, # rotary position embedding + ) -> torch.FloatTensor: + batch_size = x.shape[0] + + # `sample` projections. + query = attn.to_q(x) + key = attn.to_k(x) + value = attn.to_v(x) + + # apply rotary position embedding + if rope is not None: + freqs, xpos_scale = rope + q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0) + + query = apply_rotary_pos_emb(query, freqs, q_xpos_scale) + key = apply_rotary_pos_emb(key, freqs, k_xpos_scale) + + # attention + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # mask. e.g. inference got a batch with different target durations, mask out the padding + if mask is not None: + attn_mask = mask + if attn_mask.dim() == 2: + attn_mask = attn_mask.unsqueeze(1).unsqueeze(1) # 'b n -> b 1 1 n' + attn_mask = attn_mask.expand(batch_size, attn.heads, query.shape[-2], key.shape[-2]) + else: + attn_mask = None + + x = F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False) + x = x.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + x = x.to(query.dtype) + + # linear proj + x = attn.to_out[0](x) + # dropout + x = attn.to_out[1](x) + + if mask is not None: + if mask.dim() == 2: + mask = mask.unsqueeze(-1) + else: + mask = mask[:, 0, -1].unsqueeze(-1) + x = x.masked_fill(~mask, 0.0) + + return x + + +# Joint Attention processor for MM-DiT +# modified from diffusers/src/diffusers/models/attention_processor.py + + +class JointAttnProcessor: + def __init__(self): + pass + + def __call__( + self, + attn: Attention, + x: float["b n d"], # noised input x # noqa: F722 + c: float["b nt d"] = None, # context c, here text # noqa: F722 + mask: bool["b n"] | None = None, # noqa: F722 + rope=None, # rotary position embedding for x + c_rope=None, # rotary position embedding for c + ) -> torch.FloatTensor: + residual = x + + batch_size = c.shape[0] + + # `sample` projections. + query = attn.to_q(x) + key = attn.to_k(x) + value = attn.to_v(x) + + # `context` projections. + c_query = attn.to_q_c(c) + c_key = attn.to_k_c(c) + c_value = attn.to_v_c(c) + + # apply rope for context and noised input independently + if rope is not None: + freqs, xpos_scale = rope + q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0) + query = apply_rotary_pos_emb(query, freqs, q_xpos_scale) + key = apply_rotary_pos_emb(key, freqs, k_xpos_scale) + if c_rope is not None: + freqs, xpos_scale = c_rope + q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0) + c_query = apply_rotary_pos_emb(c_query, freqs, q_xpos_scale) + c_key = apply_rotary_pos_emb(c_key, freqs, k_xpos_scale) + + # attention + query = torch.cat([query, c_query], dim=1) + key = torch.cat([key, c_key], dim=1) + value = torch.cat([value, c_value], dim=1) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # mask. e.g. inference got a batch with different target durations, mask out the padding + if mask is not None: + attn_mask = F.pad(mask, (0, c.shape[1]), value=True) # no mask for c (text) + attn_mask = attn_mask.unsqueeze(1).unsqueeze(1) # 'b n -> b 1 1 n' + attn_mask = attn_mask.expand(batch_size, attn.heads, query.shape[-2], key.shape[-2]) + else: + attn_mask = None + + x = F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False) + x = x.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + x = x.to(query.dtype) + + # Split the attention outputs. + x, c = ( + x[:, : residual.shape[1]], + x[:, residual.shape[1]:], + ) + + # linear proj + x = attn.to_out[0](x) + # dropout + x = attn.to_out[1](x) + if not attn.context_pre_only: + c = attn.to_out_c(c) + + if mask is not None: + mask = mask.unsqueeze(-1) + x = x.masked_fill(~mask, 0.0) + # c = c.masked_fill(~mask, 0.) # no mask for c (text) + + return x, c + + +# DiT Block + + +class DiTBlock(nn.Module): + def __init__(self, dim, heads, dim_head, ff_mult=4, dropout=0.1): + super().__init__() + + self.attn_norm = AdaLayerNormZero(dim) + self.attn = Attention( + processor=AttnProcessor(), + dim=dim, + heads=heads, + dim_head=dim_head, + dropout=dropout, + ) + + self.ff_norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward(dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh") + + def forward(self, x, t, mask=None, rope=None): # x: noised input, t: time embedding + # pre-norm & modulation for attention input + norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, emb=t) + + # attention + attn_output = self.attn(x=norm, mask=mask, rope=rope) + + # process attention output for input x + x = x + gate_msa.unsqueeze(1) * attn_output + + ff_norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + ff_output = self.ff(ff_norm) + x = x + gate_mlp.unsqueeze(1) * ff_output + + return x + + +# MMDiT Block https://arxiv.org/abs/2403.03206 + + +class MMDiTBlock(nn.Module): + r""" + modified from diffusers/src/diffusers/models/attention.py + + notes. + _c: context related. text, cond, etc. (left part in sd3 fig2.b) + _x: noised input related. (right part) + context_pre_only: last layer only do prenorm + modulation cuz no more ffn + """ + + def __init__(self, dim, heads, dim_head, ff_mult=4, dropout=0.1, context_pre_only=False): + super().__init__() + + self.context_pre_only = context_pre_only + + self.attn_norm_c = AdaLayerNormZero_Final(dim) if context_pre_only else AdaLayerNormZero(dim) + self.attn_norm_x = AdaLayerNormZero(dim) + self.attn = Attention( + processor=JointAttnProcessor(), + dim=dim, + heads=heads, + dim_head=dim_head, + dropout=dropout, + context_dim=dim, + context_pre_only=context_pre_only, + ) + + if not context_pre_only: + self.ff_norm_c = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_c = FeedForward(dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh") + else: + self.ff_norm_c = None + self.ff_c = None + self.ff_norm_x = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_x = FeedForward(dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh") + + def forward(self, x, c, t, mask=None, rope=None, c_rope=None): # x: noised input, c: context, t: time embedding + # pre-norm & modulation for attention input + if self.context_pre_only: + norm_c = self.attn_norm_c(c, t) + else: + norm_c, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.attn_norm_c(c, emb=t) + norm_x, x_gate_msa, x_shift_mlp, x_scale_mlp, x_gate_mlp = self.attn_norm_x(x, emb=t) + + # attention + x_attn_output, c_attn_output = self.attn(x=norm_x, c=norm_c, mask=mask, rope=rope, c_rope=c_rope) + + # process attention output for context c + if self.context_pre_only: + c = None + else: # if not last layer + c = c + c_gate_msa.unsqueeze(1) * c_attn_output + + norm_c = self.ff_norm_c(c) * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + c_ff_output = self.ff_c(norm_c) + c = c + c_gate_mlp.unsqueeze(1) * c_ff_output + + # process attention output for input x + x = x + x_gate_msa.unsqueeze(1) * x_attn_output + + norm_x = self.ff_norm_x(x) * (1 + x_scale_mlp[:, None]) + x_shift_mlp[:, None] + x_ff_output = self.ff_x(norm_x) + x = x + x_gate_mlp.unsqueeze(1) * x_ff_output + + return c, x + + +# time step conditioning embedding + + +class TimestepEmbedding(nn.Module): + def __init__(self, dim, freq_embed_dim=256): + super().__init__() + self.time_embed = SinusPositionEmbedding(freq_embed_dim) + self.time_mlp = nn.Sequential(nn.Linear(freq_embed_dim, dim), nn.SiLU(), nn.Linear(dim, dim)) + + def forward(self, timestep: float["b"]): # noqa: F821 + time_hidden = self.time_embed(timestep) + time_hidden = time_hidden.to(timestep.dtype) + time = self.time_mlp(time_hidden) # b d + return time diff --git a/freevideo_engine/dubbing/cosyvoice3/flow/__init__.py b/freevideo_engine/dubbing/cosyvoice3/flow/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/flow/flow.py b/freevideo_engine/dubbing/cosyvoice3/flow/flow.py new file mode 100644 index 00000000..02f92f6d --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/flow/flow.py @@ -0,0 +1,103 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified by FreeVideo: reduced to the CosyVoice3 DiT flow inference path. +from typing import Dict + +import torch +import torch.nn as nn +from torch.nn import functional as F +from omegaconf import DictConfig + +from freevideo_engine.dubbing.cosyvoice3.utils.mask import make_pad_mask + + +class CausalMaskedDiffWithDiT(torch.nn.Module): + def __init__(self, + input_size: int = 512, + output_size: int = 80, + spk_embed_dim: int = 192, + output_type: str = "mel", + vocab_size: int = 4096, + input_frame_rate: int = 50, + only_mask_loss: bool = True, + token_mel_ratio: int = 2, + pre_lookahead_len: int = 3, + pre_lookahead_layer: torch.nn.Module = None, + decoder: torch.nn.Module = None, + decoder_conf: Dict = {'in_channels': 240, 'out_channel': 80, 'spk_emb_dim': 80, 'n_spks': 1, + 'cfm_params': DictConfig({'sigma_min': 1e-06, 'solver': 'euler', 't_scheduler': 'cosine', + 'training_cfg_rate': 0.2, 'inference_cfg_rate': 0.7, 'reg_loss_type': 'l1'}), + 'decoder_params': {'channels': [256, 256], 'dropout': 0.0, 'attention_head_dim': 64, + 'n_blocks': 4, 'num_mid_blocks': 12, 'num_heads': 8, 'act_fn': 'gelu'}}): + super().__init__() + self.input_size = input_size + self.output_size = output_size + self.decoder_conf = decoder_conf + self.vocab_size = vocab_size + self.output_type = output_type + self.input_frame_rate = input_frame_rate + self.input_embedding = nn.Embedding(vocab_size, input_size) + self.spk_embed_affine_layer = torch.nn.Linear(spk_embed_dim, output_size) + self.pre_lookahead_len = pre_lookahead_len + self.pre_lookahead_layer = pre_lookahead_layer + self.decoder = decoder + self.only_mask_loss = only_mask_loss + self.token_mel_ratio = token_mel_ratio + + @torch.inference_mode() + def inference(self, + token, + token_len, + prompt_token, + prompt_token_len, + prompt_feat, + prompt_feat_len, + embedding, + streaming, + finalize): + assert token.shape[0] == 1 + # xvec projection + embedding = F.normalize(embedding, dim=1) + embedding = self.spk_embed_affine_layer(embedding) + + # concat text and prompt_text + token, token_len = torch.concat([prompt_token, token], dim=1), prompt_token_len + token_len + mask = (~make_pad_mask(token_len)).unsqueeze(-1).to(embedding) + token = self.input_embedding(torch.clamp(token, min=0)) * mask + + # text encode + if finalize is True: + h = self.pre_lookahead_layer(token) + else: + h = self.pre_lookahead_layer(token[:, :-self.pre_lookahead_len], context=token[:, -self.pre_lookahead_len:]) + h = h.repeat_interleave(self.token_mel_ratio, dim=1) + mel_len1, mel_len2 = prompt_feat.shape[1], h.shape[1] - prompt_feat.shape[1] + + # get conditions + conds = torch.zeros([1, mel_len1 + mel_len2, self.output_size], device=token.device).to(h.dtype) + conds[:, :mel_len1] = prompt_feat + conds = conds.transpose(1, 2) + + mask = (~make_pad_mask(torch.tensor([mel_len1 + mel_len2]))).to(h) + feat, _ = self.decoder( + mu=h.transpose(1, 2).contiguous(), + mask=mask.unsqueeze(1), + spks=embedding, + cond=conds, + n_timesteps=10, + streaming=streaming + ) + feat = feat[:, :, mel_len1:] + assert feat.shape[2] == mel_len2 + return feat.float(), None diff --git a/freevideo_engine/dubbing/cosyvoice3/flow/flow_matching.py b/freevideo_engine/dubbing/cosyvoice3/flow/flow_matching.py new file mode 100644 index 00000000..eab641dd --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/flow/flow_matching.py @@ -0,0 +1,238 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) +# 2025 Alibaba Inc (authors: Xiang Lyu, Bofan Zhou) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn.functional as F +from freevideo_engine.dubbing.cosyvoice3.utils.common import set_all_random_seed + + +class BASECFM(torch.nn.Module): + # inlined from Matcha-TTS (MIT): only the attributes the subclass uses + def __init__(self, n_feats, cfm_params, n_spks=1, spk_emb_dim=128): + super().__init__() + self.n_feats = n_feats + self.n_spks = n_spks + self.spk_emb_dim = spk_emb_dim + self.solver = cfm_params.solver + self.sigma_min = getattr(cfm_params, 'sigma_min', 1e-4) + self.estimator = None + + +class ConditionalCFM(BASECFM): + def __init__(self, in_channels, cfm_params, n_spks=1, spk_emb_dim=64, estimator: torch.nn.Module = None): + super().__init__( + n_feats=in_channels, + cfm_params=cfm_params, + n_spks=n_spks, + spk_emb_dim=spk_emb_dim, + ) + self.t_scheduler = cfm_params.t_scheduler + self.training_cfg_rate = cfm_params.training_cfg_rate + self.inference_cfg_rate = cfm_params.inference_cfg_rate + in_channels = in_channels + (spk_emb_dim if n_spks > 0 else 0) + # Just change the architecture of the estimator here + self.estimator = estimator + + @torch.inference_mode() + def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None, prompt_len=0, cache=torch.zeros(1, 80, 0, 2)): + """Forward diffusion + + Args: + mu (torch.Tensor): output of encoder + shape: (batch_size, n_feats, mel_timesteps) + mask (torch.Tensor): output_mask + shape: (batch_size, 1, mel_timesteps) + n_timesteps (int): number of diffusion steps + temperature (float, optional): temperature for scaling noise. Defaults to 1.0. + spks (torch.Tensor, optional): speaker ids. Defaults to None. + shape: (batch_size, spk_emb_dim) + cond: Not used but kept for future purposes + + Returns: + sample: generated mel-spectrogram + shape: (batch_size, n_feats, mel_timesteps) + """ + + z = torch.randn_like(mu).to(mu.device).to(mu.dtype) * temperature + cache_size = cache.shape[2] + # fix prompt and overlap part mu and z + if cache_size != 0: + z[:, :, :cache_size] = cache[:, :, :, 0] + mu[:, :, :cache_size] = cache[:, :, :, 1] + z_cache = torch.concat([z[:, :, :prompt_len], z[:, :, -34:]], dim=2) + mu_cache = torch.concat([mu[:, :, :prompt_len], mu[:, :, -34:]], dim=2) + cache = torch.stack([z_cache, mu_cache], dim=-1) + + t_span = torch.linspace(0, 1, n_timesteps + 1, device=mu.device, dtype=mu.dtype) + if self.t_scheduler == 'cosine': + t_span = 1 - torch.cos(t_span * 0.5 * torch.pi) + return self.solve_euler(z, t_span=t_span, mu=mu, mask=mask, spks=spks, cond=cond), cache + + def solve_euler(self, x, t_span, mu, mask, spks, cond, streaming=False): + """ + Fixed euler solver for ODEs. + Args: + x (torch.Tensor): random noise + t_span (torch.Tensor): n_timesteps interpolated + shape: (n_timesteps + 1,) + mu (torch.Tensor): output of encoder + shape: (batch_size, n_feats, mel_timesteps) + mask (torch.Tensor): output_mask + shape: (batch_size, 1, mel_timesteps) + spks (torch.Tensor, optional): speaker ids. Defaults to None. + shape: (batch_size, spk_emb_dim) + cond: Not used but kept for future purposes + """ + t, _, dt = t_span[0], t_span[-1], t_span[1] - t_span[0] + t = t.unsqueeze(dim=0) + + # I am storing this because I can later plot it by putting a debugger here and saving it to a file + # Or in future might add like a return_all_steps flag + sol = [] + + # Do not use concat, it may cause memory format changed and trt infer with wrong results! + # NOTE when flow run in amp mode, x.dtype is float32, which cause nan in trt fp16 inference, so set dtype=spks.dtype + x_in = torch.zeros([2, 80, x.size(2)], device=x.device, dtype=spks.dtype) + mask_in = torch.zeros([2, 1, x.size(2)], device=x.device, dtype=spks.dtype) + mu_in = torch.zeros([2, 80, x.size(2)], device=x.device, dtype=spks.dtype) + t_in = torch.zeros([2], device=x.device, dtype=spks.dtype) + spks_in = torch.zeros([2, 80], device=x.device, dtype=spks.dtype) + cond_in = torch.zeros([2, 80, x.size(2)], device=x.device, dtype=spks.dtype) + for step in range(1, len(t_span)): + # Classifier-Free Guidance inference introduced in VoiceBox + x_in[:] = x + mask_in[:] = mask + mu_in[0] = mu + t_in[:] = t.unsqueeze(0) + spks_in[0] = spks + cond_in[0] = cond + dphi_dt = self.forward_estimator( + x_in, mask_in, + mu_in, t_in, + spks_in, + cond_in, + streaming + ) + dphi_dt, cfg_dphi_dt = torch.split(dphi_dt, [x.size(0), x.size(0)], dim=0) + dphi_dt = ((1.0 + self.inference_cfg_rate) * dphi_dt - self.inference_cfg_rate * cfg_dphi_dt) + x = x + dt * dphi_dt + t = t + dt + sol.append(x) + if step < len(t_span) - 1: + dt = t_span[step + 1] - t + + return sol[-1].float() + + def forward_estimator(self, x, mask, mu, t, spks, cond, streaming=False): + if isinstance(self.estimator, torch.nn.Module): + return self.estimator(x, mask, mu, t, spks, cond, streaming=streaming) + else: + [estimator, stream], trt_engine = self.estimator.acquire_estimator() + # NOTE need to synchronize when switching stream + torch.cuda.current_stream().synchronize() + with stream: + estimator.set_input_shape('x', (2, 80, x.size(2))) + estimator.set_input_shape('mask', (2, 1, x.size(2))) + estimator.set_input_shape('mu', (2, 80, x.size(2))) + estimator.set_input_shape('t', (2,)) + estimator.set_input_shape('spks', (2, 80)) + estimator.set_input_shape('cond', (2, 80, x.size(2))) + data_ptrs = [x.contiguous().data_ptr(), + mask.contiguous().data_ptr(), + mu.contiguous().data_ptr(), + t.contiguous().data_ptr(), + spks.contiguous().data_ptr(), + cond.contiguous().data_ptr(), + x.data_ptr()] + for i, j in enumerate(data_ptrs): + estimator.set_tensor_address(trt_engine.get_tensor_name(i), j) + # run trt engine + assert estimator.execute_async_v3(torch.cuda.current_stream().cuda_stream) is True + torch.cuda.current_stream().synchronize() + self.estimator.release_estimator(estimator, stream) + return x + + def compute_loss(self, x1, mask, mu, spks=None, cond=None, streaming=False): + """Computes diffusion loss + + Args: + x1 (torch.Tensor): Target + shape: (batch_size, n_feats, mel_timesteps) + mask (torch.Tensor): target mask + shape: (batch_size, 1, mel_timesteps) + mu (torch.Tensor): output of encoder + shape: (batch_size, n_feats, mel_timesteps) + spks (torch.Tensor, optional): speaker embedding. Defaults to None. + shape: (batch_size, spk_emb_dim) + + Returns: + loss: conditional flow matching loss + y: conditional flow + shape: (batch_size, n_feats, mel_timesteps) + """ + b, _, t = mu.shape + + # random timestep + t = torch.rand([b, 1, 1], device=mu.device, dtype=mu.dtype) + + # sample noise p(x_0) + z = torch.randn_like(x1) + + y = (1 - (1 - self.sigma_min) * t) * z + t * x1 + u = x1 - (1 - self.sigma_min) * z + + # during training, we randomly drop condition to trade off mode coverage and sample fidelity + if self.training_cfg_rate > 0: + cfg_mask = torch.rand(b, device=x1.device) > self.training_cfg_rate + mu = mu * cfg_mask.view(-1, 1, 1) + spks = spks * cfg_mask.view(-1, 1) + cond = cond * cfg_mask.view(-1, 1, 1) + + pred = self.estimator(y, mask, mu, t.squeeze(), spks, cond, streaming=streaming) + loss = F.mse_loss(pred * mask, u * mask, reduction="sum") / (torch.sum(mask) * u.shape[1]) + return loss, y + + +class CausalConditionalCFM(ConditionalCFM): + def __init__(self, in_channels, cfm_params, n_spks=1, spk_emb_dim=64, estimator: torch.nn.Module = None): + super().__init__(in_channels, cfm_params, n_spks, spk_emb_dim, estimator) + set_all_random_seed(0) + self.rand_noise = torch.randn([1, 80, 50 * 300]) + + @torch.inference_mode() + def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None, streaming=False): + """Forward diffusion + + Args: + mu (torch.Tensor): output of encoder + shape: (batch_size, n_feats, mel_timesteps) + mask (torch.Tensor): output_mask + shape: (batch_size, 1, mel_timesteps) + n_timesteps (int): number of diffusion steps + temperature (float, optional): temperature for scaling noise. Defaults to 1.0. + spks (torch.Tensor, optional): speaker ids. Defaults to None. + shape: (batch_size, spk_emb_dim) + cond: Not used but kept for future purposes + + Returns: + sample: generated mel-spectrogram + shape: (batch_size, n_feats, mel_timesteps) + """ + + z = self.rand_noise[:, :, :mu.size(2)].to(mu.device).to(mu.dtype) * temperature + # fix prompt and overlap part mu and z + t_span = torch.linspace(0, 1, n_timesteps + 1, device=mu.device, dtype=mu.dtype) + if self.t_scheduler == 'cosine': + t_span = 1 - torch.cos(t_span * 0.5 * torch.pi) + return self.solve_euler(z, t_span=t_span, mu=mu, mask=mask, spks=spks, cond=cond, streaming=streaming), None diff --git a/freevideo_engine/dubbing/cosyvoice3/hifigan/__init__.py b/freevideo_engine/dubbing/cosyvoice3/hifigan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/hifigan/f0_predictor.py b/freevideo_engine/dubbing/cosyvoice3/hifigan/f0_predictor.py new file mode 100644 index 00000000..25abfde3 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/hifigan/f0_predictor.py @@ -0,0 +1,103 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Kai Hu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +try: + from torch.nn.utils.parametrizations import weight_norm +except ImportError: + from torch.nn.utils import weight_norm +from freevideo_engine.dubbing.cosyvoice3.transformer.convolution import CausalConv1d + + +class ConvRNNF0Predictor(nn.Module): + def __init__(self, + num_class: int = 1, + in_channels: int = 80, + cond_channels: int = 512 + ): + super().__init__() + + self.num_class = num_class + self.condnet = nn.Sequential( + weight_norm( + nn.Conv1d(in_channels, cond_channels, kernel_size=3, padding=1) + ), + nn.ELU(), + weight_norm( + nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) + ), + nn.ELU(), + weight_norm( + nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) + ), + nn.ELU(), + weight_norm( + nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) + ), + nn.ELU(), + weight_norm( + nn.Conv1d(cond_channels, cond_channels, kernel_size=3, padding=1) + ), + nn.ELU(), + ) + self.classifier = nn.Linear(in_features=cond_channels, out_features=self.num_class) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.condnet(x) + x = x.transpose(1, 2) + return torch.abs(self.classifier(x).squeeze(-1)) + + +class CausalConvRNNF0Predictor(nn.Module): + def __init__(self, + num_class: int = 1, + in_channels: int = 80, + cond_channels: int = 512 + ): + super().__init__() + + self.num_class = num_class + self.condnet = nn.Sequential( + weight_norm( + CausalConv1d(in_channels, cond_channels, kernel_size=4, causal_type='right') + ), + nn.ELU(), + weight_norm( + CausalConv1d(cond_channels, cond_channels, kernel_size=3, causal_type='left') + ), + nn.ELU(), + weight_norm( + CausalConv1d(cond_channels, cond_channels, kernel_size=3, causal_type='left') + ), + nn.ELU(), + weight_norm( + CausalConv1d(cond_channels, cond_channels, kernel_size=3, causal_type='left') + ), + nn.ELU(), + weight_norm( + CausalConv1d(cond_channels, cond_channels, kernel_size=3, causal_type='left') + ), + nn.ELU(), + ) + self.classifier = nn.Linear(in_features=cond_channels, out_features=self.num_class) + + def forward(self, x: torch.Tensor, finalize: bool = True) -> torch.Tensor: + if finalize is True: + x = self.condnet[0](x) + else: + x = self.condnet[0](x[:, :, :-self.condnet[0].causal_padding], x[:, :, -self.condnet[0].causal_padding:]) + for i in range(1, len(self.condnet)): + x = self.condnet[i](x) + x = x.transpose(1, 2) + return torch.abs(self.classifier(x).squeeze(-1)) diff --git a/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py b/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py new file mode 100644 index 00000000..3766c84e --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/hifigan/generator.py @@ -0,0 +1,726 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Kai Hu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HIFI-GAN""" + +from typing import Dict, Optional, List +import numpy as np +from scipy.signal import get_window +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import Conv1d +from torch.nn import ConvTranspose1d +from torch.nn.utils import remove_weight_norm +try: + from torch.nn.utils.parametrizations import weight_norm +except ImportError: + from torch.nn.utils import weight_norm +from torch.distributions.uniform import Uniform +from freevideo_engine.dubbing.cosyvoice3.transformer.convolution import CausalConv1d, CausalConv1dDownSample, CausalConv1dUpsample +from freevideo_engine.dubbing.cosyvoice3.transformer.activation import Snake +from freevideo_engine.dubbing.cosyvoice3.utils.common import get_padding +from freevideo_engine.dubbing.cosyvoice3.utils.common import init_weights + + +"""hifigan based generator implementation. + +This code is modified from https://github.com/jik876/hifi-gan + ,https://github.com/kan-bayashi/ParallelWaveGAN and + https://github.com/NVIDIA/BigVGAN + +""" + + +class ResBlock(torch.nn.Module): + """Residual block module in HiFiGAN/BigVGAN.""" + def __init__( + self, + channels: int = 512, + kernel_size: int = 3, + dilations: List[int] = [1, 3, 5], + causal: bool = False, + ): + super(ResBlock, self).__init__() + self.causal = causal + self.convs1 = nn.ModuleList() + self.convs2 = nn.ModuleList() + + for dilation in dilations: + self.convs1.append( + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation, + padding=get_padding(kernel_size, dilation)) if causal is False else + CausalConv1d( + channels, + channels, + kernel_size, + 1, + dilation=dilation, + causal_type='left' + ) + ) + ) + self.convs2.append( + weight_norm( + Conv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + padding=get_padding(kernel_size, 1)) if causal is False else + CausalConv1d( + channels, + channels, + kernel_size, + 1, + dilation=1, + causal_type='left' + ) + ) + ) + self.convs1.apply(init_weights) + self.convs2.apply(init_weights) + self.activations1 = nn.ModuleList([ + Snake(channels, alpha_logscale=False) + for _ in range(len(self.convs1)) + ]) + self.activations2 = nn.ModuleList([ + Snake(channels, alpha_logscale=False) + for _ in range(len(self.convs2)) + ]) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + for idx in range(len(self.convs1)): + xt = self.activations1[idx](x) + xt = self.convs1[idx](xt) + xt = self.activations2[idx](xt) + xt = self.convs2[idx](xt) + x = xt + x + return x + + def remove_weight_norm(self): + for idx in range(len(self.convs1)): + remove_weight_norm(self.convs1[idx]) + remove_weight_norm(self.convs2[idx]) + + +class SineGen(torch.nn.Module): + """ Definition of sine generator + SineGen(samp_rate, harmonic_num = 0, + sine_amp = 0.1, noise_std = 0.003, + voiced_threshold = 0, + flag_for_pulse=False) + samp_rate: sampling rate in Hz + harmonic_num: number of harmonic overtones (default 0) + sine_amp: amplitude of sine-wavefrom (default 0.1) + noise_std: std of Gaussian noise (default 0.003) + voiced_thoreshold: F0 threshold for U/V classification (default 0) + flag_for_pulse: this SinGen is used inside PulseGen (default False) + Note: when flag_for_pulse is True, the first time step of a voiced + segment is always sin(np.pi) or cos(0) + """ + + def __init__(self, samp_rate, harmonic_num=0, + sine_amp=0.1, noise_std=0.003, + voiced_threshold=0): + super(SineGen, self).__init__() + self.sine_amp = sine_amp + self.noise_std = noise_std + self.harmonic_num = harmonic_num + self.sampling_rate = samp_rate + self.voiced_threshold = voiced_threshold + + def _f02uv(self, f0): + # generate uv signal + uv = (f0 > self.voiced_threshold).type(torch.float32) + return uv + + @torch.no_grad() + def forward(self, f0): + """ sine_tensor, uv = forward(f0) + input F0: tensor(batchsize=1, dim=1, length) + f0 for unvoiced steps should be 0 + output sine_tensor: tensor(batchsize=1, length, dim) + output uv: tensor(batchsize=1, length, 1) + """ + f0 = f0.transpose(1, 2) + F_mat = torch.zeros((f0.size(0), self.harmonic_num + 1, f0.size(-1))).to(f0.device) + for i in range(self.harmonic_num + 1): + F_mat[:, i: i + 1, :] = f0 * (i + 1) / self.sampling_rate + + theta_mat = 2 * np.pi * (torch.cumsum(F_mat, dim=-1) % 1) + u_dist = Uniform(low=-np.pi, high=np.pi) + phase_vec = u_dist.sample(sample_shape=(f0.size(0), self.harmonic_num + 1, 1)).to(F_mat.device) + phase_vec[:, 0, :] = 0 + + # generate sine waveforms + sine_waves = self.sine_amp * torch.sin(theta_mat + phase_vec) + + # generate uv signal + uv = self._f02uv(f0) + + # noise: for unvoiced should be similar to sine_amp + # std = self.sine_amp/3 -> max value ~ self.sine_amp + # . for voiced regions is self.noise_std + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 + noise = noise_amp * torch.randn_like(sine_waves) + + # first: set the unvoiced part to 0 by uv + # then: additive noise + sine_waves = sine_waves * uv + noise + return sine_waves.transpose(1, 2), uv.transpose(1, 2), noise + + +class SineGen2(torch.nn.Module): + """ Definition of sine generator + SineGen(samp_rate, harmonic_num = 0, + sine_amp = 0.1, noise_std = 0.003, + voiced_threshold = 0, + flag_for_pulse=False) + samp_rate: sampling rate in Hz + harmonic_num: number of harmonic overtones (default 0) + sine_amp: amplitude of sine-wavefrom (default 0.1) + noise_std: std of Gaussian noise (default 0.003) + voiced_thoreshold: F0 threshold for U/V classification (default 0) + flag_for_pulse: this SinGen is used inside PulseGen (default False) + Note: when flag_for_pulse is True, the first time step of a voiced + segment is always sin(np.pi) or cos(0) + """ + + def __init__(self, samp_rate, upsample_scale, harmonic_num=0, + sine_amp=0.1, noise_std=0.003, + voiced_threshold=0, + flag_for_pulse=False, + causal=False): + super(SineGen2, self).__init__() + self.sine_amp = sine_amp + self.noise_std = noise_std + self.harmonic_num = harmonic_num + self.dim = self.harmonic_num + 1 + self.sampling_rate = samp_rate + self.voiced_threshold = voiced_threshold + self.flag_for_pulse = flag_for_pulse + self.upsample_scale = upsample_scale + self.causal = causal + if causal is True: + self.rand_ini = torch.rand(1, 9) + self.rand_ini[:, 0] = 0 + self.sine_waves = torch.rand(1, 300 * 24000, 9) + + def _f02uv(self, f0): + # generate uv signal + uv = (f0 > self.voiced_threshold).type(torch.float32) + return uv + + def _f02sine(self, f0_values): + """ f0_values: (batchsize, length, dim) + where dim indicates fundamental tone and overtones + """ + # convert to F0 in rad. The interger part n can be ignored + # because 2 * np.pi * n doesn't affect phase + rad_values = (f0_values / self.sampling_rate) % 1 + + # initial phase noise (no noise for fundamental component) + if self.training is False and self.causal is True: + rad_values[:, 0, :] = rad_values[:, 0, :] + self.rand_ini.to(rad_values.device) + else: + rand_ini = torch.rand(f0_values.shape[0], f0_values.shape[2], device=f0_values.device) + rand_ini[:, 0] = 0 + rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini + + # instantanouse phase sine[t] = sin(2*pi \sum_i=1 ^{t} rad) + if not self.flag_for_pulse: + rad_values = torch.nn.functional.interpolate(rad_values.transpose(1, 2), + scale_factor=1 / self.upsample_scale, + mode="linear").transpose(1, 2) + + phase = torch.cumsum(rad_values, dim=1) * 2 * np.pi + phase = torch.nn.functional.interpolate(phase.transpose(1, 2) * self.upsample_scale, + scale_factor=self.upsample_scale, mode="nearest" if self.causal is True else 'linear').transpose(1, 2) + sines = torch.sin(phase) + else: + # If necessary, make sure that the first time step of every + # voiced segments is sin(pi) or cos(0) + # This is used for pulse-train generation + + # identify the last time step in unvoiced segments + uv = self._f02uv(f0_values) + uv_1 = torch.roll(uv, shifts=-1, dims=1) + uv_1[:, -1, :] = 1 + u_loc = (uv < 1) * (uv_1 > 0) + + # get the instantanouse phase + tmp_cumsum = torch.cumsum(rad_values, dim=1) + # different batch needs to be processed differently + for idx in range(f0_values.shape[0]): + temp_sum = tmp_cumsum[idx, u_loc[idx, :, 0], :] + temp_sum[1:, :] = temp_sum[1:, :] - temp_sum[0:-1, :] + # stores the accumulation of i.phase within + # each voiced segments + tmp_cumsum[idx, :, :] = 0 + tmp_cumsum[idx, u_loc[idx, :, 0], :] = temp_sum + + # rad_values - tmp_cumsum: remove the accumulation of i.phase + # within the previous voiced segment. + i_phase = torch.cumsum(rad_values - tmp_cumsum, dim=1) + + # get the sines + sines = torch.cos(i_phase * 2 * np.pi) + return sines + + def forward(self, f0): + """ sine_tensor, uv = forward(f0) + input F0: tensor(batchsize=1, length, dim=1) + f0 for unvoiced steps should be 0 + output sine_tensor: tensor(batchsize=1, length, dim) + output uv: tensor(batchsize=1, length, 1) + """ + # fundamental component + fn = torch.multiply(f0, torch.FloatTensor([[range(1, self.harmonic_num + 2)]]).to(f0.device)) + + # generate sine waveforms + sine_waves = self._f02sine(fn) * self.sine_amp + + # generate uv signal + uv = self._f02uv(f0) + + # noise: for unvoiced should be similar to sine_amp + # std = self.sine_amp/3 -> max value ~ self.sine_amp + # . for voiced regions is self.noise_std + noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3 + if self.training is False and self.causal is True: + noise = noise_amp * self.sine_waves[:, :sine_waves.shape[1]].to(sine_waves.device) + else: + noise = noise_amp * torch.randn_like(sine_waves) + + # first: set the unvoiced part to 0 by uv + # then: additive noise + sine_waves = sine_waves * uv + noise + return sine_waves, uv, noise + + +class SourceModuleHnNSF(torch.nn.Module): + """ SourceModule for hn-nsf + SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0) + sampling_rate: sampling_rate in Hz + harmonic_num: number of harmonic above F0 (default: 0) + sine_amp: amplitude of sine source signal (default: 0.1) + add_noise_std: std of additive Gaussian noise (default: 0.003) + note that amplitude of noise in unvoiced is decided + by sine_amp + voiced_threshold: threhold to set U/V given F0 (default: 0) + Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) + F0_sampled (batchsize, length, 1) + Sine_source (batchsize, length, 1) + noise_source (batchsize, length 1) + uv (batchsize, length, 1) + """ + + def __init__(self, sampling_rate, upsample_scale, harmonic_num=0, sine_amp=0.1, + add_noise_std=0.003, voiced_threshod=0, sinegen_type='1', causal=False): + super(SourceModuleHnNSF, self).__init__() + + self.sine_amp = sine_amp + self.noise_std = add_noise_std + + # to produce sine waveforms + if sinegen_type == '1': + self.l_sin_gen = SineGen(sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod) + else: + self.l_sin_gen = SineGen2(sampling_rate, upsample_scale, harmonic_num, sine_amp, add_noise_std, voiced_threshod, causal=causal) + + # to merge source harmonics into a single excitation + self.l_linear = torch.nn.Linear(harmonic_num + 1, 1) + self.l_tanh = torch.nn.Tanh() + self.causal = causal + if causal is True: + self.uv = torch.rand(1, 300 * 24000, 1) + + def forward(self, x): + """ + Sine_source, noise_source = SourceModuleHnNSF(F0_sampled) + F0_sampled (batchsize, length, 1) + Sine_source (batchsize, length, 1) + noise_source (batchsize, length 1) + """ + # source for harmonic branch + with torch.no_grad(): + sine_wavs, uv, _ = self.l_sin_gen(x) + sine_merge = self.l_tanh(self.l_linear(sine_wavs)) + + # source for noise branch, in the same shape as uv + if self.training is False and self.causal is True: + noise = self.uv[:, :uv.shape[1]] * self.sine_amp / 3 + else: + noise = torch.randn_like(uv) * self.sine_amp / 3 + return sine_merge, noise, uv + + +class HiFTGenerator(nn.Module): + """ + HiFTNet Generator: Neural Source Filter + ISTFTNet + https://arxiv.org/abs/2309.09493 + """ + def __init__( + self, + in_channels: int = 80, + base_channels: int = 512, + nb_harmonics: int = 8, + sampling_rate: int = 22050, + nsf_alpha: float = 0.1, + nsf_sigma: float = 0.003, + nsf_voiced_threshold: float = 10, + upsample_rates: List[int] = [8, 8], + upsample_kernel_sizes: List[int] = [16, 16], + istft_params: Dict[str, int] = {"n_fft": 16, "hop_len": 4}, + resblock_kernel_sizes: List[int] = [3, 7, 11], + resblock_dilation_sizes: List[List[int]] = [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + source_resblock_kernel_sizes: List[int] = [7, 11], + source_resblock_dilation_sizes: List[List[int]] = [[1, 3, 5], [1, 3, 5]], + lrelu_slope: float = 0.1, + audio_limit: float = 0.99, + f0_predictor: torch.nn.Module = None, + ): + super(HiFTGenerator, self).__init__() + + self.out_channels = 1 + self.nb_harmonics = nb_harmonics + self.sampling_rate = sampling_rate + self.istft_params = istft_params + self.lrelu_slope = lrelu_slope + self.audio_limit = audio_limit + + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + # NOTE in CosyVoice2, we use the original SineGen implementation + self.m_source = SourceModuleHnNSF( + sampling_rate=sampling_rate, + upsample_scale=np.prod(upsample_rates) * istft_params["hop_len"], + harmonic_num=nb_harmonics, + sine_amp=nsf_alpha, + add_noise_std=nsf_sigma, + voiced_threshod=nsf_voiced_threshold, + sinegen_type='1' if self.sampling_rate == 22050 else '2', + causal=False) + self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates) * istft_params["hop_len"]) + + self.conv_pre = weight_norm( + Conv1d(in_channels, base_channels, 7, 1, padding=3) + ) + + # Up + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + ConvTranspose1d( + base_channels // (2**i), + base_channels // (2**(i + 1)), + k, + u, + padding=(k - u) // 2, + ) + ) + ) + + # Down + self.source_downs = nn.ModuleList() + self.source_resblocks = nn.ModuleList() + downsample_rates = [1] + upsample_rates[::-1][:-1] + downsample_cum_rates = np.cumprod(downsample_rates) + for i, (u, k, d) in enumerate(zip(downsample_cum_rates[::-1], source_resblock_kernel_sizes, source_resblock_dilation_sizes)): + if u == 1: + self.source_downs.append( + Conv1d(istft_params["n_fft"] + 2, base_channels // (2 ** (i + 1)), 1, 1) + ) + else: + self.source_downs.append( + Conv1d(istft_params["n_fft"] + 2, base_channels // (2 ** (i + 1)), u * 2, u, padding=(u // 2)) + ) + + self.source_resblocks.append( + ResBlock(base_channels // (2 ** (i + 1)), k, d) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = base_channels // (2**(i + 1)) + for _, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): + self.resblocks.append(ResBlock(ch, k, d)) + + self.conv_post = weight_norm(Conv1d(ch, istft_params["n_fft"] + 2, 7, 1, padding=3)) + self.ups.apply(init_weights) + self.conv_post.apply(init_weights) + self.reflection_pad = nn.ReflectionPad1d((1, 0)) + self.stft_window = torch.from_numpy(get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32)) + self.f0_predictor = f0_predictor + + def remove_weight_norm(self): + print('Removing weight norm...') + for l in self.ups: + remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() + remove_weight_norm(self.conv_pre) + remove_weight_norm(self.conv_post) + self.m_source.remove_weight_norm() + for l in self.source_downs: + remove_weight_norm(l) + for l in self.source_resblocks: + l.remove_weight_norm() + + def _stft(self, x): + spec = torch.stft( + x, + self.istft_params["n_fft"], self.istft_params["hop_len"], self.istft_params["n_fft"], window=self.stft_window.to(x.device), + return_complex=True) + spec = torch.view_as_real(spec) # [B, F, TT, 2] + return spec[..., 0], spec[..., 1] + + def _istft(self, magnitude, phase): + magnitude = torch.clip(magnitude, max=1e2) + real = magnitude * torch.cos(phase) + img = magnitude * torch.sin(phase) + inverse_transform = torch.istft(torch.complex(real, img), self.istft_params["n_fft"], self.istft_params["hop_len"], + self.istft_params["n_fft"], window=self.stft_window.to(magnitude.device)) + return inverse_transform + + def decode(self, x: torch.Tensor, s: torch.Tensor = torch.zeros(1, 1, 0)) -> torch.Tensor: + s_stft_real, s_stft_imag = self._stft(s.squeeze(1)) + s_stft = torch.cat([s_stft_real, s_stft_imag], dim=1) + + x = self.conv_pre(x) + for i in range(self.num_upsamples): + x = F.leaky_relu(x, self.lrelu_slope) + x = self.ups[i](x) + + if i == self.num_upsamples - 1: + x = self.reflection_pad(x) + + # fusion + si = self.source_downs[i](s_stft) + si = self.source_resblocks[i](si) + x = x + si + + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + + x = F.leaky_relu(x) + x = self.conv_post(x) + magnitude = torch.exp(x[:, :self.istft_params["n_fft"] // 2 + 1, :]) + phase = torch.sin(x[:, self.istft_params["n_fft"] // 2 + 1:, :]) # actually, sin is redundancy + + x = self._istft(magnitude, phase) + x = torch.clamp(x, -self.audio_limit, self.audio_limit) + return x + + def forward( + self, + batch: dict, + device: torch.device, + ) -> Dict[str, Optional[torch.Tensor]]: + speech_feat = batch['speech_feat'].transpose(1, 2).to(device) + # mel->f0 + f0 = self.f0_predictor(speech_feat) + # f0->source + s = self.f0_upsamp(f0[:, None]).transpose(1, 2) # bs,n,t + s, _, _ = self.m_source(s) + s = s.transpose(1, 2) + # mel+source->speech + generated_speech = self.decode(x=speech_feat, s=s) + return generated_speech, f0 + + @torch.inference_mode() + def inference(self, speech_feat: torch.Tensor, cache_source: torch.Tensor = torch.zeros(1, 1, 0)) -> torch.Tensor: + # mel->f0 + f0 = self.f0_predictor(speech_feat) + # f0->source + s = self.f0_upsamp(f0[:, None]).transpose(1, 2) # bs,n,t + s, _, _ = self.m_source(s) + s = s.transpose(1, 2) + # use cache_source to avoid glitch + if cache_source.shape[2] != 0: + s[:, :, :cache_source.shape[2]] = cache_source + generated_speech = self.decode(x=speech_feat, s=s) + return generated_speech, s + + +class CausalHiFTGenerator(HiFTGenerator): + """ + HiFTNet Generator: Neural Source Filter + ISTFTNet + https://arxiv.org/abs/2309.09493 + """ + def __init__( + self, + in_channels: int = 80, + base_channels: int = 512, + nb_harmonics: int = 8, + sampling_rate: int = 22050, + nsf_alpha: float = 0.1, + nsf_sigma: float = 0.003, + nsf_voiced_threshold: float = 10, + upsample_rates: List[int] = [8, 8], + upsample_kernel_sizes: List[int] = [16, 16], + istft_params: Dict[str, int] = {"n_fft": 16, "hop_len": 4}, + resblock_kernel_sizes: List[int] = [3, 7, 11], + resblock_dilation_sizes: List[List[int]] = [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + source_resblock_kernel_sizes: List[int] = [7, 11], + source_resblock_dilation_sizes: List[List[int]] = [[1, 3, 5], [1, 3, 5]], + lrelu_slope: float = 0.1, + audio_limit: float = 0.99, + conv_pre_look_right: int = 4, + f0_predictor: torch.nn.Module = None, + ): + torch.nn.Module.__init__(self) + + self.out_channels = 1 + self.nb_harmonics = nb_harmonics + self.sampling_rate = sampling_rate + self.istft_params = istft_params + self.lrelu_slope = lrelu_slope + self.audio_limit = audio_limit + + self.num_kernels = len(resblock_kernel_sizes) + self.num_upsamples = len(upsample_rates) + self.m_source = SourceModuleHnNSF( + sampling_rate=sampling_rate, + upsample_scale=np.prod(upsample_rates) * istft_params["hop_len"], + harmonic_num=nb_harmonics, + sine_amp=nsf_alpha, + add_noise_std=nsf_sigma, + voiced_threshod=nsf_voiced_threshold, + sinegen_type='1' if self.sampling_rate == 22050 else '2', + causal=True) + self.upsample_rates = upsample_rates + self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates) * istft_params["hop_len"]) + + self.conv_pre = weight_norm( + CausalConv1d(in_channels, base_channels, conv_pre_look_right + 1, 1, causal_type='right') + ) + + # Up + self.ups = nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): + self.ups.append( + weight_norm( + CausalConv1dUpsample( + base_channels // (2**i), + base_channels // (2**(i + 1)), + k, + u, + ) + ) + ) + + # Down + self.source_downs = nn.ModuleList() + self.source_resblocks = nn.ModuleList() + downsample_rates = [1] + upsample_rates[::-1][:-1] + downsample_cum_rates = np.cumprod(downsample_rates) + for i, (u, k, d) in enumerate(zip(downsample_cum_rates[::-1], source_resblock_kernel_sizes, source_resblock_dilation_sizes)): + if u == 1: + self.source_downs.append( + CausalConv1d(istft_params["n_fft"] + 2, base_channels // (2 ** (i + 1)), 1, 1, causal_type='left') + ) + else: + self.source_downs.append( + CausalConv1dDownSample(istft_params["n_fft"] + 2, base_channels // (2 ** (i + 1)), u * 2, u) + ) + + self.source_resblocks.append( + ResBlock(base_channels // (2 ** (i + 1)), k, d, causal=True) + ) + + self.resblocks = nn.ModuleList() + for i in range(len(self.ups)): + ch = base_channels // (2**(i + 1)) + for _, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): + self.resblocks.append(ResBlock(ch, k, d, causal=True)) + + self.conv_post = weight_norm(CausalConv1d(ch, istft_params["n_fft"] + 2, 7, 1, causal_type='left')) + self.ups.apply(init_weights) + self.conv_post.apply(init_weights) + self.reflection_pad = nn.ReflectionPad1d((1, 0)) + self.stft_window = torch.from_numpy(get_window("hann", istft_params["n_fft"], fftbins=True).astype(np.float32)) + self.conv_pre_look_right = conv_pre_look_right + self.f0_predictor = f0_predictor + + def decode(self, x: torch.Tensor, s: torch.Tensor = torch.zeros(1, 1, 0), finalize: bool = True) -> torch.Tensor: + s_stft_real, s_stft_imag = self._stft(s.squeeze(1)) + if finalize is True: + x = self.conv_pre(x) + else: + x = self.conv_pre(x[:, :, :-self.conv_pre_look_right], x[:, :, -self.conv_pre_look_right:]) + s_stft_real = s_stft_real[:, :, :-int(np.prod(self.upsample_rates) * self.conv_pre_look_right)] + s_stft_imag = s_stft_imag[:, :, :-int(np.prod(self.upsample_rates) * self.conv_pre_look_right)] + s_stft = torch.cat([s_stft_real, s_stft_imag], dim=1) + + for i in range(self.num_upsamples): + x = F.leaky_relu(x, self.lrelu_slope) + x = self.ups[i](x) + + if i == self.num_upsamples - 1: + x = self.reflection_pad(x) + + # fusion + si = self.source_downs[i](s_stft) + si = self.source_resblocks[i](si) + x = x + si + + xs = None + for j in range(self.num_kernels): + if xs is None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels + + x = F.leaky_relu(x) + x = self.conv_post(x) + magnitude = torch.exp(x[:, :self.istft_params["n_fft"] // 2 + 1, :]) + phase = torch.sin(x[:, self.istft_params["n_fft"] // 2 + 1:, :]) # actually, sin is redundancy + + x = self._istft(magnitude, phase) + if finalize is False: + x = x[:, :-int(np.prod(self.upsample_rates) * self.istft_params['hop_len'])] + x = torch.clamp(x, -self.audio_limit, self.audio_limit) + return x + + @torch.inference_mode() + def inference(self, speech_feat: torch.Tensor, finalize: bool = True) -> torch.Tensor: + # mel->f0 NOTE f0_predictor precision is crucial for causal inference, move self.f0_predictor to cpu if necessary + self.f0_predictor.to(torch.float64) + f0 = self.f0_predictor(speech_feat.to(torch.float64), finalize=finalize).to(speech_feat) + # f0->source + s = self.f0_upsamp(f0[:, None]).transpose(1, 2) # bs,n,t + s, _, _ = self.m_source(s) + s = s.transpose(1, 2) + if finalize is True: + generated_speech = self.decode(x=speech_feat, s=s, finalize=finalize) + else: + generated_speech = self.decode(x=speech_feat[:, :, :-self.f0_predictor.condnet[0].causal_padding], s=s, finalize=finalize) + return generated_speech, s diff --git a/freevideo_engine/dubbing/cosyvoice3/llm/__init__.py b/freevideo_engine/dubbing/cosyvoice3/llm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/llm/llm.py b/freevideo_engine/dubbing/cosyvoice3/llm/llm.py new file mode 100644 index 00000000..6874fbb8 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/llm/llm.py @@ -0,0 +1,119 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) +# 2025 Alibaba Inc (authors: Xiang Lyu, Yabin Li, Qihua, Shengqiang Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified by FreeVideo: reduced to the CosyVoice3 inference path (no training, streaming +# or vLLM code); the Qwen2 backbone is built from its config and filled from llm.pt. +from typing import Callable, Generator, List + +import torch +from torch import nn +from transformers import AutoConfig, Qwen2ForCausalLM + + +class Qwen2Encoder(torch.nn.Module): + def __init__(self, config_dir): + super().__init__() + config = AutoConfig.from_pretrained(config_dir) + config.dtype = torch.float32 + self.model = Qwen2ForCausalLM(config).to(torch.float32) + + def forward_one_step(self, xs, cache=None): + # single sequence, no padding: the causal mask comes from the cache. + # (a one-step mask of length 1 would be applied against the whole KV cache by transformers>=5) + outs = self.model( + inputs_embeds=xs, + attention_mask=None, + output_hidden_states=True, + return_dict=True, + use_cache=True, + past_key_values=cache, + ) + return outs.hidden_states[-1], outs.past_key_values + + +class CosyVoice3LM(torch.nn.Module): + def __init__( + self, + llm_input_size: int, + llm_output_size: int, + speech_token_size: int, + llm: torch.nn.Module, + sampling: Callable, + mix_ratio: List[int] = [5, 15], + ): + super().__init__() + self.llm_input_size = llm_input_size + self.llm_output_size = llm_output_size + self.speech_token_size = speech_token_size + self.sos = speech_token_size + 0 + self.eos_token = speech_token_size + 1 + self.task_id = speech_token_size + 2 + self.fill_token = speech_token_size + 3 + + self.llm = llm + self.llm_decoder = nn.Linear(llm_output_size, speech_token_size + 200, bias=False) + self.speech_embedding = torch.nn.Embedding(speech_token_size + 200, llm_input_size) + self.sampling = sampling + self.mix_ratio = mix_ratio + self.stop_token_ids = [speech_token_size + i for i in range(200)] + + def sampling_ids(self, weighted_scores: torch.Tensor, decoded_tokens: List, sampling: int, ignore_eos: bool = True): + if ignore_eos is True: + weighted_scores[self.speech_token_size] = -float('inf') + return self.sampling(weighted_scores, decoded_tokens, sampling) + + @torch.inference_mode() + def inference( + self, + text: torch.Tensor, + text_len: torch.Tensor, + prompt_text: torch.Tensor, + prompt_text_len: torch.Tensor, + prompt_speech_token: torch.Tensor, + prompt_speech_token_len: torch.Tensor, + embedding: torch.Tensor, + sampling: int = 25, + max_token_text_ratio: float = 20, + min_token_text_ratio: float = 2, + uuid: str = '', + ) -> Generator[int, None, None]: + device = text.device + text = torch.concat([prompt_text, text], dim=1) + text_len += prompt_text_len + text_emb = self.llm.model.model.embed_tokens(text) + # 151646 is the <|endofprompt|> token + assert 151646 in text, '<|endofprompt|> not detected in CosyVoice3 text or prompt_text, check your input!' + + sos_emb = self.speech_embedding.weight[self.sos].reshape(1, 1, -1) + task_id_emb = self.speech_embedding.weight[self.task_id].reshape(1, 1, -1) + if prompt_speech_token_len != 0: + prompt_speech_token_emb = self.speech_embedding(prompt_speech_token) + else: + prompt_speech_token_emb = torch.zeros(1, 0, self.llm_input_size, dtype=text_emb.dtype).to(device) + lm_input = torch.concat([sos_emb, text_emb, task_id_emb, prompt_speech_token_emb], dim=1) + + min_len = int((text_len - prompt_text_len) * min_token_text_ratio) + max_len = int((text_len - prompt_text_len) * max_token_text_ratio) + + out_tokens = [] + cache = None + for i in range(max_len): + y_pred, cache = self.llm.forward_one_step(lm_input, cache=cache) + logp = self.llm_decoder(y_pred[:, -1]).log_softmax(dim=-1) + top_ids = self.sampling_ids(logp.squeeze(dim=0), out_tokens, sampling, ignore_eos=True if i < min_len else False) + if top_ids in self.stop_token_ids: + break + yield top_ids + out_tokens.append(top_ids) + lm_input = self.speech_embedding.weight[top_ids].reshape(1, 1, -1) diff --git a/freevideo_engine/dubbing/cosyvoice3/tokenizer/__init__.py b/freevideo_engine/dubbing/cosyvoice3/tokenizer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/tokenizer/tokenizer.py b/freevideo_engine/dubbing/cosyvoice3/tokenizer/tokenizer.py new file mode 100644 index 00000000..3fc95a51 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/tokenizer/tokenizer.py @@ -0,0 +1,107 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified by FreeVideo: reduced to the Qwen-based CosyVoice tokenizers (no tiktoken). +from functools import lru_cache + +import torch +from transformers import AutoTokenizer + + +class CosyVoice2Tokenizer(): + def __init__(self, token_path, skip_special_tokens=True): + super().__init__() + # NOTE: non-chat model, all these special tokens keep randomly initialized. + special_tokens = { + 'eos_token': '<|endoftext|>', + 'pad_token': '<|endoftext|>', + 'additional_special_tokens': [ + '<|im_start|>', '<|im_end|>', '<|endofprompt|>', + '[breath]', '', '', '[noise]', + '[laughter]', '[cough]', '[clucking]', '[accent]', + '[quick_breath]', + "", "", + "[hissing]", "[sigh]", "[vocalized-noise]", + "[lipsmack]", "[mn]" + ] + } + self.special_tokens = special_tokens + self.tokenizer = AutoTokenizer.from_pretrained(token_path) + self.tokenizer.add_special_tokens(special_tokens) + self.skip_special_tokens = skip_special_tokens + + def encode(self, text, **kwargs): + tokens = self.tokenizer([text], return_tensors="pt") + tokens = tokens["input_ids"][0].cpu().tolist() + return tokens + + def decode(self, tokens): + tokens = torch.tensor(tokens, dtype=torch.int64) + text = self.tokenizer.batch_decode([tokens], skip_special_tokens=self.skip_special_tokens)[0] + return text + + +class CosyVoice3Tokenizer(CosyVoice2Tokenizer): + def __init__(self, token_path, skip_special_tokens=True): + # NOTE: non-chat model, all these special tokens keep randomly initialized. + special_tokens = { + 'eos_token': '<|endoftext|>', + 'pad_token': '<|endoftext|>', + 'additional_special_tokens': [ + '<|im_start|>', '<|im_end|>', '<|endofprompt|>', + '[breath]', '', '', '[noise]', + '[laughter]', '[cough]', '[clucking]', '[accent]', + '[quick_breath]', + "", "", + "[hissing]", "[sigh]", "[vocalized-noise]", + "[lipsmack]", "[mn]", "<|endofsystem|>", + "[AA]", "[AA0]", "[AA1]", "[AA2]", "[AE]", "[AE0]", "[AE1]", "[AE2]", "[AH]", "[AH0]", "[AH1]", "[AH2]", + "[AO]", "[AO0]", "[AO1]", "[AO2]", "[AW]", "[AW0]", "[AW1]", "[AW2]", "[AY]", "[AY0]", "[AY1]", "[AY2]", + "[B]", "[CH]", "[D]", "[DH]", "[EH]", "[EH0]", "[EH1]", "[EH2]", "[ER]", "[ER0]", "[ER1]", "[ER2]", "[EY]", + "[EY0]", "[EY1]", "[EY2]", "[F]", "[G]", "[HH]", "[IH]", "[IH0]", "[IH1]", "[IH2]", "[IY]", "[IY0]", "[IY1]", + "[IY2]", "[JH]", "[K]", "[L]", "[M]", "[N]", "[NG]", "[OW]", "[OW0]", "[OW1]", "[OW2]", "[OY]", "[OY0]", + "[OY1]", "[OY2]", "[P]", "[R]", "[S]", "[SH]", "[T]", "[TH]", "[UH]", "[UH0]", "[UH1]", "[UH2]", "[UW]", + "[UW0]", "[UW1]", "[UW2]", "[V]", "[W]", "[Y]", "[Z]", "[ZH]", + "[a]", "[ai]", "[an]", "[ang]", "[ao]", "[b]", "[c]", "[ch]", "[d]", "[e]", "[ei]", "[en]", "[eng]", "[f]", + "[g]", "[h]", "[i]", "[ian]", "[in]", "[ing]", "[iu]", "[ià]", "[iàn]", "[iàng]", "[iào]", "[iá]", "[ián]", + "[iáng]", "[iáo]", "[iè]", "[ié]", "[iòng]", "[ióng]", "[iù]", "[iú]", "[iā]", "[iān]", "[iāng]", "[iāo]", + "[iē]", "[iě]", "[iōng]", "[iū]", "[iǎ]", "[iǎn]", "[iǎng]", "[iǎo]", "[iǒng]", "[iǔ]", "[j]", "[k]", "[l]", + "[m]", "[n]", "[o]", "[ong]", "[ou]", "[p]", "[q]", "[r]", "[s]", "[sh]", "[t]", "[u]", "[uang]", "[ue]", + "[un]", "[uo]", "[uà]", "[uài]", "[uàn]", "[uàng]", "[uá]", "[uái]", "[uán]", "[uáng]", "[uè]", "[ué]", "[uì]", + "[uí]", "[uò]", "[uó]", "[uā]", "[uāi]", "[uān]", "[uāng]", "[uē]", "[uě]", "[uī]", "[uō]", "[uǎ]", "[uǎi]", + "[uǎn]", "[uǎng]", "[uǐ]", "[uǒ]", "[vè]", "[w]", "[x]", "[y]", "[z]", "[zh]", "[à]", "[ài]", "[àn]", "[àng]", + "[ào]", "[á]", "[ái]", "[án]", "[áng]", "[áo]", "[è]", "[èi]", "[èn]", "[èng]", "[èr]", "[é]", "[éi]", "[én]", + "[éng]", "[ér]", "[ì]", "[ìn]", "[ìng]", "[í]", "[ín]", "[íng]", "[ò]", "[òng]", "[òu]", "[ó]", "[óng]", "[óu]", + "[ù]", "[ùn]", "[ú]", "[ún]", "[ā]", "[āi]", "[ān]", "[āng]", "[āo]", "[ē]", "[ēi]", "[ēn]", "[ēng]", "[ě]", + "[ěi]", "[ěn]", "[ěng]", "[ěr]", "[ī]", "[īn]", "[īng]", "[ō]", "[ōng]", "[ōu]", "[ū]", "[ūn]", "[ǎ]", "[ǎi]", + "[ǎn]", "[ǎng]", "[ǎo]", "[ǐ]", "[ǐn]", "[ǐng]", "[ǒ]", "[ǒng]", "[ǒu]", "[ǔ]", "[ǔn]", "[ǘ]", "[ǚ]", "[ǜ]" + ] + } + self.special_tokens = special_tokens + self.tokenizer = AutoTokenizer.from_pretrained(token_path) + self.tokenizer.add_special_tokens(special_tokens) + self.skip_special_tokens = skip_special_tokens + + +@lru_cache(maxsize=None) +def get_qwen_tokenizer( + token_path: str, + skip_special_tokens: bool, + version: str = 'cosyvoice2' +): + if version == 'cosyvoice2': + return CosyVoice2Tokenizer(token_path=token_path, skip_special_tokens=skip_special_tokens) + elif version == 'cosyvoice3': + return CosyVoice3Tokenizer(token_path=token_path, skip_special_tokens=skip_special_tokens) + else: + raise ValueError diff --git a/freevideo_engine/dubbing/cosyvoice3/transformer/__init__.py b/freevideo_engine/dubbing/cosyvoice3/transformer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/transformer/activation.py b/freevideo_engine/dubbing/cosyvoice3/transformer/activation.py new file mode 100644 index 00000000..8cea5481 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/transformer/activation.py @@ -0,0 +1,84 @@ +# Copyright (c) 2020 Johns Hopkins University (Shinji Watanabe) +# 2020 Northwestern Polytechnical University (Pengcheng Guo) +# 2020 Mobvoi Inc (Binbin Zhang) +# 2024 Alibaba Inc (Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Swish() activation function for Conformer.""" + +import torch +from torch import nn, sin, pow +from torch.nn import Parameter + + +class Swish(torch.nn.Module): + """Construct an Swish object.""" + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Return Swish activation function.""" + return x * torch.sigmoid(x) + + +# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license. +# LICENSE is in incl_licenses directory. +class Snake(nn.Module): + ''' + Implementation of a sine-based periodic activation function + Shape: + - Input: (B, C, T) + - Output: (B, C, T), same shape as the input + Parameters: + - alpha - trainable parameter + References: + - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda: + https://arxiv.org/abs/2006.08195 + Examples: + >>> a1 = snake(256) + >>> x = torch.randn(256) + >>> x = a1(x) + ''' + def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False): + ''' + Initialization. + INPUT: + - in_features: shape of the input + - alpha: trainable parameter + alpha is initialized to 1 by default, higher values = higher-frequency. + alpha will be trained along with the rest of your model. + ''' + super(Snake, self).__init__() + self.in_features = in_features + + # initialize alpha + self.alpha_logscale = alpha_logscale + if self.alpha_logscale: # log scale alphas initialized to zeros + self.alpha = Parameter(torch.zeros(in_features) * alpha) + else: # linear scale alphas initialized to ones + self.alpha = Parameter(torch.ones(in_features) * alpha) + + self.alpha.requires_grad = alpha_trainable + + self.no_div_by_zero = 0.000000001 + + def forward(self, x): + ''' + Forward pass of the function. + Applies the function to the input elementwise. + Snake ∶= x + 1/a * sin^2 (xa) + ''' + alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T] + if self.alpha_logscale: + alpha = torch.exp(alpha) + x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2) + + return x diff --git a/freevideo_engine/dubbing/cosyvoice3/transformer/convolution.py b/freevideo_engine/dubbing/cosyvoice3/transformer/convolution.py new file mode 100644 index 00000000..edb32b24 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/transformer/convolution.py @@ -0,0 +1,258 @@ +# Copyright (c) 2020 Mobvoi Inc. (authors: Binbin Zhang, Di Wu) +# 2024 Alibaba Inc (Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified from ESPnet(https://github.com/espnet/espnet) +"""ConvolutionModule definition.""" + +from typing import Tuple + +import torch +from torch import nn +import torch.nn.functional as F + + +class ConvolutionModule(nn.Module): + """ConvolutionModule in Conformer model.""" + + def __init__(self, + channels: int, + kernel_size: int = 15, + activation: nn.Module = nn.ReLU(), + norm: str = "batch_norm", + causal: bool = False, + bias: bool = True): + """Construct an ConvolutionModule object. + Args: + channels (int): The number of channels of conv layers. + kernel_size (int): Kernel size of conv layers. + causal (int): Whether use causal convolution or not + """ + super().__init__() + + self.pointwise_conv1 = nn.Conv1d( + channels, + 2 * channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias, + ) + # self.lorder is used to distinguish if it's a causal convolution, + # if self.lorder > 0: it's a causal convolution, the input will be + # padded with self.lorder frames on the left in forward. + # else: it's a symmetrical convolution + if causal: + padding = 0 + self.lorder = kernel_size - 1 + else: + # kernel_size should be an odd number for none causal convolution + assert (kernel_size - 1) % 2 == 0 + padding = (kernel_size - 1) // 2 + self.lorder = 0 + self.depthwise_conv = nn.Conv1d( + channels, + channels, + kernel_size, + stride=1, + padding=padding, + groups=channels, + bias=bias, + ) + + assert norm in ['batch_norm', 'layer_norm'] + if norm == "batch_norm": + self.use_layer_norm = False + self.norm = nn.BatchNorm1d(channels) + else: + self.use_layer_norm = True + self.norm = nn.LayerNorm(channels) + + self.pointwise_conv2 = nn.Conv1d( + channels, + channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias, + ) + self.activation = activation + + def forward( + self, + x: torch.Tensor, + mask_pad: torch.Tensor = torch.ones((0, 0, 0), dtype=torch.bool), + cache: torch.Tensor = torch.zeros((0, 0, 0)), + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute convolution module. + Args: + x (torch.Tensor): Input tensor (#batch, time, channels). + mask_pad (torch.Tensor): used for batch padding (#batch, 1, time), + (0, 0, 0) means fake mask. + cache (torch.Tensor): left context cache, it is only + used in causal convolution (#batch, channels, cache_t), + (0, 0, 0) meas fake cache. + Returns: + torch.Tensor: Output tensor (#batch, time, channels). + """ + # exchange the temporal dimension and the feature dimension + x = x.transpose(1, 2) # (#batch, channels, time) + + # mask batch padding + if mask_pad.size(2) > 0: # time > 0 + x.masked_fill_(~mask_pad, 0.0) + + if self.lorder > 0: + if cache.size(2) == 0: # cache_t == 0 + x = nn.functional.pad(x, (self.lorder, 0), 'constant', 0.0) + else: + assert cache.size(0) == x.size(0) # equal batch + assert cache.size(1) == x.size(1) # equal channel + x = torch.cat((cache, x), dim=2) + assert (x.size(2) > self.lorder) + new_cache = x[:, :, -self.lorder:] + else: + # It's better we just return None if no cache is required, + # However, for JIT export, here we just fake one tensor instead of + # None. + new_cache = torch.zeros((0, 0, 0), dtype=x.dtype, device=x.device) + + # GLU mechanism + x = self.pointwise_conv1(x) # (batch, 2*channel, dim) + x = nn.functional.glu(x, dim=1) # (batch, channel, dim) + + # 1D Depthwise Conv + x = self.depthwise_conv(x) + if self.use_layer_norm: + x = x.transpose(1, 2) + x = self.activation(self.norm(x)) + if self.use_layer_norm: + x = x.transpose(1, 2) + x = self.pointwise_conv2(x) + # mask batch padding + if mask_pad.size(2) > 0: # time > 0 + x.masked_fill_(~mask_pad, 0.0) + + return x.transpose(1, 2), new_cache + + +# NOTE(Xiang Lyu) causal conv module used in convolution-based vocoder +class CausalConv1d(torch.nn.Conv1d): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + dilation: int = 1, + groups: int = 1, + bias: bool = True, + padding_mode: str = 'zeros', + causal_type: str = 'left', + device=None, + dtype=None + ) -> None: + super(CausalConv1d, self).__init__(in_channels, out_channels, + kernel_size, stride=1, + padding=0, dilation=dilation, + groups=groups, bias=bias, + padding_mode=padding_mode, + device=device, dtype=dtype) + assert stride == 1 + self.causal_padding = int((kernel_size * dilation - dilation) / 2) * 2 + (kernel_size + 1) % 2 + assert causal_type in ['left', 'right'] + self.causal_type = causal_type + + def forward(self, x: torch.Tensor, cache: torch.Tensor = torch.zeros(0, 0, 0)) -> Tuple[torch.Tensor]: + input_timestep = x.shape[2] + if cache.size(2) == 0: + cache = torch.zeros(x.shape[0], x.shape[1], self.causal_padding).to(x) + assert cache.size(2) == self.causal_padding + if self.causal_type == 'left': + x = torch.concat([cache, x], dim=2) + else: + x = torch.concat([x, cache], dim=2) + x = super(CausalConv1d, self).forward(x) + assert x.shape[2] == input_timestep + return x + + +class CausalConv1dDownSample(torch.nn.Conv1d): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + dilation: int = 1, + groups: int = 1, + bias: bool = True, + padding_mode: str = 'zeros', + device=None, + dtype=None + ) -> None: + super(CausalConv1dDownSample, self).__init__(in_channels, out_channels, + kernel_size, stride, + padding=0, dilation=dilation, + groups=groups, bias=bias, + padding_mode=padding_mode, + device=device, dtype=dtype) + assert stride != 1 and dilation == 1 + assert kernel_size % stride == 0 + self.causal_padding = stride - 1 + + def forward(self, x: torch.Tensor, cache: torch.Tensor = torch.zeros(0, 0, 0)) -> Tuple[torch.Tensor, torch.Tensor]: + if cache.size(2) == 0: + x = F.pad(x, (self.causal_padding, 0), value=0.0) + else: + assert cache.size(2) == self.causal_padding + x = torch.concat([cache, x], dim=2) + x = super(CausalConv1dDownSample, self).forward(x) + return x + + +class CausalConv1dUpsample(torch.nn.Conv1d): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + dilation: int = 1, + groups: int = 1, + bias: bool = True, + padding_mode: str = 'zeros', + device=None, + dtype=None + ) -> None: + super(CausalConv1dUpsample, self).__init__(in_channels, out_channels, + kernel_size, 1, + padding=0, dilation=dilation, + groups=groups, bias=bias, + padding_mode=padding_mode, + device=device, dtype=dtype) + assert dilation == 1 + self.causal_padding = kernel_size - 1 + self.upsample = torch.nn.Upsample(scale_factor=stride, mode='nearest') + + def forward(self, x: torch.Tensor, cache: torch.Tensor = torch.zeros(0, 0, 0)) -> Tuple[torch.Tensor, torch.Tensor]: + x = self.upsample(x) + input_timestep = x.shape[2] + if cache.size(2) == 0: + x = F.pad(x, (self.causal_padding, 0), value=0.0) + else: + assert cache.size(2) == self.causal_padding + x = torch.concat([cache, x], dim=2) + x = super(CausalConv1dUpsample, self).forward(x) + assert input_timestep == x.shape[2] + return x diff --git a/freevideo_engine/dubbing/cosyvoice3/transformer/upsample_encoder.py b/freevideo_engine/dubbing/cosyvoice3/transformer/upsample_encoder.py new file mode 100644 index 00000000..4067e8e0 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/transformer/upsample_encoder.py @@ -0,0 +1,60 @@ +# Copyright (c) 2021 Mobvoi Inc (Binbin Zhang, Di Wu) +# 2022 Xingchen Song (sxc19@mails.tsinghua.edu.cn) +# 2024 Alibaba Inc (Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified from ESPnet(https://github.com/espnet/espnet) +# Modified by FreeVideo: only PreLookaheadLayer is kept. +import torch +from torch import nn +from torch.nn import functional as F + + +class PreLookaheadLayer(nn.Module): + def __init__(self, in_channels: int, channels: int, pre_lookahead_len: int = 1): + super().__init__() + self.in_channels = in_channels + self.channels = channels + self.pre_lookahead_len = pre_lookahead_len + self.conv1 = nn.Conv1d( + in_channels, channels, + kernel_size=pre_lookahead_len + 1, + stride=1, padding=0, + ) + self.conv2 = nn.Conv1d( + channels, in_channels, + kernel_size=3, stride=1, padding=0, + ) + + def forward(self, inputs: torch.Tensor, context: torch.Tensor = torch.zeros(0, 0, 0)) -> torch.Tensor: + """ + inputs: (batch_size, seq_len, channels) + """ + outputs = inputs.transpose(1, 2).contiguous() + context = context.transpose(1, 2).contiguous() + # look ahead + if context.size(2) == 0: + outputs = F.pad(outputs, (0, self.pre_lookahead_len), mode='constant', value=0.0) + else: + assert self.training is False, 'you have passed context, make sure that you are running inference mode' + assert context.size(2) == self.pre_lookahead_len + outputs = F.pad(torch.concat([outputs, context], dim=2), (0, self.pre_lookahead_len - context.size(2)), mode='constant', value=0.0) + outputs = F.leaky_relu(self.conv1(outputs)) + # outputs + outputs = F.pad(outputs, (self.conv2.kernel_size[0] - 1, 0), mode='constant', value=0.0) + outputs = self.conv2(outputs) + outputs = outputs.transpose(1, 2).contiguous() + + # residual connection + outputs = outputs + inputs + return outputs diff --git a/freevideo_engine/dubbing/cosyvoice3/utils/__init__.py b/freevideo_engine/dubbing/cosyvoice3/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freevideo_engine/dubbing/cosyvoice3/utils/common.py b/freevideo_engine/dubbing/cosyvoice3/utils/common.py new file mode 100644 index 00000000..3a8ad7ab --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/utils/common.py @@ -0,0 +1,167 @@ +# Copyright (c) 2020 Mobvoi Inc (Binbin Zhang) +# 2024 Alibaba Inc (authors: Xiang Lyu) +# 2025 Alibaba Inc (authors: Xiang Lyu, Bofan Zhou) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified from ESPnet(https://github.com/espnet/espnet) +"""Unility functions for Transformer.""" + +import random +from typing import List + +import numpy as np +import torch + +IGNORE_ID = -1 + +def pad_list(xs: List[torch.Tensor], pad_value: int): + """Perform padding for the list of tensors. + + Args: + xs (List): List of Tensors [(T_1, `*`), (T_2, `*`), ..., (T_B, `*`)]. + pad_value (float): Value for padding. + + Returns: + Tensor: Padded tensor (B, Tmax, `*`). + + Examples: + >>> x = [torch.ones(4), torch.ones(2), torch.ones(1)] + >>> x + [tensor([1., 1., 1., 1.]), tensor([1., 1.]), tensor([1.])] + >>> pad_list(x, 0) + tensor([[1., 1., 1., 1.], + [1., 1., 0., 0.], + [1., 0., 0., 0.]]) + + """ + max_len = max([len(item) for item in xs]) + batchs = len(xs) + ndim = xs[0].ndim + if ndim == 1: + pad_res = torch.zeros(batchs, + max_len, + dtype=xs[0].dtype, + device=xs[0].device) + elif ndim == 2: + pad_res = torch.zeros(batchs, + max_len, + xs[0].shape[1], + dtype=xs[0].dtype, + device=xs[0].device) + elif ndim == 3: + pad_res = torch.zeros(batchs, + max_len, + xs[0].shape[1], + xs[0].shape[2], + dtype=xs[0].dtype, + device=xs[0].device) + else: + raise ValueError(f"Unsupported ndim: {ndim}") + pad_res.fill_(pad_value) + for i in range(batchs): + pad_res[i, :len(xs[i])] = xs[i] + return pad_res + + +def th_accuracy(pad_outputs: torch.Tensor, pad_targets: torch.Tensor, + ignore_label: int) -> torch.Tensor: + """Calculate accuracy. + + Args: + pad_outputs (Tensor): Prediction tensors (B * Lmax, D). + pad_targets (LongTensor): Target label tensors (B, Lmax). + ignore_label (int): Ignore label id. + + Returns: + torch.Tensor: Accuracy value (0.0 - 1.0). + + """ + pad_pred = pad_outputs.view(pad_targets.size(0), pad_targets.size(1), + pad_outputs.size(1)).argmax(2) + mask = pad_targets != ignore_label + numerator = torch.sum( + pad_pred.masked_select(mask) == pad_targets.masked_select(mask)) + denominator = torch.sum(mask) + return (numerator / denominator).detach() + + +def get_padding(kernel_size, dilation=1): + return int((kernel_size * dilation - dilation) / 2) + + +def init_weights(m, mean=0.0, std=0.01): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + m.weight.data.normal_(mean, std) + + +# Repetition Aware Sampling in VALL-E 2 +def ras_sampling(weighted_scores, decoded_tokens, sampling, top_p=0.8, top_k=25, win_size=10, tau_r=0.1): + top_ids = nucleus_sampling(weighted_scores, top_p=top_p, top_k=top_k) + rep_num = (torch.tensor(decoded_tokens[-win_size:]).to(weighted_scores.device) == top_ids).sum().item() + if rep_num >= win_size * tau_r: + weighted_scores[top_ids] = -float('inf') + top_ids = random_sampling(weighted_scores, decoded_tokens, sampling) + return top_ids + + +def nucleus_sampling(weighted_scores, top_p=0.8, top_k=25): + prob, indices = [], [] + cum_prob = 0.0 + sorted_value, sorted_idx = weighted_scores.softmax(dim=0).sort(descending=True, stable=True) + for i in range(len(sorted_idx)): + # sampling both top-p and numbers. + if cum_prob < top_p and len(prob) < top_k: + cum_prob += sorted_value[i] + prob.append(sorted_value[i]) + indices.append(sorted_idx[i]) + else: + break + prob = torch.tensor(prob).to(weighted_scores) + indices = torch.tensor(indices, dtype=torch.long).to(weighted_scores.device) + top_ids = indices[prob.multinomial(1, replacement=True)].item() + return top_ids + + +def random_sampling(weighted_scores, decoded_tokens, sampling): + top_ids = weighted_scores.softmax(dim=0).multinomial(1, replacement=True).item() + return top_ids + + +def fade_in_out(fade_in_mel, fade_out_mel, window): + device = fade_in_mel.device + fade_in_mel, fade_out_mel = fade_in_mel.cpu(), fade_out_mel.cpu() + mel_overlap_len = int(window.shape[0] / 2) + if fade_in_mel.device == torch.device('cpu'): + fade_in_mel = fade_in_mel.clone() + fade_in_mel[..., :mel_overlap_len] = fade_in_mel[..., :mel_overlap_len] * window[:mel_overlap_len] + \ + fade_out_mel[..., -mel_overlap_len:] * window[mel_overlap_len:] + return fade_in_mel.to(device) + + +def set_all_random_seed(seed): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def mask_to_bias(mask: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + assert mask.dtype == torch.bool + assert dtype in [torch.float32, torch.bfloat16, torch.float16] + mask = mask.to(dtype) + # attention mask bias + # NOTE(Mddct): torch.finfo jit issues + # chunk_masks = (1.0 - chunk_masks) * torch.finfo(dtype).min + mask = (1.0 - mask) * -1.0e+10 + return mask diff --git a/freevideo_engine/dubbing/cosyvoice3/utils/file_utils.py b/freevideo_engine/dubbing/cosyvoice3/utils/file_utils.py new file mode 100644 index 00000000..e9a6e363 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/utils/file_utils.py @@ -0,0 +1,32 @@ +# Copyright (c) 2021 Mobvoi Inc. (authors: Binbin Zhang) +# 2024 Alibaba Inc (authors: Xiang Lyu, Zetao Hu) +# 2025 Alibaba Inc (authors: Xiang Lyu, Yabin Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Modified by FreeVideo: only load_wav is kept. + +import logging + +import torch +import torchaudio + + +def load_wav(wav, target_sr, min_sr=16000): + from freevideo_engine.dubbing import wavio + data, sample_rate = wavio.read(wav) + speech = torch.from_numpy(data.T.copy()) + speech = speech.mean(dim=0, keepdim=True) + if sample_rate != target_sr: + assert sample_rate >= min_sr, 'wav sample rate {} must be greater than {}'.format(sample_rate, target_sr) + speech = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=target_sr)(speech) + return speech diff --git a/freevideo_engine/dubbing/cosyvoice3/utils/frontend_utils.py b/freevideo_engine/dubbing/cosyvoice3/utils/frontend_utils.py new file mode 100644 index 00000000..ea1c9fc8 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/utils/frontend_utils.py @@ -0,0 +1,136 @@ +# Copyright (c) 2024 Alibaba Inc (authors: Xiang Lyu, Zhihao Du) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import regex +chinese_char_pattern = re.compile(r'[\u4e00-\u9fff]+') + + +# whether contain chinese character +def contains_chinese(text): + return bool(chinese_char_pattern.search(text)) + + +# replace special symbol +def replace_corner_mark(text): + text = text.replace('²', '平方') + text = text.replace('³', '立方') + return text + + +# remove meaningless symbol +def remove_bracket(text): + text = text.replace('（', '').replace('）', '') + text = text.replace('【', '').replace('】', '') + text = text.replace('`', '').replace('`', '') + text = text.replace("——", " ") + return text + + +# spell Arabic numerals +def spell_out_number(text: str, inflect_parser): + new_text = [] + st = None + for i, c in enumerate(text): + if not c.isdigit(): + if st is not None: + num_str = inflect_parser.number_to_words(text[st: i]) + new_text.append(num_str) + st = None + new_text.append(c) + else: + if st is None: + st = i + if st is not None and st < len(text): + num_str = inflect_parser.number_to_words(text[st:]) + new_text.append(num_str) + return ''.join(new_text) + + +# split paragrah logic： +# 1. per sentence max len token_max_n, min len token_min_n, merge if last sentence len less than merge_len +# 2. cal sentence len according to lang +# 3. split sentence according to puncatation +def split_paragraph(text: str, tokenize, lang="zh", token_max_n=80, token_min_n=60, merge_len=20, comma_split=False): + def calc_utt_length(_text: str): + if lang == "zh": + return len(_text) + else: + return len(tokenize(_text)) + + def should_merge(_text: str): + if lang == "zh": + return len(_text) < merge_len + else: + return len(tokenize(_text)) < merge_len + + if lang == "zh": + pounc = ['。', '？', '！', '；', '：', '、', '.', '?', '!', ';'] + else: + pounc = ['.', '?', '!', ';', ':'] + if comma_split: + pounc.extend(['，', ',']) + + if text[-1] not in pounc: + if lang == "zh": + text += "。" + else: + text += "." + + st = 0 + utts = [] + for i, c in enumerate(text): + if c in pounc: + if len(text[st: i]) > 0: + utts.append(text[st: i] + c) + if i + 1 < len(text) and text[i + 1] in ['"', '”']: + tmp = utts.pop(-1) + utts.append(tmp + text[i + 1]) + st = i + 2 + else: + st = i + 1 + + final_utts = [] + cur_utt = "" + for utt in utts: + if calc_utt_length(cur_utt + utt) > token_max_n and calc_utt_length(cur_utt) > token_min_n: + final_utts.append(cur_utt) + cur_utt = "" + cur_utt = cur_utt + utt + if len(cur_utt) > 0: + if should_merge(cur_utt) and len(final_utts) != 0: + final_utts[-1] = final_utts[-1] + cur_utt + else: + final_utts.append(cur_utt) + + return final_utts + + +# remove blank between chinese character +def replace_blank(text: str): + out_str = [] + for i, c in enumerate(text): + if c == " ": + if ((text[i + 1].isascii() and text[i + 1] != " ") and + (text[i - 1].isascii() and text[i - 1] != " ")): + out_str.append(c) + else: + out_str.append(c) + return "".join(out_str) + + +def is_only_punctuation(text): + # Regular expression: Match strings that consist only of punctuation marks or are empty. + punctuation_pattern = r'^[\p{P}\p{S}]*$' + return bool(regex.fullmatch(punctuation_pattern, text)) diff --git a/freevideo_engine/dubbing/cosyvoice3/utils/mask.py b/freevideo_engine/dubbing/cosyvoice3/utils/mask.py new file mode 100644 index 00000000..5d3dfd6c --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/utils/mask.py @@ -0,0 +1,265 @@ +# Copyright (c) 2019 Shigeki Karita +# 2020 Mobvoi Inc (Binbin Zhang) +# 2024 Alibaba Inc (authors: Xiang Lyu) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +''' +def subsequent_mask( + size: int, + device: torch.device = torch.device("cpu"), +) -> torch.Tensor: + """Create mask for subsequent steps (size, size). + + This mask is used only in decoder which works in an auto-regressive mode. + This means the current step could only do attention with its left steps. + + In encoder, fully attention is used when streaming is not necessary and + the sequence is not long. In this case, no attention mask is needed. + + When streaming is need, chunk-based attention is used in encoder. See + subsequent_chunk_mask for the chunk-based attention mask. + + Args: + size (int): size of mask + str device (str): "cpu" or "cuda" or torch.Tensor.device + dtype (torch.device): result dtype + + Returns: + torch.Tensor: mask + + Examples: + >>> subsequent_mask(3) + [[1, 0, 0], + [1, 1, 0], + [1, 1, 1]] + """ + ret = torch.ones(size, size, device=device, dtype=torch.bool) + return torch.tril(ret) +''' + + +def subsequent_mask( + size: int, + device: torch.device = torch.device("cpu"), +) -> torch.Tensor: + """Create mask for subsequent steps (size, size). + + This mask is used only in decoder which works in an auto-regressive mode. + This means the current step could only do attention with its left steps. + + In encoder, fully attention is used when streaming is not necessary and + the sequence is not long. In this case, no attention mask is needed. + + When streaming is need, chunk-based attention is used in encoder. See + subsequent_chunk_mask for the chunk-based attention mask. + + Args: + size (int): size of mask + str device (str): "cpu" or "cuda" or torch.Tensor.device + dtype (torch.device): result dtype + + Returns: + torch.Tensor: mask + + Examples: + >>> subsequent_mask(3) + [[1, 0, 0], + [1, 1, 0], + [1, 1, 1]] + """ + arange = torch.arange(size, device=device) + mask = arange.expand(size, size) + arange = arange.unsqueeze(-1) + mask = mask <= arange + return mask + + +def subsequent_chunk_mask_deprecated( + size: int, + chunk_size: int, + num_left_chunks: int = -1, + device: torch.device = torch.device("cpu"), +) -> torch.Tensor: + """Create mask for subsequent steps (size, size) with chunk size, + this is for streaming encoder + + Args: + size (int): size of mask + chunk_size (int): size of chunk + num_left_chunks (int): number of left chunks + <0: use full chunk + >=0: use num_left_chunks + device (torch.device): "cpu" or "cuda" or torch.Tensor.device + + Returns: + torch.Tensor: mask + + Examples: + >>> subsequent_chunk_mask(4, 2) + [[1, 1, 0, 0], + [1, 1, 0, 0], + [1, 1, 1, 1], + [1, 1, 1, 1]] + """ + ret = torch.zeros(size, size, device=device, dtype=torch.bool) + for i in range(size): + if num_left_chunks < 0: + start = 0 + else: + start = max((i // chunk_size - num_left_chunks) * chunk_size, 0) + ending = min((i // chunk_size + 1) * chunk_size, size) + ret[i, start:ending] = True + return ret + + +def subsequent_chunk_mask( + size: int, + chunk_size: int, + num_left_chunks: int = -1, + device: torch.device = torch.device("cpu"), +) -> torch.Tensor: + """Create mask for subsequent steps (size, size) with chunk size, + this is for streaming encoder + + Args: + size (int): size of mask + chunk_size (int): size of chunk + num_left_chunks (int): number of left chunks + <0: use full chunk + >=0: use num_left_chunks + device (torch.device): "cpu" or "cuda" or torch.Tensor.device + + Returns: + torch.Tensor: mask + + Examples: + >>> subsequent_chunk_mask(4, 2) + [[1, 1, 0, 0], + [1, 1, 0, 0], + [1, 1, 1, 1], + [1, 1, 1, 1]] + """ + # NOTE this modified implementation meets onnx export requirements, but it doesn't support num_left_chunks + pos_idx = torch.arange(size, device=device) + block_value = (torch.div(pos_idx, chunk_size, rounding_mode='trunc') + 1) * chunk_size + ret = pos_idx.unsqueeze(0) < block_value.unsqueeze(1) + return ret + + +def add_optional_chunk_mask(xs: torch.Tensor, + masks: torch.Tensor, + use_dynamic_chunk: bool, + use_dynamic_left_chunk: bool, + decoding_chunk_size: int, + static_chunk_size: int, + num_decoding_left_chunks: int, + enable_full_context: bool = True): + """ Apply optional mask for encoder. + + Args: + xs (torch.Tensor): padded input, (B, L, D), L for max length + mask (torch.Tensor): mask for xs, (B, 1, L) + use_dynamic_chunk (bool): whether to use dynamic chunk or not + use_dynamic_left_chunk (bool): whether to use dynamic left chunk for + training. + decoding_chunk_size (int): decoding chunk size for dynamic chunk, it's + 0: default for training, use random dynamic chunk. + <0: for decoding, use full chunk. + >0: for decoding, use fixed chunk size as set. + static_chunk_size (int): chunk size for static chunk training/decoding + if it's greater than 0, if use_dynamic_chunk is true, + this parameter will be ignored + num_decoding_left_chunks: number of left chunks, this is for decoding, + the chunk size is decoding_chunk_size. + >=0: use num_decoding_left_chunks + <0: use all left chunks + enable_full_context (bool): + True: chunk size is either [1, 25] or full context(max_len) + False: chunk size ~ U[1, 25] + + Returns: + torch.Tensor: chunk mask of the input xs. + """ + # Whether to use chunk mask or not + if use_dynamic_chunk: + max_len = xs.size(1) + if decoding_chunk_size < 0: + chunk_size = max_len + num_left_chunks = -1 + elif decoding_chunk_size > 0: + chunk_size = decoding_chunk_size + num_left_chunks = num_decoding_left_chunks + else: + # chunk size is either [1, 25] or full context(max_len). + # Since we use 4 times subsampling and allow up to 1s(100 frames) + # delay, the maximum frame is 100 / 4 = 25. + chunk_size = torch.randint(1, max_len, (1, )).item() + num_left_chunks = -1 + if chunk_size > max_len // 2 and enable_full_context: + chunk_size = max_len + else: + chunk_size = chunk_size % 25 + 1 + if use_dynamic_left_chunk: + max_left_chunks = (max_len - 1) // chunk_size + num_left_chunks = torch.randint(0, max_left_chunks, + (1, )).item() + chunk_masks = subsequent_chunk_mask(xs.size(1), chunk_size, + num_left_chunks, + xs.device) # (L, L) + chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L) + chunk_masks = masks & chunk_masks # (B, L, L) + elif static_chunk_size > 0: + num_left_chunks = num_decoding_left_chunks + chunk_masks = subsequent_chunk_mask(xs.size(1), static_chunk_size, + num_left_chunks, + xs.device) # (L, L) + chunk_masks = chunk_masks.unsqueeze(0) # (1, L, L) + chunk_masks = masks & chunk_masks # (B, L, L) + else: + chunk_masks = masks + assert chunk_masks.dtype == torch.bool + if (chunk_masks.sum(dim=-1) == 0).sum().item() != 0: + print('get chunk_masks all false at some timestep, force set to true, make sure they are masked in futuer computation!') + chunk_masks[chunk_masks.sum(dim=-1) == 0] = True + return chunk_masks + + +def make_pad_mask(lengths: torch.Tensor, max_len: int = 0) -> torch.Tensor: + """Make mask tensor containing indices of padded part. + + See description of make_non_pad_mask. + + Args: + lengths (torch.Tensor): Batch of lengths (B,). + Returns: + torch.Tensor: Mask tensor containing indices of padded part. + + Examples: + >>> lengths = [5, 3, 2] + >>> make_pad_mask(lengths) + masks = [[0, 0, 0, 0 ,0], + [0, 0, 0, 1, 1], + [0, 0, 1, 1, 1]] + """ + batch_size = lengths.size(0) + max_len = max_len if max_len > 0 else lengths.max().item() + seq_range = torch.arange(0, + max_len, + dtype=torch.int64, + device=lengths.device) + seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len) + seq_length_expand = lengths.unsqueeze(-1) + mask = seq_range_expand >= seq_length_expand + return mask diff --git a/freevideo_engine/dubbing/cosyvoice3/utils/mel.py b/freevideo_engine/dubbing/cosyvoice3/utils/mel.py new file mode 100644 index 00000000..bca293d1 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3/utils/mel.py @@ -0,0 +1,98 @@ +import numpy as np +import torch + + +def _hz_to_mel(f): + f = np.asarray(f, dtype=np.float64) + lin = f / (200.0 / 3) + log = 15.0 + np.log(np.maximum(f, 1e-10) / 1000.0) / (np.log(6.4) / 27.0) + return np.where(f >= 1000.0, log, lin) + + +def _mel_to_hz(m): + m = np.asarray(m, dtype=np.float64) + lin = (200.0 / 3) * m + log = 1000.0 * np.exp((np.log(6.4) / 27.0) * (m - 15.0)) + return np.where(m >= 15.0, log, lin) + + +def librosa_mel_fn(sr, n_fft, n_mels, fmin=0.0, fmax=None): + """Slaney-scale, Slaney-normalised mel filterbank (same result as librosa.filters.mel, float32).""" + fmax = sr / 2.0 if fmax is None else fmax + freqs = np.linspace(0.0, sr / 2.0, n_fft // 2 + 1) + pts = _mel_to_hz(np.linspace(_hz_to_mel(fmin), _hz_to_mel(fmax), n_mels + 2)) + fdiff = np.diff(pts) + ramps = pts[:, None] - freqs[None, :] + lower = -ramps[:-2] / fdiff[:-1, None] + upper = ramps[2:] / fdiff[1:, None] + w = np.maximum(0.0, np.minimum(lower, upper)) + w *= (2.0 / (pts[2:n_mels + 2] - pts[:n_mels]))[:, None] + return w.astype(np.float32) + +_FILTERS = {} + + +def log_mel_spectrogram(audio, n_mels=128, n_fft=400, hop=160, sr=16000): + """Whisper-compatible log-mel spectrogram (Slaney filters, log10, floor at max - 8, scaled to ~[-1, 1]).""" + key = (n_mels, n_fft, sr, audio.device) + if key not in _FILTERS: + _FILTERS[key] = torch.from_numpy(librosa_mel_fn(sr, n_fft, n_mels)).to(audio.device) + window = torch.hann_window(n_fft).to(audio.device) + stft = torch.stft(audio, n_fft, hop, window=window, return_complex=True) + magnitudes = stft[..., :-1].abs() ** 2 + log_spec = torch.clamp(_FILTERS[key] @ magnitudes, min=1e-10).log10() + log_spec = torch.maximum(log_spec, log_spec.max() - 8.0) + return (log_spec + 4.0) / 4.0 + + +# mel_spectrogram below is copied from Matcha-TTS matcha/utils/audio.py (MIT; derived from HiFi-GAN, MIT) +def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def spectral_normalize_torch(magnitudes): + return dynamic_range_compression_torch(magnitudes) + + +mel_basis = {} +hann_window = {} + + +def mel_spectrogram(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): + if torch.min(y) < -1.0: + print("min value is ", torch.min(y)) + if torch.max(y) > 1.0: + print("max value is ", torch.max(y)) + + global mel_basis, hann_window # pylint: disable=global-statement,global-variable-not-assigned + if f"{str(fmax)}_{str(y.device)}" not in mel_basis: + mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) + mel_basis[str(fmax) + "_" + str(y.device)] = torch.from_numpy(mel).float().to(y.device) + hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device) + + y = torch.nn.functional.pad( + y.unsqueeze(1), (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), mode="reflect" + ) + y = y.squeeze(1) + + spec = torch.view_as_real( + torch.stft( + y, + n_fft, + hop_length=hop_size, + win_length=win_size, + window=hann_window[str(y.device)], + center=center, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + ) + + spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)) + + spec = torch.matmul(mel_basis[str(fmax) + "_" + str(y.device)], spec) + spec = spectral_normalize_torch(spec) + + return spec diff --git a/freevideo_engine/dubbing/cosyvoice3_pack.json b/freevideo_engine/dubbing/cosyvoice3_pack.json new file mode 100644 index 00000000..9becf958 --- /dev/null +++ b/freevideo_engine/dubbing/cosyvoice3_pack.json @@ -0,0 +1,159 @@ +{ + "format": 1, + "id": "cosyvoice3", + "name": "Fun-CosyVoice3-0.5B-2512", + "license": "Apache-2.0", + "source": "https://github.com/FunAudioLLM/CosyVoice", + "modelscope": { + "repo": "FunAudioLLM/Fun-CosyVoice3-0.5B-2512", + "revision": "master" + }, + "huggingface": { + "repo": "FunAudioLLM/Fun-CosyVoice3-0.5B-2512" + }, + "files": [ + { + "file": "llm.pt", + "bytes": 2024669519, + "sha256": "69f43bd545131c30e98947fb360ea8b4dc9916d8e83dded7757c7ea4f5a24970" + }, + { + "file": "flow.pt", + "bytes": 1329116148, + "sha256": "a6fab32a7825e5b0bc855ddd948f8db9370b0a786fbc249caa4595e95b608e4b" + }, + { + "file": "hift.pt", + "bytes": 83202622, + "sha256": "b279d7641eb97ae55b3b540cfba4f953c26492a2df758328a89a4d007ab87a65" + }, + { + "file": "campplus.onnx", + "bytes": 28303423, + "sha256": "a6ac6a63997761ae2997373e2ee1c47040854b4b759ea41ec48e4e42df0f4d73" + }, + { + "file": "speech_tokenizer_v3.onnx", + "bytes": 969451503, + "sha256": "23236a74175dbdda47afc66dbadd5bcb41303c467a57c261cb8539ad9db9208d" + }, + { + "file": "CosyVoice-BlankEN/config.json", + "bytes": 659, + "sha256": "168aa1bd401abc3bc262ba15ba4e499627a8b4e006e9d050b47c22de20660185" + }, + { + "file": "CosyVoice-BlankEN/generation_config.json", + "bytes": 242, + "sha256": "e558847a8b4402616f1273797b015104dc266fe4b520056fca88823ba8f8ebe6" + }, + { + "file": "CosyVoice-BlankEN/merges.txt", + "bytes": 1402109, + "sha256": "ac8ff86a72bee70828fbc1119bc4398c6f3a9a6e490d7b0dbe917be025478bd0" + }, + { + "file": "CosyVoice-BlankEN/tokenizer_config.json", + "bytes": 1287, + "sha256": "482bd979881423375ca5414e4e0d94cd7c5349dbb17fffd46b4d36d71e62a1bc" + }, + { + "file": "CosyVoice-BlankEN/vocab.json", + "bytes": 2776833, + "sha256": "ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910" + } + ], + "runtime": { + "python": "cp312", + "note": "Hash-locked PyPI wheels, downloaded when the voice pack is installed; not part of the installer.", + "wheels": { + "linux-x86_64": [ + { + "name": "onnxruntime", + "version": "1.31.1", + "license": "MIT", + "file": "onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_x86_64.whl", + "bytes": 23760988, + "sha256": "1f3dea47f2f0e5117c2404cea7d21fd253ae7836a3bf167a501cdfbb2600d536", + "path": "eb/27/c6a823bac55b2977bea2027c3ec9393a2c1002cb9a09eb40354563e02730/onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_x86_64.whl" + }, + { + "name": "flatbuffers", + "version": "25.12.19", + "license": "Apache-2.0", + "file": "flatbuffers-25.12.19-py2.py3-none-any.whl", + "bytes": 26661, + "sha256": "7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", + "path": "e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl" + }, + { + "name": "protobuf", + "version": "7.36.2", + "license": "BSD-3-Clause", + "file": "protobuf-7.36.2-cp310-abi3-manylinux2014_x86_64.whl", + "bytes": 343223, + "sha256": "89f23aa53c24553a2416fd4fd1ec06f74fa42b14b546d8883128813f775bbfd2", + "path": "db/f3/3996583dd2906297a637af12114deddf7658af6e683fedb83be061983fb5/protobuf-7.36.2-cp310-abi3-manylinux2014_x86_64.whl" + } + ], + "windows-x64": [ + { + "name": "onnxruntime", + "version": "1.31.1", + "license": "MIT", + "file": "onnxruntime-1.31.1-cp312-cp312-win_amd64.whl", + "bytes": 14888466, + "sha256": "ae8a9a351efda685cc59a66226cdefaa37a126c1d66872ca23a2de9425326130", + "path": "45/b2/793b4051d8dc6acb379ead9b72050f0ffa283ca49a6904add2e218b83e2e/onnxruntime-1.31.1-cp312-cp312-win_amd64.whl" + }, + { + "name": "flatbuffers", + "version": "25.12.19", + "license": "Apache-2.0", + "file": "flatbuffers-25.12.19-py2.py3-none-any.whl", + "bytes": 26661, + "sha256": "7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", + "path": "e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl" + }, + { + "name": "protobuf", + "version": "7.36.2", + "license": "BSD-3-Clause", + "file": "protobuf-7.36.2-cp310-abi3-win_amd64.whl", + "bytes": 456514, + "sha256": "a300819d441e078a5608c0d3c709796bb548136058fda017ae51d425b44fd353", + "path": "8a/55/b77bda4e5e5f5971fb51b07663694690e9afdb9402136c16a522bd621cad/protobuf-7.36.2-cp310-abi3-win_amd64.whl" + } + ], + "macos-arm64": [ + { + "name": "onnxruntime", + "version": "1.31.1", + "license": "MIT", + "file": "onnxruntime-1.31.1-cp312-cp312-macosx_14_0_arm64.whl", + "bytes": 20882125, + "sha256": "96dd2564b10ea7153ad9c207e7cd8e5917cb3ba521e0e0a62ac03486fdc8d56c", + "path": "0e/f7/cecc0efeeea83bff2f7789ae1b80f27fa0f0210d6d9d66d1f0031a6b5e47/onnxruntime-1.31.1-cp312-cp312-macosx_14_0_arm64.whl" + }, + { + "name": "flatbuffers", + "version": "25.12.19", + "license": "Apache-2.0", + "file": "flatbuffers-25.12.19-py2.py3-none-any.whl", + "bytes": 26661, + "sha256": "7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", + "path": "e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl" + }, + { + "name": "protobuf", + "version": "7.36.2", + "license": "BSD-3-Clause", + "file": "protobuf-7.36.2-cp310-abi3-macosx_10_9_universal2.whl", + "bytes": 456039, + "sha256": "cbc70b17ee27e28894c7fee8bb04be1abead49e936bc70eb60052531eee2079e", + "path": "32/72/98342feb672507c8f3a69e34b4fa8961f608edba5c1a48a6f47156d92cb5/protobuf-7.36.2-cp310-abi3-macosx_10_9_universal2.whl" + } + ] + } + } +} diff --git a/freevideo_engine/dubbing/pack.py b/freevideo_engine/dubbing/pack.py new file mode 100644 index 00000000..51548698 --- /dev/null +++ b/freevideo_engine/dubbing/pack.py @@ -0,0 +1,365 @@ +"""On-demand download of the dubbing model (Fun-CosyVoice3-0.5B-2512) and its inference runtime. + +Nothing here ships in the installer. The catalog (`cosyvoice3_pack.json`) lists every file with its +size and sha256; a file is used only after it matched. Layout under `/dubbing/`: +`cosyvoice3/` (the weights), `wheels/` (the runtime wheels as downloaded), `runtime//` (the wheels +unpacked; put on `sys.path` before `import onnxruntime`) and `licenses/` (the runtime licence texts). +ModelScope has no commit that matches a Hugging Face one, so every source is accepted only when the +size and SHA-256 in the catalog match, and the file names are the same on all of them. The runtime +wheels are the PyPI files, fetched from PyPI or its domestic mirror by the same rule. onnxruntime also +requires packaging and sympy; both come with torch in the unified environment, so they are not listed. +The wheels are chosen for the Python of the service process, which is the pinned 3.12 of the product's +own environment. A user's own ComfyUI Python of another version would fail the import in the worker, +which reports it as a general failure. +""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import sys +import zipfile + +from .. import disk_space + +FAMILY = 'image-models' +RESERVE = 256 * 2**20 +LEDGER = 'download-verified.json' +NAME = 'cosyvoice3' +COMPLETE = 'complete.json' +MAX_UNPACKED = 256 * 2**20 +LICENSES = 'RUNTIME-LICENSES.txt' +# The two PyPI hosts of the `pypi` download family; a wheel's `path` is the same on both. +PYPI_FILES = {'official': 'https://files.pythonhosted.org/packages/', 'tuna': 'https://pypi.tuna.tsinghua.edu.cn/packages/'} +MODULES = (('onnxruntime', '__init__.py'), ('flatbuffers', '__init__.py'), ('google', 'protobuf', '__init__.py')) + + +class PackError(Exception): + def __init__(self, reason): + super().__init__(reason) + self.reason = reason + + +def catalog(): + """The file list shipped beside this file; a frozen launcher's bundled copy answers when it is gone.""" + name = 'cosyvoice3_pack.json' + paths = [Path(__file__).with_name(name)] + bundle = getattr(sys, '_MEIPASS', None) + if bundle: + paths.append(Path(bundle) / 'engine-source' / 'freevideo_engine' / 'dubbing' / name) + for path in paths: + try: + return json.loads(path.read_text(encoding='utf-8')) + except (OSError, ValueError) as error: + missing = error + raise missing + + +def rows(spec=None): + return list((spec or catalog())['files']) + + +def base(root=None): + if root is None: + from ..paths import model_root + root = model_root() + return Path(root) / 'dubbing' + + +def directory(root=None): + return base(root) / NAME + + +def _ledger(folder): + try: + value = json.loads((folder / LEDGER).read_text(encoding='utf-8')) + return value if isinstance(value, dict) else {} + except (OSError, ValueError): + return {} + + +def _intact(folder, row, ledger): + """Size and the stamp recorded after the last hash match; no hashing here.""" + from .. import provision + path = folder / row['file'] + try: + if not path.is_file() or path.stat().st_size != row['bytes']: + return False + prior = ledger.get(str(path)) + return isinstance(prior, dict) and all( + prior.get(key) == value for key, value in provision.file_identity(path, row).items()) + except OSError: + return False + + +def wheel_rows(spec=None): + """This machine's runtime wheels, or None (other platform or Python, or none listed).""" + from ..subtitle_pack import platform_key + runtime = (spec or catalog()).get('runtime') or {} + if runtime.get('python') != 'cp%d%d' % sys.version_info[:2]: + return None + return list((runtime.get('wheels') or {}).get(platform_key()) or []) or None + + +def runtime_id(wheels): + return hashlib.sha256('\n'.join(row['sha256'] for row in wheels).encode('ascii')).hexdigest()[:12] + + +def runtime_folder(root=None, spec=None): + wheels = wheel_rows(spec) + return base(root) / 'runtime' / runtime_id(wheels) if wheels else None + + +def _unpacked(root=None, spec=None): + folder = runtime_folder(root, spec) + if folder is None: + return False + try: + receipt = json.loads((folder / COMPLETE).read_text(encoding='utf-8')) + return receipt.get('id') == folder.name and all(folder.joinpath(*parts).is_file() for parts in MODULES) + except (OSError, ValueError, KeyError, TypeError): + return False + + +def runtime_path(root=None): + """The folder to put on sys.path for onnxruntime, or None while it is not unpacked.""" + try: + return str(runtime_folder(root)) if _unpacked(root) else None + except (OSError, ValueError, KeyError, TypeError): + return None + + +def activate(path): + """Put an unpacked runtime on sys.path (once); Windows also needs its DLL folders.""" + if not path: + return False + folder = Path(path) + if str(folder) not in sys.path: + sys.path.insert(0, str(folder)) + if sys.platform == 'win32' and hasattr(os, 'add_dll_directory'): + for item in {folder} | {found.parent for found in folder.rglob('*.dll')}: + os.add_dll_directory(str(item)) + return True + + +def ready(root=None, spec=None): + """Cheap and safe on a UI thread: sizes and the identity stamps of the last content check.""" + try: + spec = spec or catalog() + folder = directory(root) + ledger = _ledger(folder) + return _unpacked(root, spec) and all(_intact(folder, row, ledger) for row in rows(spec)) + except (OSError, ValueError, KeyError, TypeError): + return False + + +def settle(root=None, spec=None): + """Same answer, but a file that moved without a stamp (a copied folder) is hashed here once. + For request handlers, never for a UI thread or under a lock.""" + if ready(root, spec): + return True + try: + from .. import provision + from ..monitoring import save + spec = spec or catalog() + folder = directory(root) + if not _unpacked(root, spec) or not folder.is_dir(): + return False + ledger = _ledger(folder) + found = all(provision.verified(folder / row['file'], row, ledger) for row in rows(spec)) + if found: + save(folder / LEDGER, ledger) + return found + except (OSError, ValueError, KeyError, TypeError): + return False + + +def missing_bytes(root=None, spec=None): + spec = spec or catalog() + folder = directory(root) + ledger = _ledger(folder) + weights = sum(row['bytes'] for row in rows(spec) if not _intact(folder, row, ledger)) + return weights + (0 if _unpacked(root, spec) else sum(row['bytes'] for row in wheel_rows(spec) or ())) + + +def status(root=None): + """What a page may show: the size of the download and whether the model is in place.""" + spec = catalog() + wheels = wheel_rows(spec) + total = sum(row['bytes'] for row in rows(spec)) + sum(row['bytes'] for row in wheels or ()) + return dict(name=NAME, license=spec.get('license'), bytes=missing_bytes(root, spec), total=total, + ready=ready(root, spec), supported=wheels is not None, + published=bool(spec.get('modelscope') or spec.get('huggingface'))) + + +def remote(spec, row): + """One catalog file with every source FreeVideo may fetch it from (ModelScope, Hugging Face, mirror).""" + from ..network import source_url + sources = [] + mirror = spec.get('modelscope') + if mirror: + sources.append(dict(id='modelscope', url=source_url(FAMILY, 'modelscope') + '/models/%s/resolve/%s/%s' % ( + mirror['repo'], mirror.get('revision', 'master'), row['file']))) + hub = spec.get('huggingface') + if hub: + path = '/%s/resolve/%s/%s' % (hub['repo'], hub.get('revision', 'main'), row['file']) + sources.append(dict(id='official', url=source_url('models', 'official') + path)) + sources.append(dict(id='hf-mirror', url=source_url('models', 'hf-mirror') + path)) + return dict(repo=(mirror or hub)['repo'], revision=(mirror or hub).get('revision', 'main'), file=row['file'], + family='image', sources=sources) + + +def runtime_plan(plan): + """The plan for the runtime wheels: the `image-models` list holds the PyPI hosts in the order the + plan prefers them (PyPI, or its domestic mirror).""" + from ..network import ordered + names = [name for name in ordered(plan, 'pypi') if name in PYPI_FILES] or ['official'] + return dict(plan, sources=dict(plan.get('sources', {}), **{FAMILY: [{'id': name} for name in names]})) + + +def runtime_remote(row): + return dict(repo='pypi/' + row['name'], revision=row['version'], file=row['file'], family='image', + sources=[dict(id=name, url=PYPI_FILES[name] + row['path']) for name in PYPI_FILES]) + + +def _unpack(root, wheels, check): + """Extract the verified wheels into runtime// and drop older runtimes. A file that two wheels + both carry, an unsafe name, or too much data fails the whole unpack.""" + folder = runtime_folder(root) + parent = folder.parent + parent.mkdir(parents=True, exist_ok=True) + work = parent / ('.unpack-' + folder.name) + shutil.rmtree(work, ignore_errors=True) + total = 0 + try: + for row in wheels: + with zipfile.ZipFile(wheels_folder(root) / row['file']) as archive: + for item in archive.infolist(): + check() + name = item.filename + parts = Path(name).parts + if (name.startswith('/') or '\\' in name or ':' in name or not parts or '..' in parts + or (item.external_attr >> 16) & 0o170000 == 0o120000): + raise PackError('failed') + total += item.file_size + if total > MAX_UNPACKED: + raise PackError('failed') + target = work.joinpath(*parts) + if item.is_dir(): + target.mkdir(parents=True, exist_ok=True) + continue + if target.exists(): + raise PackError('failed') + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(item) as source, target.open('wb') as stream: + shutil.copyfileobj(source, stream, 1024 * 1024) + if not all(work.joinpath(*parts).is_file() for parts in MODULES): + raise PackError('failed') + write_licenses(root, work, wheels) + (work / COMPLETE).write_text(json.dumps(dict(id=folder.name, bytes=total)), encoding='utf-8') + shutil.rmtree(folder, ignore_errors=True) + os.replace(work, folder) + finally: + shutil.rmtree(work, ignore_errors=True) + for other in parent.iterdir(): + if other.is_dir() and other != folder and not other.name.startswith('.unpack-'): + shutil.rmtree(other, ignore_errors=True) + + +def write_licenses(root, unpacked, wheels): + """The runtime's licence texts, with the pack and not with the installer: the name, version and + licence of every wheel, then the licence files its metadata carries.""" + parts = ['Runtime libraries of the dubbing model pack. They are downloaded when the pack is installed.\n'] + for row in wheels: + parts.append('\n%s\n%s %s, %s\n%s\n' % ('=' * 72, row['name'], row['version'], row['license'], '=' * 72)) + for info in sorted(unpacked.glob('%s-%s.dist-info' % (row['name'], row['version']))): + for path in sorted(info.rglob('*')): + if path.is_file() and path.suffix in ('', '.txt', '.md') and any( + word in path.name.upper() for word in ('LICENSE', 'NOTICE', 'COPYING')): + parts.append('\n--- %s ---\n%s\n' % (path.name, path.read_text(encoding='utf-8', errors='replace'))) + folder = base(root) / 'licenses' + folder.mkdir(parents=True, exist_ok=True) + (folder / LICENSES).write_text(''.join(parts), encoding='utf-8') + + +def wheels_folder(root=None): + return base(root) / 'wheels' + + +def install(root=None, progress=None, check=None): + """Download and verify the missing runtime wheels and model files, then unpack the runtime. + Resumable; a file is kept only after its hash matched.""" + from .. import network, provision + from ..adaln_assets import _download_plan + from ..monitoring import save + spec = catalog() + progress = progress or (lambda value: None) + check = check or (lambda: None) + if not (spec.get('modelscope') or spec.get('huggingface')): + raise PackError('not_published') + wheels = wheel_rows(spec) + if wheels is None: + raise PackError('unsupported') + folder = directory(root) + folder.mkdir(parents=True, exist_ok=True) + cache = wheels_folder(root) + cache.mkdir(parents=True, exist_ok=True) + ledger = _ledger(folder) + cached = _ledger(cache) + # A file without a stamp (a copied folder) is hashed once here and stamped. + todo = [row for row in rows(spec) if not provision.verified(folder / row['file'], row, ledger)] + save(folder / LEDGER, ledger) + fetch = [] if _unpacked(root, spec) else [row for row in wheels if not provision.verified(cache / row['file'], row, cached)] + save(cache / LEDGER, cached) + total = sum(row['bytes'] for row in todo) + sum(row['bytes'] for row in fetch) + retained = 0 + for row, where in [(row, folder) for row in todo] + [(row, cache) for row in fetch]: + partial = (where / row['file']).with_suffix((Path(row['file']).suffix) + '.partial') + if partial.is_file() and partial.stat().st_size < row['bytes']: + retained += partial.stat().st_size + if total and disk_space.free_bytes(folder) < total - retained + RESERVE: + raise PackError('disk_space') + plan = _download_plan(root) + # The first dubbing download follows the same hosts as the image model; installs whose plan has + # no entry for the family try ModelScope first, then the two Hugging Face hosts. + sources = plan['sources'] = dict(plan.get('sources', {})) + if not sources.get(FAMILY): + sources[FAMILY] = [{'id': 'modelscope'}] + [ + dict(item) for item in sources.get('models', []) if item['id'] in ('official', 'hf-mirror')] + if len(sources[FAMILY]) == 1: + sources[FAMILY] += [{'id': 'official'}, {'id': 'hf-mirror'}] + plan.update(quiet=True, resource_check=check) + wheel_plan = runtime_plan(plan) + done = 0 + + def transfer(row, path, remote_row, ledger_file, stamps, plan=plan): + nonlocal done + check() + path.parent.mkdir(parents=True, exist_ok=True) + + def moved(current, size, speed, **kwargs): + progress(dict(phase='download', done=done + current, total=total)) + network.download(network.model_urls(plan, remote_row), path, row['sha256'], moved, + network=plan, size=row['bytes'], category=FAMILY, keep_partial=True, + stall_seconds=30, slow_seconds=15, low_speed_limit=64 * 1024) + stamps[str(path)] = provision.file_identity(path, row) + save(ledger_file, stamps) + done += row['bytes'] + try: + for row in fetch: + transfer(row, cache / row['file'], runtime_remote(row), cache / LEDGER, cached, plan=wheel_plan) + if fetch or not _unpacked(root, spec): + progress(dict(phase='unpack', done=done, total=total)) + _unpack(root, wheels, check) + for row in todo: + transfer(row, folder / row['file'], remote(spec, row), folder / LEDGER, ledger) + except PackError: + raise + except InterruptedError: + raise + except ValueError as error: + raise PackError('disk_space' if str(error) == 'disk_space' else 'download_failed') from error + except OSError as error: + raise PackError('disk_space' if getattr(error, 'errno', None) == 28 else 'download_failed') from error + except Exception as error: + raise PackError('download_failed') from error diff --git a/freevideo_engine/dubbing/reference.py b/freevideo_engine/dubbing/reference.py new file mode 100644 index 00000000..89ea87af --- /dev/null +++ b/freevideo_engine/dubbing/reference.py @@ -0,0 +1,39 @@ +"""The voice reference of a dubbing job: an audio file, or the sound of a video, as a short mono wav.""" +import numpy as np + +from . import backends, wavio + +RATE = 24000 +MIN_SECONDS = 1.0 + + +def extract(source, target, start=0., seconds=backends.MAX_REFERENCE_SECONDS): + """Decode up to `seconds` of the first audio stream of `source` (audio or video) from `start` + into a mono 16-bit wav at `target`. Returns the seconds written.""" + import av + chunks = [] + wanted = int(seconds * RATE) + got = 0 + with av.open(str(source)) as container: + if not container.streams.audio: + raise ValueError('no_audio') + stream = container.streams.audio[0] + if start > 0: + container.seek(int(start / stream.time_base) if stream.time_base else 0, stream=stream) + resampler = av.AudioResampler(format='flt', layout='mono', rate=RATE) + for frame in container.decode(stream): + if frame.time is not None and frame.time + frame.samples / max(frame.rate, 1) <= start: + continue + for part in resampler.resample(frame): + chunk = part.to_ndarray().reshape(-1) + chunks.append(chunk) + got += len(chunk) + if got >= wanted: + break + if not chunks: + raise ValueError('no_audio') + samples = np.concatenate(chunks)[:wanted] + if len(samples) < MIN_SECONDS * RATE: + raise ValueError('reference_short') + wavio.write(target, samples, RATE) + return len(samples) / RATE diff --git a/freevideo_engine/dubbing/service.py b/freevideo_engine/dubbing/service.py new file mode 100644 index 00000000..a0ae7b23 --- /dev/null +++ b/freevideo_engine/dubbing/service.py @@ -0,0 +1,516 @@ +"""Dubbing for the editor page, the Agent and the voice change: one job at a time, one interface. + +A job takes lines (text and target seconds) and one reference voice, and writes one wav per line +into ComfyUI's input folder (`freevideo-dubbing//`), where the other tools already find media. +The model runs in its own process, so its memory leaves with it. The model is downloaded on the +first use after the user agreed (`install`). +""" +import json +import logging +import os +from pathlib import Path +import secrets +import shutil +import subprocess +import threading +import time +import uuid + +from . import backends, pack +from ..prompt_vlm.service import Worker + +OUTPUT = 'freevideo-dubbing' +MAX_LINES = 500 +MAX_TEXT = 2000 +MAX_SECONDS = 120. +MIN_SECONDS = .3 +RUN_SECONDS = 300 +LINE_SECONDS = 45 +RUN_LIMIT = 3600 +KEPT_RUNS = 8 +LINE_KEYS = ('id', 'seconds', 'target', 'samples', 'onset', 'offset', 'onset_error', 'offset_error', 'error', 'speed', + 'passes', 'stretch', 'fitted', 'warning', 'overrun', 'gain_db', 'reference', 'code') +CODES = {'invalid_request', 'unsupported_media', 'busy', 'model_missing', 'consent_required', 'unknown_job', + 'reference_text_required', 'reference_short', 'no_audio'} +KNOWN = {'disk_space', 'download_failed', 'not_published', 'unsupported', 'timeout', 'worker_exit', 'worker_cleanup_failed', + 'out_of_memory', 'model_missing', 'invalid_request', 'reference_text_required', 'reference_short'} + + +def reference_of(value, *, required): + """{file, text, start[, end]} of a job or line reference; None when absent and not required.""" + if value is None and not required: + return None + if not isinstance(value, dict) or not isinstance(value.get('file'), str) or not value['file']: + raise ValueError('invalid_request') + text = value.get('text', '') + start = value.get('start', 0) + end = value.get('end') + if (not isinstance(text, str) or len(text) > 1000 or isinstance(start, bool) + or not isinstance(start, (int, float)) or not 0 <= start <= 36000): + raise ValueError('invalid_request') + result = dict(file=value['file'], text=text.strip(), start=float(start)) + if end is not None: + if isinstance(end, bool) or not isinstance(end, (int, float)) or not start < end <= 36000: + raise ValueError('invalid_request') + result['end'] = float(end) + return result + + +def request(value): + """The job a page, the Agent or the voice change may send; anything else is refused.""" + if not isinstance(value, dict): + raise ValueError('invalid_request') + lines = value.get('lines') + if not isinstance(lines, list) or not 0 < len(lines) <= MAX_LINES: + raise ValueError('invalid_request') + clean = [] + seen = set() + for index, line in enumerate(lines): + if not isinstance(line, dict): + raise ValueError('invalid_request') + text = line.get('text') + seconds = line.get('seconds') + name = str(line.get('id', 'line%03d' % index)) + if (not isinstance(text, str) or not text.strip() or len(text) > MAX_TEXT + or isinstance(seconds, bool) or not isinstance(seconds, (int, float)) + or not MIN_SECONDS <= seconds <= MAX_SECONDS + or not name or len(name) > 64 or not all(c.isalnum() or c in '-_' for c in name) + or name in seen): + raise ValueError('invalid_request') + seen.add(name) + row = dict(id=name, text=text.strip(), seconds=float(seconds)) + onset, offset = line.get('onset'), line.get('offset') + for key, number in (('onset', onset), ('offset', offset)): + if number is None: + continue + if isinstance(number, bool) or not isinstance(number, (int, float)) or not 0 <= number <= seconds + 1e-9: + raise ValueError('invalid_request') + row[key] = float(number) + if row.get('offset', seconds) - row.get('onset', 0.) < MIN_SECONDS / 2: + raise ValueError('invalid_request') + if line.get('reference') is not None: + row['reference'] = reference_of(line['reference'], required=True) + instruction = line.get('instruction') + if instruction not in (None, ''): + if not isinstance(instruction, str) or len(instruction) > 200: + raise ValueError('invalid_request') + row['instruction'] = instruction.strip() + clean.append(row) + reference = reference_of(value.get('reference'), required=not all('reference' in row for row in clean)) + result = dict(lines=clean) + if reference is not None: + result['reference'] = reference + seed = value.get('seed') + if seed is not None: + if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed < 2**31: + raise ValueError('invalid_request') + result['seed'] = seed + name = value.get('backend') + if name is not None: + if name not in backends.names(): + raise ValueError('invalid_request') + result['backend'] = name + return result + + +class Service: + def __init__(self, installation, input_directory): + self.installation = installation + self.input_directory = input_directory + self.mutex = threading.Lock() + self.stop = threading.Event() + self.state = dict(phase='idle') + self.key = None + self.thread = None + self.process = None + self.result_expires = 0. + self.report = {} + self.keep = False + self.folder = None + self.closed = False + import atexit + atexit.register(self.close) + + def info(self, wait=True): + root, _ = self.installation() + try: + state = pack.status(root) + except (OSError, ValueError): + return dict(ready=False, bytes=0, busy=self.busy(), backends=backends.names()) + if wait and not state['ready']: + state['ready'] = pack.settle(root) + if state['ready']: + state['bytes'] = 0 + return dict(ready=state['ready'], model=pack.NAME, license=state['license'], bytes=state['bytes'], + published=state['published'], supported=state['supported'], busy=self.busy(), backends=backends.names()) + + def busy(self): + return self.thread is not None and self.thread.is_alive() + + def check(self): + if self.stop.is_set(): + raise InterruptedError('cancelled') + + def progress(self, row): + with self.mutex: + self.state.update(row) + + def start(self, action, value, *, trusted=False, keep=False): + """`trusted`: reference files are absolute paths the caller has already checked (the voice change reads + its own separated files); `keep`: a cancelled or failed run leaves the finished lines in its folder, + and the caller removes the folder.""" + if action not in ('install', 'dub'): + raise ValueError('invalid_request') + if self.busy(): + raise ValueError('busy') + root, machine = self.installation() + if action == 'install': + if not isinstance(value, dict) or value.get('accept_download') is not True: + raise ValueError('consent_required') + else: + if not pack.settle(root): + raise ValueError('model_missing') + value = request(value) + for item in [value.get('reference')] + [line.get('reference') for line in value['lines']]: + if item: + item['path'] = item['file'] if trusted and os.path.isabs(item['file']) else self.resolve(item['file']) + with self.mutex: + if self.busy(): + raise ValueError('busy') + self.key = secrets.token_urlsafe(24) + self.stop.clear() + self.result_expires = 0. + self.state = dict(phase='starting') + self.report = {} + self.keep = keep + self.folder = None + self.thread = threading.Thread(target=self.work, args=(root, machine, action, value), + name='freevideo-dubbing', daemon=True) + self.thread.start() + return dict(job=self.key) + + def resolve(self, name): + """A reference inside ComfyUI's input folder; audio, or a video whose sound is used.""" + from ..comfy_assets import media_kind, resolve_assets + resolved = resolve_assets(json.dumps([dict(file=name, role='reference')]), self.input_directory()) + path = resolved['references'][0]['path'] + if media_kind(path) not in ('audio', 'video'): + raise ValueError('unsupported_media') + return path + + def status(self, key): + if not isinstance(key, str) or not self.key or not secrets.compare_digest(key, self.key): + raise ValueError('unknown_job') + with self.mutex: + if self.result_expires and time.monotonic() > self.result_expires: + self.state.pop('lines', None) + self.state.update(phase='failed', error='expired') + row = dict(self.state) + # Do not announce completion while the runtime lease is still held. + if self.busy() and row.get('phase') in ('complete', 'failed', 'cancelled'): + row = dict(phase='finishing') + return row + + def cancel(self, key): + self.status(key) + self.stop.set() + return dict(phase='cancelling') + + def close(self): + self.closed = True + self.stop.set() + process = self.process + if process is not None and process.poll() is None: + from .. import processes + processes.stop(process, grace=5) + + @staticmethod + def release_others(): + """Give the GPU back: compile, idle image worker, resident engine process, ComfyUI models.""" + from ..background_compile import COMPILE + COMPILE.stop() + try: + from server import PromptServer + from ..image_gen.service import release_idle_worker + release_idle_worker(PromptServer.instance) + except Exception: + pass + from ..resident_process import OWNER + if OWNER.active: + raise BlockingIOError('busy') + OWNER.stop_prewarm() + from ..encoder_prewarm import IDLE + IDLE.stop() + from ..comfy_residency import release_native + from ..image_gen.service import comfy_server + try: + release_native(comfy_server()) + except Exception: # only frees memory early; the worker's own allocation still decides + logging.warning('[FreeVideo] Could not unload ComfyUI models before dubbing', exc_info=True) + + def output_folder(self, key_id): + base = Path(self.input_directory()) / OUTPUT + base.mkdir(parents=True, exist_ok=True) + # Keep the latest few runs; older ones are the page's old results. + runs = sorted((p for p in base.iterdir() if p.is_dir()), key=lambda p: p.stat().st_mtime) + for old in runs[:max(0, len(runs) - KEPT_RUNS + 1)]: + shutil.rmtree(old, ignore_errors=True) + folder = base / key_id + folder.mkdir() + return folder + + def work(self, root, machine, action, value): + from ..locking import runtime_lock + from ..monitoring import save + started = time.monotonic() + try: + if action == 'install': + with runtime_lock(Path(root) / 'dubbing-install.lock', inherit=False): + self.check() + pack.install(root, self.progress, self.check) + self.check() + self.progress(dict(phase='complete')) + else: + self.release_others() + with runtime_lock(Path(root) / 'engine.lock', inherit=False) as descriptor: + self.check() + self.dub(root, machine, descriptor, value) + except InterruptedError: + self.progress(dict(phase='cancelled')) + except BlockingIOError: + self.progress(dict(phase='failed', error='busy')) + except pack.PackError as error: + self.progress(dict(phase='failed', error=error.reason)) + except Exception as error: + reason = str(error) + self.progress(dict(phase='failed', exception=type(error).__name__, + error=reason if reason in KNOWN else 'download_failed' if action == 'install' + else 'dub_failed')) + finally: + if self.stop.is_set(): + self.progress(dict(phase='cancelled')) + self.result_expires = time.monotonic() + 600 + # Allowlisted metadata only: never the text, file names or paths. + report = {key: item for key, item in self.state.items() + if key in ('phase', 'error', 'exception', 'timings', 'worker_returncode', 'backend', + 'sample_rate', 'reference_seconds', 'loaded_seconds')} + if self.state.get('lines'): + report['lines'] = [{key: row.get(key) for key in LINE_KEYS if key not in ('id', 'reference')} + for row in self.state['lines']] + report.update(schema='freevideo.dubbing', schema_version=1, action=action, created_epoch=time.time(), + report_id=uuid.uuid4().hex, elapsed_seconds=time.monotonic() - started) + self.report = report + try: + save(Path(root) / 'dubbing-report.json', report) + except OSError: + pass + + @staticmethod + def environment(root): + from ..comfy_environment import isolated_environment + from ..comfy_bridge import source_root + from .. import processes, resident_process + env = isolated_environment(root, source_root()) + for name in ('FREEVIDEO_RUNTIME_LOCK_FD', processes.HANDLE_ENV, resident_process.ENV): + env.pop(name, None) + return env + + @staticmethod + def spawn(root, machine, env, descriptor=None): + from .. import processes + command = processes.module_command('freevideo_engine.dubbing.worker', pack.directory(root), pack.runtime_path(root) or '') + command[0] = machine['comfy_python'] + return processes.popen(command, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='replace', + pass_fds=() if descriptor is None else (descriptor,), + supervise=True, start_new_session=True) + + def dub(self, root, machine, descriptor, value): + from .. import processes, resident_process + from . import reference + import psutil + env = self.environment(root) + key_id = uuid.uuid4().hex[:12] + folder = self.output_folder(key_id) + self.folder = folder + self.progress(dict(phase='preparing')) + extracted = {} + + def extract(ref, name): + key = (ref['path'], ref['start'], ref.get('end')) + if key not in extracted: + wav = folder / name + span = ref['end'] - ref['start'] if ref.get('end') else backends.MAX_REFERENCE_SECONDS + try: + reference.extract(ref['path'], wav, start=ref['start'], seconds=min(span, backends.MAX_REFERENCE_SECONDS)) + except ValueError as error: + if str(error) not in CODES: + raise ValueError('unsupported_media') from None + extracted[key] = str(error) + return extracted[key] + except Exception: + raise ValueError('unsupported_media') from None + extracted[key] = str(wav) + return extracted[key] + job = dict(value, output_dir=str(folder)) + if value.get('reference'): + made = extract(value['reference'], 'reference.wav') + if made not in (str(folder / 'reference.wav'),): + raise ValueError(made) + job['reference'] = dict(path=made, text=value['reference']['text']) + job['lines'] = [] + for index, line in enumerate(value['lines']): + row = {key: item for key, item in line.items() if key != 'reference'} + if line.get('reference'): + made = extract(line['reference'], 'reference-%d.wav' % index) + if os.path.isfile(made): + row['reference'] = dict(path=made, text=line['reference']['text']) + elif made == 'reference_short' and job.get('reference'): + # the job's reference is used, and the line says so + row['reference_short'] = True + else: + raise ValueError(made) + job['lines'].append(row) + session = Worker(self.spawn(root, machine, env, descriptor)) + child = session.process + self.process = child + result = {} + lines = [] + write_errors = [] + + def submit(): + try: + child.stdin.write(json.dumps(job, ensure_ascii=True)) + child.stdin.close() + except Exception as error: + write_errors.append(error) + writer = threading.Thread(target=submit, daemon=True) + + def read_rows(): + import queue + while True: + try: + row = session.rows.get_nowait() + except queue.Empty: + return + phase = row.get('phase') + if phase in ('complete', 'failed', 'cancelled'): + result.update(row) + elif phase == 'line': + entry = {key: row[key] for key in LINE_KEYS if key in row} + if row.get('file'): + entry['file'] = '%s/%s/%s' % (OUTPUT, key_id, row['file']) + lines.append(entry) + self.progress(dict(phase='synthesizing', index=row.get('index'), count=len(job['lines']), + lines=list(lines))) + elif phase in ('loading', 'loaded', 'synthesizing'): + self.progress(dict((k, v) for k, v in row.items() if k in ('phase', 'index', 'count', 'seconds'))) + observed = {} + try: + writer.start() + deadline = time.monotonic() + min(RUN_LIMIT, RUN_SECONDS + LINE_SECONDS * len(job['lines'])) + while child.poll() is None: + read_rows() + self.check() + if write_errors: + raise write_errors[0] + if time.monotonic() > deadline: + raise TimeoutError('timeout') + try: + for process in psutil.Process(child.pid).children(recursive=True): + observed[process.pid] = process + except psutil.Error: + pass + self.stop.wait(.25) + session.reader.join(timeout=5) + read_rows() + self.check() + if session.reader.is_alive() or not result: + raise RuntimeError('worker_exit') + if result.get('phase') == 'failed': + raise RuntimeError(result.get('code') if result.get('code') in KNOWN else 'dub_failed') + if result.get('phase') == 'cancelled': + raise InterruptedError('cancelled') + if child.returncode: + raise RuntimeError('worker_exit') + self.progress(dict(phase='complete', lines=lines, backend=result.get('backend'), + sample_rate=result.get('sample_rate'), + reference_seconds=result.get('reference_seconds'), + worker_returncode=child.returncode)) + finally: + if child.poll() is None: + processes.stop(child, grace=15) + child.wait() + writer.join(timeout=5) + session.close() + self.process = None + for wav in folder.glob('reference*.wav'): + try: + wav.unlink() + except OSError: + pass + surviving = [] + for process in observed.values(): + try: + if process.is_running() and process.status() != 'zombie': + surviving.append(process) + except psutil.NoSuchProcess: + pass + for process in surviving: + try: + process.kill() + except psutil.NoSuchProcess: + pass + if surviving: + psutil.wait_procs(surviving, timeout=5) + self.progress(dict(error='worker_cleanup_failed')) + # Failed or cancelled runs leave nothing behind, unless the caller keeps the finished lines. + if self.state.get('phase') != 'complete' and not self.keep: + shutil.rmtree(folder, ignore_errors=True) + + +def register(): + import asyncio + from aiohttp import web + from server import PromptServer + import folder_paths + from ..comfy_bridge import installation + server = PromptServer.instance + if server is None or getattr(server, '_freevideo_dubbing', None) is not None: + return + service = Service(installation, folder_paths.get_input_directory) + server._freevideo_dubbing = service + token = secrets.token_urlsafe(24) + + @server.routes.get('/freevideo/dubbing') + async def info(request): + try: + return web.json_response(dict(await asyncio.to_thread(service.info, False), token=token)) + except (OSError, ValueError): + return web.json_response(dict(error='setup_required'), status=400) + + @server.routes.post('/freevideo/dubbing/{action}') + async def action(request): + if not secrets.compare_digest(request.headers.get('X-FreeVideo-Dubbing', ''), token): + return web.json_response(dict(error='reload'), status=403) + try: + name = request.match_info['action'] + if request.content_length and request.content_length > 512 * 1024: + raise ValueError('invalid_request') + value = await request.json() + if not isinstance(value, dict): + raise ValueError('invalid_request') + if name == 'status': + result = service.status(value.get('job')) + elif name == 'cancel': + result = service.cancel(value.get('job')) + else: + running, pending = server.prompt_queue.get_current_queue() + if running or pending: + raise ValueError('busy') + if name == 'dub': + await asyncio.to_thread(pack.settle, service.installation()[0]) + result = service.start(name, value) + return web.json_response(result) + except (ValueError, OSError) as error: + return web.json_response(dict(error=str(error) if str(error) in CODES else 'invalid_media'), status=400) diff --git a/freevideo_engine/dubbing/wavio.py b/freevideo_engine/dubbing/wavio.py new file mode 100644 index 00000000..9112b240 --- /dev/null +++ b/freevideo_engine/dubbing/wavio.py @@ -0,0 +1,27 @@ +"""16-bit PCM wav files, read and written without an audio library.""" +import wave + +import numpy as np + + +def read(path): + """Float32 samples shaped (frames, channels) and the sample rate of a 16-bit PCM wav.""" + with wave.open(str(path), 'rb') as file: + if file.getsampwidth() != 2: + raise ValueError('unsupported_wav') + channels, rate = file.getnchannels(), file.getframerate() + raw = file.readframes(file.getnframes()) + data = np.frombuffer(raw, dtype='= LEVEL_TARGET: + return data, 1. + if rate: + rms = voiced_level(data, rate) or rms + gain = max(1., min(LEVEL_TARGET / rms, LEVEL_PEAK / peak, LEVEL_GAIN)) + if gain == 1.: + return data, 1. + return data * np.float32(gain), gain + + +MATCH_RANGE = (0.1, 10.) + + +def voiced_level(data, rate): + """RMS of the voiced part of `data` (None when there is none).""" + import numpy as np + span = align.voiced_span(data, rate) + if span is None: + return None + part = np.asarray(data[span[0]:span[1]], dtype=np.float64) + return float(np.sqrt(np.mean(np.square(part)))) if len(part) else None + + +def matched(samples, rate, onset, offset, level): + """`samples` scaled so that the voiced part (from `onset` to `offset` seconds) has RMS `level`: the line + comes back as loud as the original line it replaces, whatever level the model was given. Returns + (samples, gain); the gain stays inside MATCH_RANGE and never clips.""" + import numpy as np + current = voiced_level(samples[int(onset * rate):int(offset * rate)], rate) + peak = float(np.max(np.abs(samples))) if len(samples) else 0. + if not level or not current or peak <= 0: + return samples, 1. + gain = min(max(level / current, MATCH_RANGE[0]), MATCH_RANGE[1], LEVEL_PEAK / peak) + if abs(gain - 1.) < 0.01: + return samples, 1. + return samples * np.float32(gain), gain + + +CLAUSE_END = '。！？；，、.!?;,\n' + + +def cut_text(text, share): + """The longest start of `text` that is no more than `share` of it and ends where a clause or a word + ends; returns (start of the text, its share of the whole text).""" + limit = int(len(text) * share) + if limit <= 0: + return '', 0. + stops = [i for i in range(limit, 0, -1) if text[i - 1] in CLAUSE_END or text[i - 1] == ' ' or text[i:i + 1] == ' '] + end = next((i for i in stops if i >= limit // 2), limit) + kept = text[:end].strip() + return kept, len(kept) / len(text) + + +def usable_reference(path, folder, name='reference-prepared.wav', text='', texts=None): + """Return a mono wav no longer than the models accept and not too quiet (a changed copy if either + applies), and its seconds. A reference that is too long is cut where its text is cut (the model reads the + text it is given, so a text longer than the audio is spoken into the line): the cut text goes to + `texts[name]`.""" + from . import wavio + import numpy as np + data, rate = wavio.read(path) + seconds = len(data) / rate + if seconds < 1.0: + raise ValueError('reference_short') + changed = False + if texts is not None: + texts[name] = text + if seconds > backends.MAX_REFERENCE_SECONDS: + end = int(backends.MAX_REFERENCE_SECONDS * rate) + if text: + kept, share = cut_text(text, backends.MAX_REFERENCE_SECONDS / seconds) + if kept: + text, end = kept, int(len(data) * share) + # the cut falls on the quietest 10 ms of the last half second + frame = max(1, rate // 100) + first = max(0, end - rate // 2) // frame + count = (end - first * frame) // frame + if count > 1: + rms = np.sqrt(np.square(np.asarray(data[first * frame:(first + count) * frame], dtype=np.float64)) + .reshape(count, frame).mean(axis=1)) + end = (first + int(np.argmin(rms))) * frame + frame // 2 + if texts is not None: + texts[name] = text + data = data[:end] + seconds = len(data) / rate + changed = True + data, gain = leveled(data, rate) + if gain != 1.: + changed = True + if not changed: + return path, seconds + target = os.path.join(folder, name) + wavio.write(target, data, rate) + return target, seconds + + +def validate(request): + lines = request.get('lines') + if not isinstance(lines, list) or not lines or len(lines) > 500: + raise ValueError('invalid_request') + for line in lines: + if not isinstance(line, dict) or not str(line.get('text') or '').strip(): + raise ValueError('invalid_request') + if not float(line.get('seconds') or 0) > 0: + raise ValueError('invalid_request') + reference = request.get('reference') or {} + own = [(line.get('reference') or {}) for line in lines] + if any(not isinstance(item, dict) for item in own): + raise ValueError('invalid_request') + for item in own: + if item and not os.path.isfile(str(item.get('path') or '')): + raise ValueError('invalid_request') + if (reference or not all(own)) and not os.path.isfile(str(reference.get('path') or '')): + raise ValueError('invalid_request') + if not os.path.isdir(str(request.get('output_dir') or '')): + raise ValueError('invalid_request') + return lines, reference + + +def run(request, model_dir, device='cuda'): + from . import wavio + from ..prompt_vlm.worker import Cancelled + lines, reference = validate(request) + folder = request['output_dir'] + name = request.get('backend') or backends.DEFAULT + seed = request.get('seed') + # References are prepared before the model loads: a bad one is refused in seconds, not after the load. + prepared = {} + levels = {} + cut = {} + + def prepare(source, key): + if key not in prepared: + name = 'reference-%s.wav' % key + prepared[key] = usable_reference(source['path'], folder, name, str(source.get('text') or '').strip(), cut) + data, rate = wavio.read(source['path']) + levels[key] = voiced_level(data, rate) + return prepared[key] + job_ref = None + if reference: + job_ref = prepare(reference, 'job') + (cut['reference-job.wav'],) + used = [] + match = {} # per line with its own reference: the voiced RMS the line is to come back with + for index, line in enumerate(lines): + own = line.get('reference') + if own: + try: + used.append(('line',) + prepare(own, 'line%d' % index) + (cut['reference-line%d.wav' % index], None)) + match[index] = levels.get('line%d' % index) + continue + except ValueError as error: + if str(error) != 'reference_short' or job_ref is None: + raise + used.append(('job',) + job_ref + ('reference_short',)) + continue + used.append(('job',) + job_ref + ('reference_short' if line.get('reference_short') else None,)) + emit(phase='loading') + started = time.monotonic() + backend = backends.create(name, model_dir, device=device) + emit(phase='loaded', seconds=round(time.monotonic() - started, 2)) + try: + if backend.needs_reference_text and any(not item[3] for item in used): + raise ValueError('reference_text_required') + results = [] + for index, line in enumerate(lines): + emit(phase='synthesizing', index=index, count=len(lines)) + source, ref_path, ref_seconds, ref_text, note = used[index] + if seed is not None: + from .cosyvoice3.utils.common import set_all_random_seed + set_all_random_seed(int(seed) + index) + instruction = (line.get('instruction') or '').strip() or None + text = str(line['text']).strip() + target = float(line['seconds']) + row = dict(id=line.get('id'), target=target, reference=source) + try: + def synth(speed, text=text, instruction=instruction, ref_path=ref_path, ref_text=ref_text): + return backend.synth(text, ref_path, ref_text, speed, instruction) + + fitted = align.fit(synth, target, onset=float(line.get('onset') or 0.), + offset=None if line.get('offset') is None else float(line['offset'])) + except Cancelled: + raise + except Exception as error: + traceback.print_exc(file=sys.stderr) + row.update(failure(error)) + results.append(row) + emit(phase='line', index=index, **row) + continue + samples, gain = fitted['samples'], 1. + if match.get(index): + samples, gain = matched(samples, fitted['rate'], fitted['onset'], fitted['offset'], match[index]) + path = os.path.join(folder, '%s.wav' % (line.get('id') or 'line%03d' % index)) + wavio.write(path, samples, fitted['rate']) + row.update(file=os.path.basename(path), seconds=round(fitted['seconds'], 4), samples=len(fitted['samples']), + onset=round(fitted['onset'], 3), offset=round(fitted['offset'], 3), + onset_error=round(fitted['onset_error'], 3), offset_error=round(fitted['offset_error'], 3), + error=round(fitted['error'], 3), speed=round(fitted['speed'], 3), passes=fitted['passes'], + stretch=round(fitted['stretch'], 3), fitted=bool(fitted['fitted']), + warning=fitted['warning'] or (note if note == 'reference_short' else None), + overrun=round(fitted['overrun'], 3), gain_db=round(20 * math.log10(gain), 1)) + results.append(row) + emit(phase='line', index=index, **row) + finally: + backend.close() + if all('code' in row for row in results): + raise _LineFailure(results[0]) + return dict(lines=results, reference_seconds=round(used[0][2], 2), backend=name, sample_rate=backend.sample_rate) + + +class _LineFailure(Exception): + """Every line failed: the job fails with the first line's code.""" + def __init__(self, row): + super().__init__(row['code']) + self.row = row + + +CODES = ('invalid_request', 'reference_text_required', 'reference_short') + + +def failure(error): + if isinstance(error, _LineFailure): + return dict(code=error.row['code'], **({'error': error.row['error']} if 'error' in error.row else {})) + text = str(error) + if isinstance(error, ValueError) and text in CODES: + return dict(code=text) + if isinstance(error, FileNotFoundError): + return dict(code='model_missing') + low = text.lower() + if isinstance(error, MemoryError) or 'out of memory' in low: + return dict(code='out_of_memory') + return dict(code='failed', error=type(error).__name__) + + +def main(): + from ..prompt_vlm.worker import Cancelled, stop_on_signal + stop_on_signal() + try: + request = json.loads(sys.stdin.buffer.read(1024 * 1024)) + if not isinstance(request, dict): + raise ValueError('invalid_request') + if len(sys.argv) > 2 and sys.argv[2]: + # -I ignores PYTHONPATH: the downloaded runtime (onnxruntime) is handed over as an argument. + from .pack import activate + activate(sys.argv[2]) + result = run(request, sys.argv[1], device=request.get('device') or 'cuda') + emit(phase='complete', **result) + return 0 + except Cancelled: + emit(phase='cancelled') + return 0 + except Exception as error: + traceback.print_exc(file=sys.stderr) + emit(phase='failed', **failure(error)) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index b91e6f4d..f4705690 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -843,6 +843,9 @@ def interrupted(signum, frame): from .continuation import first_frame_prompt prompt_text = first_frame_prompt(prompt_text) prompt_text = without_music(prompt_text) + # The model has no negative prompt: sound the user excluded is worded as the sounds the scene has. + from .sound_prompt import positive_sound + prompt_text = positive_sound(prompt_text) encoding = {'prompt': prompt_text, 'encoder': args.encoder, 'comfy_root': str(args.comfy_root or comfy_root()), 'model_paths': str(args.model_paths.resolve()) if args.model_paths else None, diff --git a/freevideo_engine/licenses/CosyVoice-Apache-2.0.txt b/freevideo_engine/licenses/CosyVoice-Apache-2.0.txt new file mode 100644 index 00000000..31e1c88f --- /dev/null +++ b/freevideo_engine/licenses/CosyVoice-Apache-2.0.txt @@ -0,0 +1,206 @@ +Copyright (c) 2024 Alibaba Inc (FunAudioLLM). CosyVoice, Fun-CosyVoice3-0.5B-2512 and the code in +freevideo_engine/dubbing/cosyvoice3 adapted from https://github.com/FunAudioLLM/CosyVoice are licensed +under the Apache License 2.0 reproduced below. FreeVideo reduced the code to the zero-shot and instruct2 +inputs and changed its file, audio and optional-dependency handling; each source file keeps its header. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 FreeToken Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/freevideo_engine/licenses/CosyVoice-NOTICE.txt b/freevideo_engine/licenses/CosyVoice-NOTICE.txt new file mode 100644 index 00000000..482132a6 --- /dev/null +++ b/freevideo_engine/licenses/CosyVoice-NOTICE.txt @@ -0,0 +1,29 @@ +Notices for code adapted into freevideo_engine/dubbing/cosyvoice3 (dubbing, optional download of the model). + +CosyVoice / Fun-CosyVoice3-0.5B-2512, Copyright (c) 2024 Alibaba Inc (FunAudioLLM): Apache License 2.0, +see CosyVoice-Apache-2.0.txt. The model weights are downloaded on request and are not part of the installer. + +The following parts keep the MIT licence of their authors: +- utils/mel.py mel_spectrogram, from Matcha-TTS (matcha/utils/audio.py), Copyright (c) 2023 Shivam Mehta, + derived from HiFi-GAN, Copyright (c) 2020 Jungil Kong. +- flow/flow_matching.py attributes inlined from Matcha-TTS, Copyright (c) 2023 Shivam Mehta. +- flow/DiT/modules.py rotary embedding, re-implemented from x_transformers and rotary-embedding-torch, + Copyright (c) 2020 Phil Wang. +- transformer/activation.py Snake activation, from https://github.com/EdwardDixon/snake. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index 9cc4c84e..2c7255bc 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -534,6 +534,7 @@ def encode_latents(normalized, value, base, canvas, *, resident=None, device='cu empty_cache() if any(row.get('audio') for row in normalized): print(json.dumps({'event': 'media_encode_phase', 'phase': 'Encoding reference audio'}), flush=True) + from .vae_weights import load_audio_vae vae = (resident.input_audio_encoder(base, canvas, device=device) if resident is not None and torch.device(device).type != 'mps' else load_audio_vae(base, device=device)[0]) diff --git a/freevideo_engine/pool_resources.py b/freevideo_engine/pool_resources.py index c90eff5c..59c229c4 100644 --- a/freevideo_engine/pool_resources.py +++ b/freevideo_engine/pool_resources.py @@ -231,4 +231,35 @@ def _estimate(self, prompt): # Reserve the full device capacity for future WDDM allocation, following # runtime.gpu_commit_reserve, with no credit for already reserved VRAM. gpu_commit = max(device.total_bytes for device in self.devices) if windows() else 0 + if len(self.devices) >= 2: + # The measured private peak is for the reference request; larger canvases, + # attached media and continuations add what the workspace estimate adds. + reference = geometry(1344, 768, frames=243) + reference_workspace = (HOST_WEIGHT_HEADROOM + ENCODER_HOST_BYTES + residual_host_headroom(reference) + + decoder_workspace(reference) + 3 * (1344 * 768 * 243 * 3) + + UPSCALER_HOST_BYTES + adaln_extra_bytes(reference)) + return multi_gpu_envelope(self.devices, gpu_commit, max(0, workspace - reference_workspace)) return Envelope(host, host - pinned + gpu_commit, pinned) + + +GIB = 1 << 30 +# Two or more GPUs: measured per-task host use of one resident worker. Private +# anonymous memory peaked at 6.4 GiB on the largest admitted request; pinned +# weights follow each card's placement, 16.9 GiB on 16 GB-class cards and +# 8.9 GiB on 24 GB-class cards, and cards of 32 GB and above are bounded by the +# 24 GB figure. Mapped weight files add 0.3-0.5 GiB of page cache once per machine. +PRIVATE_TASK_BYTES = int(7.5 * GIB) +MAPPED_ONCE_BYTES = GIB // 2 +PINNED_SMALL_CARD_BYTES = 17 * GIB +PINNED_LARGE_CARD_BYTES = 9 * GIB +SMALL_CARD_LIMIT_BYTES = 20 * GIB +multi_gpu_calls = 0 + + +def multi_gpu_envelope(devices, gpu_commit, extra_private=0): + global multi_gpu_calls + multi_gpu_calls += 1 + smallest = min(device.total_bytes for device in devices) + pinned = PINNED_SMALL_CARD_BYTES if smallest < SMALL_CARD_LIMIT_BYTES else PINNED_LARGE_CARD_BYTES + host = PRIVATE_TASK_BYTES + extra_private + MAPPED_ONCE_BYTES + pinned + return Envelope(host, host - pinned + gpu_commit, pinned) diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index 64627f72..4dc6c4d1 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,41 +1,68 @@ { "schema": 1, - "product_version": "0.4.31", + "product_version": "0.4.32", "en": { - "summary": "The Edit view can now record audio against the picture, a sound clip can be separated into dialogue, music and ambience tracks, and timed lines and cuts written in a description are placed at those times in the generated video.", + "summary": "A picture can be generated for a span of imported sound, and the top bar has a permanent Tutorial entry.", "changes": [ - "Recording: the Edit view can now record audio against the picture. The recording is placed on a sound track, aligned with the picture.", - "Separate tracks: a sound clip can be split into dialogue, music and ambience tracks, each adjusted and exported on its own. Separated music can be replaced from the generation bar with \"Replace music\": new music is made at the original length and lowered where people speak.", - "Timed lines and cuts: write lines such as \"0:02 Jay: ...\" or \"0:05 Cut: ...\" in the description, and the generated video places dialogue and shot changes at those times. A time beyond the video length or an empty cut is flagged before generation.", - "Preview first: the editor's generation bar can make a preview first. A selected preview part can run its second pass with \"Run second pass\" and is replaced in place at full quality. \"Reuse settings\" also fills in the original seed.", - "Ripple delete: deleting a clip in the editor also closes the gap it leaves.", - "Recent videos and Creations fold the videos of one task into a group. Click the cover to open it.", - "Progress estimate: the estimated total time of a request with a long reference video is no longer far above the real time. Speed and results are unchanged.", - "Start-up: the launcher now opens when the bridge folder is redirected. A failed model upgrade keeps its cause in the diagnostic report and points to \"Try again\" or \"Report a problem\". The launch progress names the GPU detection, acceleration test and workflow-node connection steps, a slow full log export shows how many files have been packed, and a failed start in the terminal points to the diagnose command.", - "Install: on some Linux systems, setup no longer stops while building GPU acceleration.", - "Mac: a fault while encoding a reference video is corrected or reported within seconds instead of at the end of encoding. When a Light or Medium text-to-video generation without references or continuation is interrupted during the first pass, the automatic retry continues from the last completed step instead of starting the pass over.", - "Speed: on 6 GB-class cards on Windows, each sampling step of a High generation with references is faster.", - "Audio cards: \"Use as is\" now stays clickable when Preview first is on or a video is being continued. Choosing it turns the preview off with a one-line note, and a continuation says why it cannot be used and what to remove." + "Sound-led video: drag a span over imported sound in the Edit view, describe the picture, and generate a video for that span while the original sound is kept.", + "Tutorial: the top bar has a permanent Tutorial entry. Three new feature examples cover making a first video, characters and the Edit view. These and some tutorial chapters have \"Take me there\", which highlights the matching control.", + "Audio descriptions: when a description asks to leave out music or other sounds, that request is removed before generation and, where needed, a sentence is added saying the sound comes from the action and the surroundings, so unwanted background music appears less often. Descriptions that ask for music or silence, and quoted lines, are unchanged.", + "Two GPUs: a machine with two GPUs now decides whether tasks can run together from the memory each task actually uses. On Linux, two 16 GB cards with 64 GB of RAM can run two tasks at once. Single-GPU machines are unaffected.", + "Editor: deleting an edit no longer keeps failing after a brief loss of contact with the background service.", + "Editor import: a video that is cut off or damaged part-way now keeps the part that can be read instead of failing the whole import.", + "Reference audio: generating with reference audio works again on Mac, and no longer fails on other systems when the model has to be loaded first." ] }, "zh": { - "summary": "剪辑视图可以对着画面录音；一段声音可以分成人声、音乐和环境声三条轨道；在描述里按时间写的台词和切镜，生成的视频会按写定的时间安排。", + "summary": "可以为导入声音的一段生成画面；顶栏有常驻的教程入口。", "changes": [ - "录音：在剪辑视图里对着画面录一段声音，录好后放到声音轨上，与画面对齐。", - "分离音轨：把一段声音分成人声、音乐和环境声三条轨道，每条都可单独调整和导出。分离出的音乐可在生成条点“替换配乐”，按原长度重新配乐，并在说话处压低。", - "按时间写台词和切镜：在描述里写“0:02 阿杰：……”或“0:05 切：……”，生成的视频会按写定的时间安排台词和镜头切换；时间超出视频长度或“切：”后面没有内容时，生成前给出提示。", - "先出预览：剪辑视图生成条可选“先出预览”；选中预览片段可点“继续二采”，在原位置换成完整画质。点“复用设置”时，原来的种子也一并填入。", - "波纹删除：在剪辑视图删除片段时，同时合上它留下的空隙。", - "最近作品和作品库将同一任务的多条视频折叠成一组，点封面展开。", - "进度估计：带较长参考视频的请求，估计的总时间不再远高于实际用时。速度和结果不变。", - "启动：启动器在桥接目录被重定向时也能打开。模型升级失败时，诊断报告保留失败原因，并提示可点“重试”或“提交问题”。启动进度会写出检测显卡、测试加速和连接工作流节点这几步；导出完整日志较慢时显示已打包的文件数；在终端里启动失败时，会提示运行诊断命令。", - "安装：部分 Linux 系统上，安装不再停在构建 GPU 加速这一步。", - "Mac：参考视频编码出错时，几秒内即修正或报告，不再等到编码结束。轻量或标准档、不带参考、不续接的文生视频在第一遍被中断后，自动重试会从最后完成的步骤继续，不再从头开始这一遍。", - "速度：Windows 上的 6 GB 级显卡，带参考的精细档生成每一步采样更快。", - "音频卡：“先出预览”开启或接着生成时，“原样使用”不再变灰。选择它会关闭预览并用一行说明；接着生成时说明原因和需要移除的内容。" + "声音带动画面：在剪辑视图里，在导入的声音上拖出一段，写上画面描述，为这一段生成视频，原声保留。", + "教程：顶栏有常驻的“教程”入口。功能示例新增三张：做出第一条视频、角色、剪辑视图。这三张和部分教程章节有“带我去”，点后高亮对应的位置。", + "声音描述：写了不要音乐或某种声音时，生成前会去掉这类说法，必要时补一句说明声音来自动作和环境，减少无意带出的背景音乐。要求音乐或静音的描述、引号里的台词不改。", + "双显卡：有两张显卡的机器按每个任务的实际内存用量判断能否同时运行。在 Linux 上，两张 16 GB 显卡配 64 GB 内存可以同时运行两个任务。单卡机器不受影响。", + "剪辑：与后台的通信短暂中断后，删除剪辑不再连续失败。", + "剪辑导入：中途被截断或损坏的视频，现在会保留可读取的部分，不再整个导入失败。", + "参考音频：使用参考音频生成在 Mac 上恢复可用；其他系统需要先载入模型时也不再失败。" ] }, "history": [ + { + "product_version": "0.4.31", + "en": { + "summary": "The Edit view can now record audio against the picture, a sound clip can be separated into dialogue, music and ambience tracks, and timed lines and cuts written in a description are placed at those times in the generated video.", + "changes": [ + "Recording: the Edit view can now record audio against the picture. The recording is placed on a sound track, aligned with the picture.", + "Separate tracks: a sound clip can be split into dialogue, music and ambience tracks, each adjusted and exported on its own. Separated music can be replaced from the generation bar with \"Replace music\": new music is made at the original length and lowered where people speak.", + "Timed lines and cuts: write lines such as \"0:02 Jay: ...\" or \"0:05 Cut: ...\" in the description, and the generated video places dialogue and shot changes at those times. A time beyond the video length or an empty cut is flagged before generation.", + "Preview first: the editor's generation bar can make a preview first. A selected preview part can run its second pass with \"Run second pass\" and is replaced in place at full quality. \"Reuse settings\" also fills in the original seed.", + "Ripple delete: deleting a clip in the editor also closes the gap it leaves.", + "Recent videos and Creations fold the videos of one task into a group. Click the cover to open it.", + "Progress estimate: the estimated total time of a request with a long reference video is no longer far above the real time. Speed and results are unchanged.", + "Start-up: the launcher now opens when the bridge folder is redirected. A failed model upgrade keeps its cause in the diagnostic report and points to \"Try again\" or \"Report a problem\". The launch progress names the GPU detection, acceleration test and workflow-node connection steps, a slow full log export shows how many files have been packed, and a failed start in the terminal points to the diagnose command.", + "Install: on some Linux systems, setup no longer stops while building GPU acceleration.", + "Mac: a fault while encoding a reference video is corrected or reported within seconds instead of at the end of encoding. When a Light or Medium text-to-video generation without references or continuation is interrupted during the first pass, the automatic retry continues from the last completed step instead of starting the pass over.", + "Speed: on 6 GB-class cards on Windows, each sampling step of a High generation with references is faster.", + "Audio cards: \"Use as is\" now stays clickable when Preview first is on or a video is being continued. Choosing it turns the preview off with a one-line note, and a continuation says why it cannot be used and what to remove." + ] + }, + "zh": { + "summary": "剪辑视图可以对着画面录音；一段声音可以分成人声、音乐和环境声三条轨道；在描述里按时间写的台词和切镜，生成的视频会按写定的时间安排。", + "changes": [ + "录音：在剪辑视图里对着画面录一段声音，录好后放到声音轨上，与画面对齐。", + "分离音轨：把一段声音分成人声、音乐和环境声三条轨道，每条都可单独调整和导出。分离出的音乐可在生成条点“替换配乐”，按原长度重新配乐，并在说话处压低。", + "按时间写台词和切镜：在描述里写“0:02 阿杰：……”或“0:05 切：……”，生成的视频会按写定的时间安排台词和镜头切换；时间超出视频长度或“切：”后面没有内容时，生成前给出提示。", + "先出预览：剪辑视图生成条可选“先出预览”；选中预览片段可点“继续二采”，在原位置换成完整画质。点“复用设置”时，原来的种子也一并填入。", + "波纹删除：在剪辑视图删除片段时，同时合上它留下的空隙。", + "最近作品和作品库将同一任务的多条视频折叠成一组，点封面展开。", + "进度估计：带较长参考视频的请求，估计的总时间不再远高于实际用时。速度和结果不变。", + "启动：启动器在桥接目录被重定向时也能打开。模型升级失败时，诊断报告保留失败原因，并提示可点“重试”或“提交问题”。启动进度会写出检测显卡、测试加速和连接工作流节点这几步；导出完整日志较慢时显示已打包的文件数；在终端里启动失败时，会提示运行诊断命令。", + "安装：部分 Linux 系统上，安装不再停在构建 GPU 加速这一步。", + "Mac：参考视频编码出错时，几秒内即修正或报告，不再等到编码结束。轻量或标准档、不带参考、不续接的文生视频在第一遍被中断后，自动重试会从最后完成的步骤继续，不再从头开始这一遍。", + "速度：Windows 上的 6 GB 级显卡，带参考的精细档生成每一步采样更快。", + "音频卡：“先出预览”开启或接着生成时，“原样使用”不再变灰。选择它会关闭预览并用一行说明；接着生成时说明原因和需要移除的内容。" + ] + } + }, { "product_version": "0.4.30", "en": { diff --git a/freevideo_engine/sound_prompt.py b/freevideo_engine/sound_prompt.py new file mode 100644 index 00000000..8fb8c319 --- /dev/null +++ b/freevideo_engine/sound_prompt.py @@ -0,0 +1,78 @@ +"""Positive sound wording for the text encoder. + +The model has no negative prompt, so "no music" only draws attention to music. Sound requests that +exclude something are replaced by the sounds the scene actually has. A prompt that asks for music +is returned unchanged, and quoted or tagged speech is never edited. A prompt with spoken lines, quoted or described, gets no added +sentence either: it only lost the excluded sound words, because the sentence made the spoken lines less accurate. +""" +import re + +from .prompt_vlm.rules import QUOTED, QUOTES + +MUSIC = (r'(?:background\s+music|music|soundtrack|BGM|cinematic\s+(?:orchestral\s+)?score|orchestral\s+score|song|' + r'背景音乐|音乐|配乐|歌曲|歌声)') +PERFORMED = (r'(?:\b(?:sings?|singing|singers?|bands?|DJ|whistl(?:e|es|ed|ing)|choir)\b|' + r'\b(?:plays?|playing|played)\s+(?:an?\s+|the\s+)?(?:\w+\s+)?(?:guitar|piano|violin|cello|drums?|saxophone|trumpet|flute|keyboard|harp|accordion|ukulele|bass)\b|' + r'唱歌|歌手|乐队|演奏|弹(?:吉他|钢琴)|拉(?:小提琴|大提琴)|吹(?:口哨|笛子|萨克斯)|口哨|打鼓|合唱)') +ANY_MUSIC = r'(?:' + MUSIC + '|' + PERFORMED + ')' +NEGATION = (r'(?:\b(?:no|without|avoid|exclude)\b|\bdo\s+not\s+(?:add|include|use|play)\b|' + r'请?(?:不要|别|不用|不需要|无需|不加|没有|去掉|去除|禁止|无)(?:任何|有|放|加|使用|出现)?)') +SOUND_WORD = (r'(?:background\s+music|music|soundtrack|BGM|subtitles?|captions?|narration|voiceover|singing|audio|sound|' + r'背景音乐|音乐|配乐|歌曲|歌声|字幕|旁白|解说|声音|音效)') +# An exclusion is cut out only when it ends the clause; "no music at all" or "没有音乐的派对" would be left broken. +NEGATED = re.compile(r'(?:(?:[,，]\s*)|\b(?:with|and|but|plus)\s+)?' + NEGATION + r'\s*(?:any\s+)?' + SOUND_WORD + + r'(?:\s+(?:or|and)\s+(?:music|subtitles?|captions?|narration|voiceover)|[和或及、]\s*(?:音乐|背景音乐|字幕|旁白|解说|音效))*' + r'(?:\s+of\s+[^,.;，。]*)?(?=\s*(?:[,，;；.。!?！？]|$))', re.I) +SILENT = re.compile(r'\bsilent\b|\bmute\b|\bno\s+(?:sound|audio)\b(?!\s+of)|无声|静音', re.I) +TAGGED = re.compile(r'<(d|en|zh|ja|ko|cn)>.*?', re.S) +SOUNDSCAPE = re.compile(r'(?m)^[ \t]*overall_soundscape[ \t]*:') +CLAUSE_EN = 'The sounds in the scene are the natural sounds of what is happening and its surroundings.' +CLAUSE_ZH = '画面里的声音是正在发生的动作和周围环境自然发出的声音。' +SPEECH = re.compile(r'\b(?:says?|said|speaks?|speaking|talks?|talking|whispers?|shouts?|asks?|replies|tells?|announces?)\b|' + r'说|讲|喊|问|答|念|朗读|台词|对白', re.I) +CHINESE = re.compile(r'[一-鿿]') + + +def _protect(prompt): + """Prompt with quoted and tagged speech swapped for placeholders, and the saved pieces.""" + kept = [] + + def save(match): + kept.append(match.group()) + return '\x00%d\x00' % (len(kept) - 1) + + return QUOTED.sub(save, TAGGED.sub(save, prompt)), kept + + +def _restore(text, kept): + return re.sub('\x00(\\d+)\x00', lambda m: kept[int(m.group(1))], text) + + +def wants_music(prompt): + """True when music is requested. Negated mentions and the visual 'music video' style are not requests.""" + body, _ = _protect(prompt) + excluded = any(re.search(MUSIC, m.group(), re.I) for m in NEGATED.finditer(body)) + body = NEGATED.sub('', body) + body = re.sub(r'\bmusic\s+video\b|\bMV\b|音乐视频', '', body, flags=re.I) + # Music that was explicitly excluded is not requested back by someone who sings or plays in the scene. + return bool(re.search(MUSIC if excluded else ANY_MUSIC, body, re.I)) + + +def positive_sound(prompt): + if not prompt.strip() or SOUNDSCAPE.search(prompt) or wants_music(prompt): + return prompt + body, kept = _protect(prompt.strip()) + if SILENT.search(body): + return prompt + chinese = bool(CHINESE.search(body)) + body = re.sub(r'\bmusic\s+video\b', '\x01', body, flags=re.I) + body = NEGATED.sub('', body) + body = re.sub(r'\s+([,.;!?])', r'\1', body) + body = re.sub(r'[,，;；]\s*(?=[,，;；.。!?！？]|$)', '', body) + body = re.sub(r'([.。!?！？])(?:\s*[.。])+', r'\1', body).strip(' ,，;；').lstrip('.。 ').replace('\x01', 'music video') + clause = CLAUSE_ZH if chinese else CLAUSE_EN + if kept or SPEECH.search(body) or clause in _restore(body, kept): + return _restore(body, kept) + if body and _restore(body, kept)[-1] not in '.。!?！？' + ''.join(QUOTES.values()): + body += '。' if chinese else '.' + return (_restore(body, kept) + ' ' + clause).strip() diff --git a/freevideo_engine/stems/separate.py b/freevideo_engine/stems/separate.py index 2059ab9f..4a5d788c 100644 --- a/freevideo_engine/stems/separate.py +++ b/freevideo_engine/stems/separate.py @@ -17,6 +17,11 @@ CHUNK_SECONDS = 6.0 HOP_SECONDS = 3.0 BATCH = 4 +# The network leaves speech in the music part when the whole track is quiet (an old recording at -37 dBFS comes +# back with an empty voice part). A quiet track is lifted to this level for the model; the parts are scaled back. +QUIET_RMS = .04 +TARGET_RMS = .08 +MAX_LIFT = 12. MIX_NAME = 'mix.wav' # Model output name -> track name. The order is the order of the sub-tracks in the editor. @@ -53,6 +58,14 @@ def run(chunks): return run +def lift_for_model(audio): + """The gain that brings a quiet track (float [channels, samples]) to the level the model separates well; 1 otherwise.""" + level = float(np.sqrt(np.mean(np.square(audio, dtype=np.float64)))) if audio.size else 0. + if level <= 1e-5 or level >= QUIET_RMS: + return 1. + return min(TARGET_RMS / level, MAX_LIFT) + + def separate_array(function, audio, *, chunk=None, hop=None, batch=BATCH, progress=lambda fraction: None): """audio: float tensor [channels, samples]; function maps chunks [n, 1, chunk] to {stem: [n, 1, chunk]}. @@ -111,11 +124,12 @@ def read_audio(path): return audio, audio.shape[1] / SAMPLE_RATE -def write_wav(path, audio): - """Atomically write float [2, samples] as 16-bit PCM at 44.1 kHz.""" +def write_wav(path, audio, *, exact=False): + """Atomically write float [2, samples] as 16-bit PCM at 44.1 kHz. `exact` scales by 32768, the divisor the readers use, + so samples read from a 16-bit file are written back unchanged (the default scales by 32767).""" path = Path(path) audio = np.asarray(audio) - pcm = np.rint(np.clip(audio.T, -1, 1) * 32767).astype(' 2 else (crossfade, crossfade) + voice = np.asarray(voice, dtype=np.float32) + if voice.ndim == 1: + voice = np.repeat(voice[:, None], result.shape[1], axis=1) + elif voice.shape[1] != result.shape[1]: + voice = np.repeat(voice[:, :1], result.shape[1], axis=1) + start = max(0, int(start)) + end = min(len(result), start + len(voice)) + if end <= start: + continue + voice = voice[:end - start] + head = min(int(round(fades[0] * rate)), (end - start) // 2) + tail = min(int(round(fades[1] * rate)), (end - start) // 2) + ramp = np.ones(end - start, dtype=np.float32) + if head: + ramp[:head] = np.linspace(0., 1., head, endpoint=False, dtype=np.float32) + if tail: + ramp[-tail:] = np.linspace(0., 1., tail, endpoint=False, dtype=np.float32)[::-1] + result[start:end] = result[start:end] * (1. - ramp[:, None]) + voice * ramp[:, None] + return result[:, 0] if flat else result diff --git a/freevideo_engine/voice/sentences.py b/freevideo_engine/voice/sentences.py new file mode 100644 index 00000000..aaf64aad --- /dev/null +++ b/freevideo_engine/voice/sentences.py @@ -0,0 +1,125 @@ +"""Sentences of a separated voice: where each line starts and ends, and what it says. + +The recogniser's word times are only a few tens of milliseconds sharp, and its word ends are capped, so they cannot +mark a pause. The boundaries come from the separated voice itself: stretches of speech energy, joined across gaps +shorter than `GAP`. The recogniser gives the words; a word belongs to the stretch that holds its middle. +""" +import numpy as np + +GAP = .25 # a silence at least this long ends a line (the pauses a replacement must keep) +LONGEST = 29. # longest line, seconds (the longest reference the voice model takes) +SHORTEST = .3 # a stretch shorter than this is a click or a breath, unless it holds a word +FRAME = .01 +LEVEL = .06 # speech is above this share of the loud part of the voice (voiced RMS), ... +FLOOR = 4. # ... and above this many times the quiet floor (10th percentile of the frames) +PAD = .03 # soft onsets and the tail of the last sound sit just outside the loud frames + + +def frame_levels(samples, rate): + """RMS of every 10 ms frame of a mono signal.""" + size = max(1, int(round(rate * FRAME))) + count = len(samples) // size + if not count: + return np.zeros(0, dtype=np.float32) + return np.sqrt((np.asarray(samples[:count * size], dtype=np.float32).reshape(count, size) ** 2).mean(axis=1)) + + +def speech_runs(samples, rate, *, gap=GAP): + """[(start, end)] seconds of the voiced stretches; silences shorter than `gap` do not split a stretch.""" + levels = frame_levels(samples, rate) + if not len(levels): + return [] + loud = float(np.percentile(levels, 95)) + if loud <= 1e-4: + return [] + floor = float(np.percentile(levels, 10)) + # a clip that is nearly all speech has a loud 10th percentile, so the floor term never passes a third of the loud part + threshold = max(LEVEL * loud, min(FLOOR * floor, .3 * loud)) + voiced = levels > threshold + runs, begin = [], None + for index, flag in enumerate(voiced): + if flag and begin is None: + begin = index + elif not flag and begin is not None: + runs.append([begin, index]) + begin = None + if begin is not None: + runs.append([begin, len(voiced)]) + merged = [] + for first, last in runs: + if merged and (first - merged[-1][1]) * FRAME < gap: + merged[-1][1] = last + else: + merged.append([first, last]) + total = len(samples) / rate + return [(max(0., first * FRAME - PAD), min(total, last * FRAME + PAD)) for first, last in merged] + + +def _join(words): + text = '' + for word in words: + piece = word['text'] + if text and piece[:1].isascii() and piece[:1].isalnum() and text[-1].isascii(): + text += ' ' + text += piece + return text + + +def _split_long(runs, samples, rate, longest): + """A stretch longer than `longest` is cut at its quietest 10 ms near the middle of what is left.""" + levels = frame_levels(samples, rate) + result = [] + for start, end in runs: + while end - start > longest: + low, high = int((start + longest * .5) / FRAME), int((start + longest) / FRAME) + window = levels[low:high] + cut = (low + int(np.argmin(window))) * FRAME if len(window) else start + longest + result.append((start, cut)) + start = cut + result.append((start, end)) + return result + + +def lines(words, samples, rate, *, gap=GAP, longest=LONGEST): + """The lines of one voice: [{start, end, text, words}], in order, from word times and the voice samples. + + `words` are [{text, start, end}]. A stretch without a word is not a line; a word outside every stretch + joins the nearest one, because the recogniser's times can lead or lag a little.""" + runs = _split_long(speech_runs(samples, rate, gap=gap), samples, rate, longest) + held = [[] for _ in runs] + for word in words: + middle = (word['start'] + word['end']) / 2 + if not runs: + break + best = min(range(len(runs)), key=lambda i: 0. if runs[i][0] <= middle <= runs[i][1] + else min(abs(middle - runs[i][0]), abs(middle - runs[i][1]))) + held[best].append(word) + result = [] + for (start, end), group in zip(runs, held): + if not group: + continue + group.sort(key=lambda word: word['start']) + result.append(dict(start=round(start, 3), end=round(end, 3), text=_join(group), words=group)) + # a stretch too short to dub joins the neighbour it is closest to when that one is near enough + index = 0 + while index < len(result): + row = result[index] + if row['end'] - row['start'] < SHORTEST and len(result) > 1: + before = result[index - 1] if index else None + after = result[index + 1] if index + 1 < len(result) else None + gaps = [(row['start'] - before['end'], index - 1) if before else None, + (after['start'] - row['end'], index + 1) if after else None] + gaps = [item for item in gaps if item] + near, neighbour = min(gaps) + if near < .6 and (row['end'] - result[neighbour]['start'] if neighbour < index + else result[neighbour]['end'] - row['start']) <= longest: + other = result[neighbour] + first, second = (other, row) if neighbour < index else (row, other) + merged = dict(start=first['start'], end=second['end'], text=_join(first['words'] + second['words']), + words=first['words'] + second['words']) + low = min(index, neighbour) + result[low:low + 2] = [merged] + index = max(0, low) + continue + index += 1 + return result diff --git a/freevideo_source_bootstrap.py b/freevideo_source_bootstrap.py index fcc4c466..0a5cd05a 100644 --- a/freevideo_source_bootstrap.py +++ b/freevideo_source_bootstrap.py @@ -105,6 +105,12 @@ def root(): 'web/assets/tour/c10-person.webp', 'web/assets/tour/c10-s1.mp4', 'web/assets/tour/c10-s1.webp', + 'web/assets/tour/c11-s1.mp4', + 'web/assets/tour/c11-s1.webp', + 'web/assets/tour/c12-s1.mp4', + 'web/assets/tour/c12-s1.webp', + 'web/assets/tour/c13-s1.mp4', + 'web/assets/tour/c13-s1.webp', 'web/assets/tour/c10-source.webp', 'web/assets/tour/fox-sheet.webp', )) diff --git a/pyproject.toml b/pyproject.toml index 448e5858..51ea79d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,5 +35,6 @@ include = ["freevideo_engine*"] [tool.setuptools.package-data] "freevideo_engine.prompt_vlm" = ["model.json"] +"freevideo_engine.dubbing" = ["cosyvoice3_pack.json"] "freevideo_engine.image_gen" = ["image_models.json", "licenses/NOTICE.txt"] freevideo_engine = ["stage_estimate_params.json", "build-version.txt", "build-identity.json", "release_notes.json", "dependencies.json", "model_files.json", "model_shares.json", "prepared_models.json", "macos_reference_catalog.json", "superseded_tables.json", "bootstrap_versions.json", "curl_windows.json", "git_linux.json", "git_macos.json", "gcc_linux.json", "uv_downloads.json", "torch_downloads.json", "test_prompts.json", "windows_bootstrap.ps1", "assets/*.png", "assets/*.ico", "launcher/*.qml", "launcher/licenses/*.txt", "licenses/*.txt", "licenses/runtime/*.txt", "subtitle_pack.json", "frame_preview_weights.json", "agent/skill/freevideo/*.txt", "agent/skill/freevideo/references/*.txt", "depth_reference_model.json", "swap_lora.json"] diff --git a/web/agent_connect.css b/web/agent_connect.css index 5e78fdc5..73d65384 100644 --- a/web/agent_connect.css +++ b/web/agent_connect.css @@ -41,6 +41,7 @@ .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-header-actions>button{flex:none} /* Compact spacing only once the column is short of room, so wide headers keep their idle rhythm. */ @container fv-autopilot-actions (max-width:671px){.fv-header:has(.fv-autopilot-now:not([hidden])) .fv-header-actions>button{padding-inline:5px}} +@container fv-autopilot-actions (max-width:632px){.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-autopilot-lamp:lang(zh){width:30px;height:30px;padding:0;justify-content:center}.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-autopilot-lamp:lang(zh)>span{display:none}} .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-autopilot-now{flex:0 1 auto;min-width:0} .fv-studio .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-autopilot-now-go{flex:0 1 auto;min-width:0;gap:5px;padding-left:8px} .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-autopilot-now-go>:not(.fv-autopilot-now-name){flex:none} diff --git a/web/assets/tour/c11-s1.mp4 b/web/assets/tour/c11-s1.mp4 new file mode 100644 index 00000000..60506e4a Binary files /dev/null and b/web/assets/tour/c11-s1.mp4 differ diff --git a/web/assets/tour/c11-s1.webp b/web/assets/tour/c11-s1.webp new file mode 100644 index 00000000..ef5c0092 Binary files /dev/null and b/web/assets/tour/c11-s1.webp differ diff --git a/web/assets/tour/c12-s1.mp4 b/web/assets/tour/c12-s1.mp4 new file mode 100644 index 00000000..61abd005 Binary files /dev/null and b/web/assets/tour/c12-s1.mp4 differ diff --git a/web/assets/tour/c12-s1.webp b/web/assets/tour/c12-s1.webp new file mode 100644 index 00000000..ca90cf28 Binary files /dev/null and b/web/assets/tour/c12-s1.webp differ diff --git a/web/assets/tour/c13-s1.mp4 b/web/assets/tour/c13-s1.mp4 new file mode 100644 index 00000000..6d847e3a Binary files /dev/null and b/web/assets/tour/c13-s1.mp4 differ diff --git a/web/assets/tour/c13-s1.webp b/web/assets/tour/c13-s1.webp new file mode 100644 index 00000000..338bb14e Binary files /dev/null and b/web/assets/tour/c13-s1.webp differ diff --git a/web/autopilot.js b/web/autopilot.js index 7fe11558..9eb380e3 100644 --- a/web/autopilot.js +++ b/web/autopilot.js @@ -8,7 +8,7 @@ import { openEditorView } from './editor_loader.js'; import { closeDialog } from './motion.js'; import { productVersion } from './updates.js'; import { openSetup } from './setup.js'; -import { feedbackButton } from './feedback.js'; +import { feedbackButton, guideButton } from './feedback.js'; import { openExtendEditor, openStudio, showNodes } from './studio.js'; import { agentTabReady, viewChanged, viewSwitch } from './view_navigation.js'; import { wordmark } from './wordmark.js'; @@ -37,7 +37,7 @@ export function openAutopilotView(node, {studio = null, initialPrompt = '', init halt.title = t('Stop all', '全部停止'); halt.setAttribute('aria-label', halt.title); const line = el('i', 'fv-autopilot-line'); const phoneAccess = createLanAccess(t, api); - tools.append(now, lamp, phoneAccess.element, feedbackButton(t), settings); + tools.append(now, lamp, phoneAccess.element, guideButton(t), feedbackButton(t), settings); // The view chosen next takes over once this page has closed; Escape goes back to what is underneath. let next = () => studio?.open ? viewChanged('studio', node) : viewChanged('nodes', node); const leave = action => { next = action; closeDialog(dialog); }; diff --git a/web/cast_library.js b/web/cast_library.js index 0f21f51e..d06ebb68 100644 --- a/web/cast_library.js +++ b/web/cast_library.js @@ -42,7 +42,7 @@ export function createCastLibrary({api, text, viewURL, uploadOne, onChange = () const avatars = el('div', 'fv-cast-avatars'); const skipped = el('div', 'fv-cast-unreadable'); skipped.hidden = true; const add = button('fv-cast-add', '+', () => openDialog()); - add.setAttribute('aria-label', word(text('New character', '新建角色'), text('New scene', '新建场景'))); + add.setAttribute('aria-label', word(text('New character', '新建角色'), text('New scene', '新建场景'))); add.dataset.tourId = 'cast-add'; element.append(heading, avatars, add, skipped); async function value(response) { diff --git a/web/editor.css b/web/editor.css index 9bb0997e..667f5c41 100644 --- a/web/editor.css +++ b/web/editor.css @@ -253,7 +253,8 @@ .fv-studio.fv-editor .fv-header[data-fit~=wrap]>.fv-view-switch{grid-column:1/-1;grid-row:2;justify-self:center} .fv-studio.fv-editor .fv-header[data-fit~=wrap]>.fv-header-actions{grid-column:2;grid-row:1} .fv-studio.fv-editor .fv-header[data-fit~=wrap] .fv-ed-name{max-width:320px} -.fv-studio.fv-editor .fv-header[data-fit~=lean] .fv-feedback-entry{display:none} +.fv-studio.fv-editor .fv-header[data-fit~=lean] .fv-feedback-entry:not(.fv-guide-entry){display:none} +.fv-studio.fv-editor .fv-header[data-fit~=least] .fv-guide-entry{display:none} .fv-ed-volume-popover{position:absolute;top:calc(100% + 8px);left:0;display:flex;align-items:center;gap:8px;padding:10px 12px;border:1px solid var(--fv-border);border-radius:8px;background:var(--fv-card);white-space:nowrap;z-index:8} .fv-ed-volume{width:90px}.fv-ed-volume-value{font-variant-numeric:tabular-nums;font-size:12px;min-width:34px} diff --git a/web/editor.js b/web/editor.js index b64a124d..37885935 100644 --- a/web/editor.js +++ b/web/editor.js @@ -6,13 +6,15 @@ import {isText} from './editor_text_model.js'; import {viewSwitch, viewChanged} from './view_navigation.js'; import {wordmark} from './wordmark.js'; import {openSetup} from './setup.js'; -import {feedbackButton} from './feedback.js'; +import {feedbackButton, guideButton} from './feedback.js'; import {issueButton} from './issue_button.js'; import * as model from './editor_model.js'; import {discardEmptyEdit, hiddenEmptyEdit, deleteEditText} from './editor_lifecycle.js'; import {createEditorState} from './editor_state.js'; import {createEditorStore, sourceKey, mediaURL} from './editor_store.js'; import {generationSize, continuedSize} from './editor_generation.js'; +import {uploadMedia} from './media_upload.js'; +import {checkOriginal} from './audio_original.js'; import {createEditorLibrary} from './editor_library.js'; import {createEditorPlayer} from './editor_player.js'; import {createEditorTransport, registerEditorShortcuts} from './editor_transport.js'; @@ -27,6 +29,8 @@ import {createSpeedComponent, readSpeedAudioSamples, handleSpeedKey, speedShortc import {createEditorGenbar, loadEditorComponents, editorBlockStates, blockMessages} from './editor_genbar.js'; import {createEditorMusic} from './editor_music.js'; import {createEditorRecord, RECORD_ICON} from './editor_record.js'; +import {createEditorVoice} from './editor_voice.js'; +import {createVoiceStore, voiceAvailable} from './editor_voice_store.js'; import {MAX_MUSIC_TEXT as MUSIC_TEXT_LIMIT} from './editor_music_model.js'; import {createProgressConnection} from './progress_connection.js'; import {createSelectionActions, deletionStatus, isMultiple, isTyping, playheadIds, removableGaps, removeGaps} from './editor_select.js'; @@ -122,6 +126,12 @@ function saveReason(failure) { if (!failure) return 'unsaved'; return failure.reason || (failure.status ? 'http_' + failure.status : isNetworkTypeError(failure) ? 'network' : undefined); } +// Names the failed delete for the report: a reason given by the server or the page as it is, else the HTTP status, a lost connection or the error's own code or name, prefixed with the step when it was not the delete request itself. +function deleteReason(value, step = 'delete') { + if (value?.reason) return value.reason; + const code = value?.status ? 'http_' + value.status : isNetworkTypeError(value) ? 'network' : value?.code || value?.name; + return code && step !== 'delete' ? step + '_' + code : code; +} function restoreGeneration(record, before, prepared) { const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); const entries = value => new Map(value.tracks.flatMap(track => track.clips.map(clip => [clip.id, {track: track.id, clip}]))); @@ -209,7 +219,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const top = el('header', 'fv-header fv-ed-top'), libraryHost = el('aside', 'fv-ed-library'), preview = el('section', 'fv-ed-preview'), transport = el('div', 'fv-ed-transport'), genHost = el('div', 'fv-ed-genbar'), tracksHost = el('div', 'fv-ed-tracks'), zoomHost = el('div', 'fv-ed-zoom'); const adjustHost = el('aside', 'fv-ed-adjust-host'); - tracksHost.dataset.fvGuide = 'editor-timeline'; + tracksHost.dataset.fvGuide = 'editor-timeline'; tracksHost.dataset.tourId = 'editor-timeline'; const button = (label, action, cls = '') => { const value = el('button', cls, label); value.type = 'button'; value.onclick = action; return value; }; const navigation = viewSwitch('editor', () => handle.switchView('studio'), () => handle.switchView('nodes'), () => {}, () => handle.switchView('autopilot')); for (const control of navigation.querySelectorAll('button')) control.removeAttribute('title'); @@ -230,7 +240,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given exportBox.append(exportHint, exportButton, exportCancel); brand.append(wordmark(), title); const phoneAccess = createLanAccess(t, api); listeners.push(phoneAccess.dispose); - headerActions.append(exportBox, phoneAccess.element, feedbackButton(t), button(t('Settings', '设置'), openSetup, 'fv-quiet')); + headerActions.append(exportBox, phoneAccess.element, guideButton(t), feedbackButton(t), button(t('Settings', '设置'), openSetup, 'fv-quiet')); top.append(brand, navigation, headerActions); const headerFit = fitEditorHeader(top, brand, title, navigation, headerActions); const menu = el('div', 'fv-ed-name-menu'); menu.hidden = true; menu.setAttribute('role', 'menu'); @@ -259,7 +269,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given let stems = null, disposed = false, suppressSave = false, savedContent = '', initializing = true, tracks = null, genbar = null, adjustPanel = null, exportStatus = null, pointerEditing = false, deferredStamp = null; let capability = {maxSeconds: 15, maxFollowSeconds: 15, maxRequestFrames: null, estimate: () => null}, generationBusy = false, switching = false, generationDone = null, finishGeneration = null; let noticeTimer = null, stopExport = null, mostRecentJob = null, exportPreparing = false, guide = null; - const jobWatchers = new Map(), progress = new Map(), jobStatuses = new Map(), runProgress = new Map(); + const jobWatchers = new Map(), progress = new Map(), jobStatuses = new Map(), runProgress = new Map(), startPlayheads = new Map(), soundJobs = new Set(), shownJobs = new Set(); const failures = createEditorFailures({api, state, jobStatuses, reasonText: value => editorErrorText(value, t)}); let submissionFailure = null; let blockStates = null, progressObserver = null, progressConnection = null; @@ -536,6 +546,40 @@ export function openEditor({api, app, from = null, panel = {}, components: given } // Music on the second sound track goes through the same bar; the bar shows a chosen piece's own job. const music = createEditorMusic({state, store, t, completeSources, onStatus: () => genbar?.update(barValue)}); + // Dubbing lines of a stretch of speech. + let voiceReady = false; + const generatedSource = value => { const match = /^generated:(.+)$/.exec(String(value || '')); return match ? {kind: 'generated', id: match[1]} : null; }; + // Playing one line: the playhead goes to its start and the picture plays up to its end, then stops. + let lineStop = null; + function playVoiceLine(line) { + const found = voice.target(); if (!found) return; + lineStop?.(); + const from = Math.round(found.clip.start + (found.start + line.start) * model.FPS), to = Math.round(found.clip.start + (found.start + line.end) * model.FPS); + state.seek(from); player.play(); + const finish = () => { unwatch(); if (lineStop === finish) lineStop = null; }; + const unwatch = state.subscribe(event => { if ((event.type ?? event) !== 'playhead') return; if (state.playhead >= to) { player.pause(); finish(); } else if (!player.playing) finish(); }); + lineStop = finish; + } + const voice = createEditorVoice({state, store: createVoiceStore({api, state, queue}), t, onChange: () => genbar?.update(barValue), available: () => voiceReady, playLine: playVoiceLine, comparePlayback: async (clipId, name) => { + // Hearing the other take changes nothing in the edit: only the preview swaps the sound. + const source = generatedSource(name); + if (!source) { player.previewClipSource(clipId, null); return; } + await store.getSources([sourceKey(source)]); + player.previewClipSource(clipId, source); + }, place: async result => { + // What to put into the project is what the service says in `apply`; the plain shape is the same as replacing a source. + const apply = result.apply || {op: 'replaceSource', clip_id: result.clip_id, source: result.sources?.new}; + const dubbed = generatedSource(result.sources?.new); + if (!dubbed || !apply.clip_id) throw new model.EditError('source'); + if (apply.op === 'placeStems' && !model.placeStems) throw new model.EditError('source'); + const stems = apply.op === 'placeStems' ? Object.fromEntries(model.STEM_TRACKS.map(name => [name, generatedSource(apply.sources?.[name])])) : null; + if (stems && (!model.placeStems || !Object.values(stems).every(Boolean))) throw new model.EditError('source'); + const single = !stems && (generatedSource(apply.source) || dubbed); + await store.getSources([...(stems ? Object.values(stems) : [single])].map(sourceKey)); + state.apply(t('Redub', '重新配音'), record => stems ? model.placeStems(record, apply.clip_id, stems) : model.replaceSource(record, apply.clip_id, single)); + const found = stems && state.record.tracks.flatMap(track => track.clips).find(clip => clip.stem_of === apply.clip_id && clip.source?.id === stems.dialogue.id); + return {voiceClip: stems ? found?.id : apply.clip_id}; + }}); let barValue = null; const exportMatches = () => exportSnapshot && exportSnapshot.id === state.record.id && exportSnapshot.revision === state.record.revision && exportSnapshot.content === content(state.record); function explainExport() { @@ -586,6 +630,28 @@ export function openEditor({api, app, from = null, panel = {}, components: given delete inputs.continue_from; delete inputs.continue_to; return {node: String(value.node), prompt: value.prompt, ...(value.extra_data ? {extra_data: value.extra_data} : {}), ...(api.clientId ? {client_id: api.clientId} : {})}; } + // A part drawn over imported sound follows that sound as it is: the picture is made for the range of the file the + // parts cover, and the sound itself is kept. + async function followSound(frozen, record, ids) { + const videos = ids.map(id => model.find(record, id)).filter(Boolean).sort((a, b) => a.start - b.start); + const sound = videos.map(clip => model.find(record, clip.link)); + if (!videos.length || sound.some(clip => clip?.source?.kind !== 'import' || store.sources.get(sourceKey(clip.source))?.has_video !== false)) return; + const source = sound[0].source, metadata = store.sources.get(sourceKey(source)); + const blob = await (await fetch(metadata.audio)).blob(), extension = /\.[a-z0-9]{1,5}$/i.exec(String(metadata.name || metadata.path || ''))?.[0] || '.wav'; + const {file} = await uploadMedia(api, new File([blob], `editor-${source.id}${extension}`, {type: blob.type})); + const rate = (await checkOriginal(api, {file})).sample_rate; + if (!(rate > 0)) throw new Error('audio_unusable'); + const first = Math.round(sound[0].in * rate / model.FPS), frames = videos.reduce((sum, clip) => sum + clip.frames, 0); + const row = {file, role: 'reference', use: 'original', range: {start_sample: first, end_sample: first + Math.round(frames * rate / model.FPS)}}; + const inputs = frozen.prompt[frozen.node].inputs, held = frozen.prompt[String(inputs.media?.[0])]; + if (held?.class_type === 'FreeVideoMedia') { + const rows = JSON.parse(held.inputs.assets || '[]').filter(item => item.use !== 'original'); + held.inputs.assets = JSON.stringify([...rows, row]); + } else { + const id = String(1 + Math.max(0, ...Object.keys(frozen.prompt).map(Number).filter(Number.isFinite))); + frozen.prompt[id] = {class_type: 'FreeVideoMedia', inputs: {assets: JSON.stringify([row])}}; inputs.media = [id, 0]; + } + } function pinStoppedJob(record, ids, job) { return model.settle(record, ids.filter(id => model.attemptClip(record, id)?.job === job), 'stopped'); } @@ -596,6 +662,10 @@ export function openEditor({api, app, from = null, panel = {}, components: given return ['queued', 'running', 'cancelling'].includes(previous.state); failures.reconcile(job, previous); mostRecentJob = job.id; jobStatuses.set(job.id, job); paintJob(job); + if (!startPlayheads.has(job.id)) startPlayheads.set(job.id, state.playhead); + // Whether the job follows imported sound is read before a result replaces the sound's link target. + if (!soundJobs.has(job.id) && (job.blocks || []).some(block => { const heard = model.find(state.record, model.attemptClip(state.record, block.id)?.link); + return heard?.source?.kind === 'import' && store.sources.get(sourceKey(heard.source))?.has_video === false; })) soundJobs.add(job.id); genbar?.update(barJob(job)); const completed = new Set(); for (const block of job.blocks || []) { @@ -616,6 +686,13 @@ export function openEditor({api, app, from = null, panel = {}, components: given // Validate the current clip first so a bad receipt is visible. model.fill(state.record, block.id, result); state.settle(record => model.fill(record, block.id, result)); + // A sound-led result the playhead has not reached is shown from its first frame, once per job and unless the viewer moved on meanwhile. + const done = model.find(state.record, block.id), picked = state.selection; + if (done && soundJobs.has(job.id) && !shownJobs.has(job.id)) { + shownJobs.add(job.id); + if (picked?.kind === 'clip' && picked.ids.includes(block.id) && !player.playing && state.playhead === startPlayheads.get(job.id) + && !(state.playhead >= done.start && state.playhead < done.start + done.frames)) state.seek(done.start); + } } catch (value) { if (!(value instanceof model.EditError)) throw value; state.settle(record => model.fail(record, block.id, 'result_mismatch', job.id)); @@ -780,22 +857,27 @@ export function openEditor({api, app, from = null, panel = {}, components: given async function deleteCurrentProject() { if (switching || closing) return; switching = true; element.inert = true; updateTop(); + let step = 'flush'; try { const words = deleteEditText(t); if (jobActive()) throw {reason: 'generation_active'}; await queue.flush(); releaseStamp(); if (queue.dirty) { notify(saveErrorText(queue.failure, t), EDITOR_ISSUE, failureKind('save', {reason: saveReason(queue.failure)})); return; } // Load the replacement before deleting, so a read failure keeps the edit open. + step = 'list'; const rows = (await store.projects()).filter(row => row.id !== state.record.id); // Most recently used: the order the edits were opened, then the last save. const order = id => { try { return Number(localStorage.getItem(usedPrefix + id)) || 0; } catch { return 0; } }; rows.sort((a, b) => order(b.id) - order(a.id) || String(b.updated_at).localeCompare(String(a.updated_at))); + step = 'load'; const record = rows.length ? await store.load(rows[0].id) : initial(); - if (!record) throw new Error(words.delete_failed); + if (!record) throw Object.assign(new Error(words.delete_failed), {code: 'no_replacement'}); + step = 'delete'; await discardProject(); + step = 'open'; await replaceProject(record); menu.hidden = true; title.setAttribute('aria-expanded', 'false'); title.focus(); } catch (value) { - const words = deleteEditText(t); notify(words[value?.reason] || words.delete_failed, EDITOR_ISSUE, failureKind('delete', {reason: value?.reason})); + const words = deleteEditText(t); notify(words[value?.reason] || words.delete_failed, EDITOR_ISSUE, failureKind('delete', {reason: deleteReason(value, step)})); } finally { switching = false; element.inert = false; updateTop(); } } function confirmDelete() { @@ -904,6 +986,11 @@ export function openEditor({api, app, from = null, panel = {}, components: given } catch (value) { if (value instanceof model.EditError) return value.code === 'busy'; throw value; } }, + // A range with no picture over imported sound: the picture is made to follow that sound. + soundLed(selection = state.selection) { + const range = selection?.kind === 'range' && selectionRange(selection); + return !!range && !!model.soundRange(state.record, range.start, range.frames); + }, // Another part generating is no reason: the server runs parts one after another. canGenerate(text, selection = state.selection) { return store.generationAvailable && actions.supportedSelection(selection) && (!!text.trim() || !actions.needsText(selection)); @@ -1077,6 +1164,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given if ('preview' in inputs || previewing) inputs.preview = previewing && alone; if ('refine_from' in inputs) inputs.refine_from = ''; } + await followSound(frozen, prepared, ids); if (allowEmpty === true && [text, ...Object.values(texts ?? {})].some(value => !value.trim())) allowEmptyPrompt({output: frozen.prompt}, frozen.node); // Parts made at another size than the edit's have that size's limit. const limit = (follow && (follow.width !== edit.width || follow.height !== edit.height) @@ -1128,6 +1216,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given async press(text, {allowEmpty = false} = {}) { if (disposed || closing || initializing) return {say: t('Opening the edit…', '正在打开…')}; text = wellFormed(text); + if (voice.session() && ['ready', 'done'].includes(voice.session().phase)) { await voice.redub(); return {}; } // A stretch of the music track, or a piece of music there: music, with or without words. if (music.target()) { if (characters(text) > MUSIC_TEXT_LIMIT) return {say: textLongText(t, MUSIC_TEXT_LIMIT), focus: true}; @@ -1211,6 +1300,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given async cancel() { if (music.target()) { await music.cancel(); return; } if (stems?.active()) { await stems.cancel(); return; } + if (voice.session()?.phase === 'dubbing') { voice.cancel(); return; } const range = selectionRange(state.selection); const own = range && state.record.tracks[0].clips.find(clip => clip.job && generating(clip) && clip.start < range.start + range.frames && clip.start + clip.frames > range.start)?.job; @@ -1339,6 +1429,8 @@ export function openEditor({api, app, from = null, panel = {}, components: given return running; } + voiceAvailable(api).then(value => { voiceReady = value; if (!disposed) voice.refresh(); }); + actions.voice = {controller: voice}; actions.music = {target: selection => music.target(selection), preset: selection => music.preset(selection), choose: value => music.choose(value), styles: music.styles, moods: music.moods, downloadBytes: music.downloadBytes}; genbar = createEditorGenbar(genHost, {state, t, actions, text: ''}); settleService(); @@ -1617,12 +1709,12 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (!queue.dirty && emptyEdit()) { try { await discardProject({automatic: true}); } catch (value) { - notify(deleteEditText(t)[value?.reason] || deleteEditText(t).delete_failed, EDITOR_ISSUE, failureKind('delete', {reason: value?.reason})); + notify(deleteEditText(t)[value?.reason] || deleteEditText(t).delete_failed, EDITOR_ISSUE, failureKind('delete', {reason: deleteReason(value)})); closing = false; element.inert = false; closePromise = null; return false; } } await store.activateProject(null).catch(() => {}); - rememberView(); disposed = true; music.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); + rememberView(); disposed = true; music.dispose(); voice.dispose(); guide?.dispose(); transportControls.dispose(); toolbarMore.dispose(); player.dispose(); textTools.dispose(); adjust.dispose(); notes.dispose(); pictureEdit.dispose(); audio.dispose(); stems?.dispose(); recording.dispose(); headerFit.dispose(); listeners.splice(0).forEach(remove => remove()); diff --git a/web/editor_adjust.js b/web/editor_adjust.js index ed90610d..206fc4f6 100644 --- a/web/editor_adjust.js +++ b/web/editor_adjust.js @@ -48,7 +48,7 @@ function selectedContext(state, t, actions) { export function createAdjustPanel(host, {state, t, actions}) { const shell = host.closest('.fv-editor'), uid = `fv-ed-adjust-${++serial}`; host.classList.add('fv-ed-adjust-host'); - const element = el('section', 'fv-ed-adjust'); element.id = uid; + const element = el('section', 'fv-ed-adjust'); element.id = uid; element.dataset.tourId = 'editor-inspector'; element.setAttribute('aria-label', t('Adjust', '调整')); const tabs = el('div', 'fv-ed-tabs fv-pass-mode'); tabs.setAttribute('role', 'tablist'); tabs.setAttribute('aria-label', t('Adjust', '调整')); @@ -56,7 +56,7 @@ export function createAdjustPanel(host, {state, t, actions}) { const buttons = new Map(), panes = new Map(), entries = new Map(); const toggleButton = button('', {onClick: () => overlay ? close({restoreFocus: true}) : open()}); toggleButton.append(editorIcon('adjust')); toggleButton.setAttribute('aria-label', t('Adjust', '调整')); toggleButton.title = t('Adjust', '调整'); - toggleButton.classList.add('fv-ed-adjust-toggle'); toggleButton.setAttribute('aria-controls', uid); + toggleButton.classList.add('fv-ed-adjust-toggle'); toggleButton.dataset.tourToggle = 'editor-inspector'; toggleButton.setAttribute('aria-controls', uid); const labels = {video: t('Video', '画面'), audio: t('Audio', '声音'), speed: t('Speed', '速度'), text: t('Text', '文字'), subtitle: t('Subtitles', '字幕')}; for (const kind of kinds) { const tab = button(labels[kind], {onClick: () => activate(kind)}); diff --git a/web/editor_audio.css b/web/editor_audio.css index 0a64fc85..f92cf08b 100644 --- a/web/editor_audio.css +++ b/web/editor_audio.css @@ -12,8 +12,9 @@ .fv-ed-track-row[data-locked=true],.fv-ed-track-row[data-locked=true] *{cursor:not-allowed!important} .fv-ed-audio-fades{position:absolute;inset:3px 0;width:100%;height:calc(100% - 6px);fill:none;stroke:var(--fv-ink);stroke-width:1.5;pointer-events:none;z-index:2} .fv-ed-audio-fades path{vector-effect:non-scaling-stroke} -.fv-ed-track-view .fv-ed-audio-junction{position:absolute;top:50%;width:10px;height:10px;transform:translate(-50%,-50%) rotate(45deg);padding:0;background:var(--fv-card);border:1px solid var(--fv-accent);border-radius:1px;z-index:7;opacity:0;cursor:pointer} +.fv-ed-track-view .fv-ed-audio-junction{position:absolute;top:50%;box-sizing:border-box;width:10px;height:10px;min-width:0;min-height:0;transform:translate(-50%,-50%) rotate(45deg);padding:0;background:var(--fv-card);border:1px solid var(--fv-accent);border-radius:1px;z-index:7;opacity:0;cursor:pointer} .fv-ed-audio-junction:hover,.fv-ed-audio-junction:focus-visible,.fv-ed-audio-junction[data-active=true]{opacity:1!important} +.fv-ed-track-view .fv-ed-audio-junction:disabled{opacity:0!important;pointer-events:none} .fv-ed-audio-junction[data-active=true]{background:var(--fv-accent)!important} .fv-ed-audio-junction[aria-pressed=true]{box-shadow:0 0 0 2px var(--fv-card),0 0 0 3px var(--fv-accent)} .fv-ed-audio-notice{position:absolute;z-index:12;padding:5px 8px;border:1px solid var(--fv-border);border-radius:6px;background:var(--fv-card);color:var(--fv-ink);font-size:12px;pointer-events:none} diff --git a/web/editor_audio.js b/web/editor_audio.js index 305b2f6e..576a3842 100644 --- a/web/editor_audio.js +++ b/web/editor_audio.js @@ -342,12 +342,12 @@ export function createSpeechSpans(fetchSpans, {retry = 1000} = {}) { } // A picture's own sound: still joined to it, or taken from it and kept. export const isOriginalSound = clip => !!clip.link || !!clip.detached_from; -export function createAudioPreview({state, sources, api, volumePreview = new Map(), readSamples = null, error = () => {}, speech = null}) { +export function createAudioPreview({state, sources, api, volumePreview = new Map(), sourcePreview = new Map(), readSamples = null, error = () => {}, speech = null}) { let context = null, generation = 0, timer = null, identity = null, pending = false, disposed = false; const cache = new Map(), nodes = new Set(); let origin = 0, cursor = 0, previousPosition = 0; const ducking = (record, rate = 48000) => speech && sound.duckSources(record.tracks).length ? sound.duckRanges(record.tracks, speech.map, rate) : []; const edited = record => record.tracks.some(track => track.muted || track.solo || track.clips.some(clip => clip.fade_in || clip.fade_out || clip.crossfade)) - || ducking(record).length > 0; + || sourcePreview.size > 0 || ducking(record).length > 0; function stop() { generation++; clearTimeout(timer); timer = null; pending = false; identity = null; nodes.forEach(node => { try { node.stop(); } catch {} }); nodes.clear(); } async function decoded(clip) { const metadata = sources.get(sourceKey(clip.source)), url = mediaURL(api, metadata?.audio); @@ -430,6 +430,7 @@ export function createAudioPreview({state, sources, api, volumePreview = new Map render({record, position, playing, playbackRate = 1, silenceOriginal = false}) { speech?.want(sound.duckSources(record.tracks)); if (!edited(record)) { if (identity) stop(); return false; } + if (sourcePreview.size) record = {...record, tracks: record.tracks.map(track => ({...track, clips: track.clips.map(clip => sourcePreview.has(clip.id) ? {...clip, source: sourcePreview.get(clip.id)} : clip)}))}; if (volumePreview.size) record = {...record, tracks: record.tracks.map(track => ({...track, clips: track.clips.map(clip => volumePreview.has(clip.id) ? {...clip, volume: volumePreview.get(clip.id)} : clip)}))}; // While a take is recorded the picture's own sound stays out of the mix. if (silenceOriginal) record = {...record, tracks: record.tracks.map(track => ({...track, clips: track.clips.filter(clip => !isOriginalSound(clip))}))}; diff --git a/web/editor_first_frame.js b/web/editor_first_frame.js index f2ae48a8..6701ca59 100644 --- a/web/editor_first_frame.js +++ b/web/editor_first_frame.js @@ -7,7 +7,7 @@ export const firstFrameOn = record => record.settings?.[FIRST_FRAME_KEY] === tru // actions.setFirstFrame(value): saves the choice with the project. export function createFirstFrameSwitch(host, {state, t, actions}) { const {row, input, setDisabled} = firstFrameRow(t, 'fv-ed-first-frame'); - host.append(row); + row.dataset.tourId = 'editor-first-frame'; host.append(row); let shown = false; // While generating the row stays as it was and cannot be changed. function update(busy = false) { diff --git a/web/editor_genbar.js b/web/editor_genbar.js index 5bb35ff9..00e2c141 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -9,6 +9,8 @@ import {waitingText} from './chain_support.js'; import {showEmptyPrompt} from './empty_prompt_notice.js'; import {segmented} from './editor_adjust.js'; import {createPassMode} from './pass_mode.js'; +import {createVoicePanel} from './editor_voice.js'; +import * as dubbing from './editor_voice_model.js'; const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; export const timeEstimate = (seconds, t) => seconds == null ? '' : seconds < 60 ? t(`about ${Math.max(1, Math.ceil(seconds))} s`, `约 ${Math.max(1, Math.ceil(seconds))} 秒`) @@ -118,6 +120,9 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { // Where a narrow English bar starts Generate on a line of its own (editor.css); hidden elsewhere. const lineBreak = el('span', 'fv-ed-genbar-break'); lineBreak.setAttribute('aria-hidden', 'true'); generate.before(lineBreak); host.lang = t('en', 'zh-CN'); + // Dubbing: the lines of a chosen stretch of speech take the description box's place. + const voice = actions.voice || null, voicePanel = voice ? createVoicePanel(host, voice.controller, {t}) : null; + if (voicePanel) description.before(voicePanel.element); const failure = el('div', 'fv-ed-gen-failure fv-result-links'), failureBar = el('div', 'fv-ed-failure-actions fv-actions'); const failureReason = el('span', 'fv-ed-failure-reason'); failureReason.setAttribute('role', 'status'); const report = el('button', 'fv-ed-failure-report fv-action fv-action-secondary', t('Download diagnostic report', '下载诊断报告')); report.type = 'button'; @@ -176,6 +181,8 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { report.hidden = !selectedFailure || selectedFailure.report === false; const running = ['queued', 'running', 'cancelling'].includes(status?.state); const musical = !!music?.target(selection); + const dub = !previewSelection && voice?.controller.target(selection) ? voice.controller.session(voice.controller.target(selection)) : null; + const voiced = !!dub && dub.phase !== 'none' && dub.phase !== 'error'; const selected = !!selection && (!!previewSelection || musical || (actions.supportedSelection?.() ?? true)); // While a part generates, choosing another place offers Generate again; it is queued behind. // A press is being sent for the parts in `pressing`; only that same selection waits for it. @@ -188,6 +195,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { const opening = (starting || !!actions.selectionSending?.()) && !held; const stopped = ['failed', 'cancelled', 'interrupted'].includes(status?.state); host.dataset.mode = busy ? 'running' : selected ? 'selected' : 'idle'; + host.dataset.voice = String(voiced); voicePanel?.update(); hint.hidden = !busy && !other; // The description and Generate stay where they are while a part is made, greyed, so nothing around them moves; // music being made shows only its progress and Cancel. @@ -195,15 +203,21 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { // A choice that will move to an empty spot can still be described first. const writable = selected || !previewSelection && !!selection && !!actions.selectionMovable?.(); input.disabled = !writable || busy; - input.placeholder = !writable ? '' : musical ? t('Describe the music', '描述配乐') : t('Describe this part', '描述这一段'); + const sound = writable && !musical && !!actions.soundLed?.(); + input.placeholder = !writable ? '' : musical ? t('Describe the music', '描述配乐') : sound ? t('Describe the picture', '描述画面') : t('Describe this part', '描述这一段'); + if (sound) host.dataset.tourId = 'video-empty-range'; else delete host.dataset.tourId; input.setAttribute('aria-label', t('Description', '描述')); - generate.classList.toggle('fv-primary', selected && !busy); + generate.classList.toggle('fv-primary', (voiced ? voice.controller.changed(dub).length > 0 : selected) && !busy); // A cancel already accepted is not offered again; the bar says it is stopping until the run ends. const cancelling = status?.state === 'cancelling' && !held; const finalization = clipFinalization(status); cancel.hidden = !busy || opening || cancelling || !held && finalization?.cancelable === false; resume.hidden = !stopped; const waiting = status?.state === 'queued' && !(status.blocks || []).some(block => ['queued', 'running'].includes(block.state) && block.prompt_id); - if (opening) hint.textContent = t('Starting…', '正在开始…'); + const dubbingNow = voiced && dub.phase === 'dubbing', finding = voiced && dub.phase === 'finding'; + if (dubbingNow || finding) { + hint.hidden = false; + hint.textContent = finding ? t('Finding the lines', '正在识别台词') : dubbing.dubbingText(Math.max(1, dub.index), dub.count, dub.remaining, t); + } else if (opening) hint.textContent = t('Starting…', '正在开始…'); else if (held || waiting) hint.textContent = waiting && !held ? waitingText(status.waiting, t) : t('Queued', '排队中'); else if (cancelling) hint.textContent = t('Stopping…', '正在停止…'); else if (running) { @@ -252,14 +266,17 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (pass) passControl.sync(pass); if (passMenu.hidden) passMenu.open = false; } + if (voiced) generate.textContent = dubbing.redubLabel(voice.controller.changed(dub).length, t); // Never disabled for a reason the press itself can explain or fix; only until the server says it can generate. - generate.disabled = busy || !!previewSelection || !!actions.generationWaiting?.(); + generate.disabled = busy || !!previewSelection || !!actions.generationWaiting?.() || (voiced && ((dub.phase !== 'ready' && dub.phase !== 'done') || !voice.controller.changed(dub).length)); + // While it dubs only the progress sentence and Cancel stay; while it recognises there is nothing to press yet. + if (voiced) { generate.hidden = dub.phase === 'dubbing' || dub.phase === 'finding'; cancel.hidden = dub.phase !== 'dubbing'; cancel.disabled = false; } resizeDescription(); input.dispatchEvent(new Event('change')); generate.title = ''; if (assets) { if (selected && !busy && !musical) assets.refresh(); else assets.element.hidden = true; } showMusicAssets(selected && !busy && musical && !previewSelection); - if (selected && !busy) { + if (selected && !busy && !voiced) { const key = JSON.stringify(selection); if (estimateCache?.key === key) generate.textContent += estimateCache.label; const seconds = await actions.estimate(selection); @@ -368,7 +385,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { if (!value && note.contains(retryService)) { retryService.remove(); note.hidden = true; } showService(); }, getText() { expand.commit(); return input.value; }, update(value) { status = value; render(); }, preview(selection) { previewSelection = selection; render(); }, setText(value) { input.value = value || ''; state.setRangeText(input.value); if (bound && input.value !== bound.text) { dirty = true; written = input.value; } render(); }, - dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; + dispose() { disposed = true; ++serial; clearTimeout(noteTimer); stopPass?.(); passControl?.dispose(); document.removeEventListener('pointerdown', closePass, true); clearTimeout(refNoticeTimer); duration.dispose(); unsubscribe(); resize?.disconnect(); expand.dispose(); references?.dispose(); voicePanel?.dispose(); assets?.dispose(); Object.values(pickers).forEach(picker => picker.dispose()); }}; } // A missing optional module leaves the editor's existing progress in place. diff --git a/web/editor_header_fit.js b/web/editor_header_fit.js index b4468219..c7083064 100644 --- a/web/editor_header_fit.js +++ b/web/editor_header_fit.js @@ -1,6 +1,7 @@ // The view switch stays centred in the editor's header while the project name next to the logo stays readable. // When it would not, the logo first takes its phone size, then gives its place to the name; if that is still too tight, the switch moves to a second row. // The feedback entry leaves before the switch moves down, and on two rows when the name is still tight. +// The Tutorial entry stays on two rows, and leaves after the feedback entry only to keep the switch on one row. // About six Chinese characters or ten Latin letters of the name itself, without the menu arrow. const READABLE = 120, COMPACT_MARK = 90; @@ -25,14 +26,18 @@ export function fitEditorHeader(header, brand, title, navigation, actions) { // On two rows the name shares the first row with the actions only; the logo shrinks too when that is still tight. const row = markWidth => inner - need - gap - markWidth - spacing; // The feedback entry is the first action to leave this header (it stays in the other views). - const feedback = actions.querySelector('.fv-feedback-entry'), spare = feedback ? feedback.offsetWidth + apart : 0; + const feedback = actions.querySelector('.fv-feedback-entry:not(.fv-guide-entry)'), spare = feedback ? feedback.offsetWidth + apart : 0; // From 681 px up the switch has one place in every view, so the logo, then the feedback entry, leave before the switch moves down. const wide = !matchMedia('(max-width:680px)').matches, bare = room => wide && side >= room && side >= wanted; // Before the feedback entry, the phone entry steps aside. + // The Tutorial entry leaves last, only where the switch would otherwise drop to a second row (it stays in the other views). + const guide = actions.querySelector('.fv-guide-entry'), room = guide ? guide.offsetWidth + apart : 0; const lan = actions.querySelector('.fv-lan'), away = lan ? lan.offsetWidth + apart : 0; // The phone entry also steps aside rather than let the view switch drop to a second row. const fitsWithout = markWidth => side >= need - away && side - markWidth - spacing >= wanted; - const fit = fits(mark.offsetWidth) ? '' : fits(compact) ? 'compact' : away && fitsWithout(mark.offsetWidth) ? 'tight' : away && fitsWithout(compact) ? 'compact tight' : bare(need) ? 'bare' : away && bare(need - away) ? 'bare tight' : away && spare && bare(need - away - spare) ? 'bare tight lean' : spare && bare(need - spare) ? 'bare lean' + // Then the feedback entry steps aside too, rather than let the switch drop to a second row. + const fitsLean = markWidth => spare && side >= need - away - spare && side - markWidth - spacing >= wanted; + const fit = fits(mark.offsetWidth) ? '' : fits(compact) ? 'compact' : away && fitsWithout(mark.offsetWidth) ? 'tight' : away && fitsWithout(compact) ? 'compact tight' : away && fitsLean(mark.offsetWidth) ? 'tight lean' : away && fitsLean(compact) ? 'compact tight lean' : bare(need) ? 'bare' : away && bare(need - away) ? 'bare tight' : away && spare && bare(need - away - spare) ? 'bare tight lean' : away && spare && room && bare(need - away - spare - room) ? 'bare tight lean least' : spare && bare(need - spare) ? 'bare lean' : row(mark.offsetWidth) >= wanted ? 'wrap' : row(compact) >= wanted || !spare ? 'wrap compact' : 'wrap compact lean'; if (fit) header.dataset.fit = fit; } diff --git a/web/editor_library.js b/web/editor_library.js index ce6f383f..c0215c54 100644 --- a/web/editor_library.js +++ b/web/editor_library.js @@ -107,7 +107,7 @@ export function createEditorLibrary(host, {store, t, actions, state}) { plus.ondblclick = event => { event.preventDefault(); event.stopPropagation(); }; plus.onkeydown = event => { if (event.key === 'Enter' || event.key === ' ') insert(event); }; container.append(button, plus); - if (kind !== 'work' && row.state === 'failed' && row.reason !== 'disk_full') container.append(issueButton(t, () => ({where: LIBRARY_ISSUE, message: t('Cannot open', '打不开')}))); + if (kind !== 'work' && row.state === 'failed' && row.reason !== 'disk_full') container.append(issueButton(t, () => ({where: LIBRARY_ISSUE, message: t('Cannot open', '打不开'), kind: row.detail || null}))); return container; } function updateUsed() { diff --git a/web/editor_model.js b/web/editor_model.js index 3838930d..76b2a2c2 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -558,6 +558,14 @@ export function placeStems(record, id, sources) { parent.stem_volume = volume; parent.volume = 0; return finish(next); } +// A replaced voice: the part keeps its id, place and range and takes a file of the same length as its sound. +export function replaceSource(record, id, source) { + const next = copy(record), clip = find(next, id); + check(clip && trackOf(next, id).kind === 'audio' && clip.source && !clip.pending, 'missing'); + check(!trackOf(next, id).locked, 'locked'); + clip.source = sourceData(source); + return finish(next); +} export function move(record, id, start, {targetTrack} = {}) { if (isText(find(record, id))) return changeText(record, id, {start}); check(whole(start), 'range'); const selected = pair(record, id); check(selected.length, 'missing'); busy(selected); @@ -663,11 +671,30 @@ export function split(record, id, frame, {by = 'user'} = {}) { if (rights.length === 2) { rights[0].link = rights[1].id; rights[1].link = rights[0].id; } return finish(next); } +// The sound under a picture planned to follow it: the imported sound, not a clip of its own. +const soundMate = (record, clip) => { + // A placeholder of a replacement is held by its transaction; this one has none and borrows the sound's source. + const mate = clip.pending && clip.link && !(record.replacements || []).some(tx => [...tx.originals, ...tx.clips].some(t => t.clips.some(c => c.id === clip.id))) ? find(record, clip.link) : null; + return mate?.source?.kind === 'import' && mate.pending && clip.source && key(clip.source) === key(mate.source) && trackBy(record, 'a1').clips.includes(mate) ? mate : null; +}; export function remove(record, ids) { assertUnlocked(record, ids); check(Array.isArray(ids), 'id'); const selected = linkedIds(record, ids); busy(all(record).filter(clip => selected.has(clip.id))); - const next = copy(record); for (const track of next.tracks) track.clips = track.clips.filter(clip => !selected.has(clip.id)); + // Discarding a planned picture that follows imported sound leaves the sound as it was before the plan. + const kept = new Set(trackBy(record, 'v1').clips.filter(clip => selected.has(clip.id)).map(clip => soundMate(record, clip)?.id).filter(Boolean)); + const next = copy(record); + for (const track of next.tracks) track.clips = track.clips.filter(clip => !selected.has(clip.id) || kept.has(clip.id)); + if (kept.size) { + for (const id of kept) Object.assign(find(next, id), {link: null, part: null, text: '', pending: null, reason: null, job: null}); + const a1 = trackBy(next, 'a1'), plain = clip => kept.has(clip.id) || isSound(clip); + a1.clips.sort((a, b) => a.start - b.start); + for (let i = a1.clips.length - 1; i > 0; i--) { + const [a, b] = [a1.clips[i - 1], a1.clips[i]]; + if ((kept.has(a.id) || kept.has(b.id)) && plain(a) && plain(b) && end(a) === b.start && key(a.source) === key(b.source) + && a.in + a.frames === b.in && a.volume === b.volume && a.length === b.length) { a.frames += b.frames; a1.clips.splice(i, 1); } + } + } return finish(next); } export function closeGap(record, trackId, frame) { @@ -987,6 +1014,23 @@ function layoutPlanned(record, id, edge, frame, limits = {}) { } return next; } +// Imported sound with no picture of its own, on the first sound row. +const isSound = clip => clip.source?.kind === 'import' && !clip.link && !clip.pending; +// Whether [start, start + frames) is empty on the picture row and lies within one piece of imported sound. +export function soundRange(record, start, frames) { + if (!whole(start) || !whole(frames, 1) || trackBy(record, 'v1').clips.some(clip => overlaps(clip, start, frames))) return null; + const clips = trackBy(record, 'a1').clips.filter(clip => overlaps(clip, start, frames)); + if (!clips.length || !clips.every(isSound) || record.tracks.find(track => track.id === 'a2')?.clips.some(clip => overlaps(clip, start, frames))) return null; + return clips.length === 1 && clips[0].start <= start && end(clips[0]) >= start + frames ? clips[0] : null; +} +function cutSound(record, start, frames) { + let next = record; + for (const frame of [start, start + frames]) { + const clip = trackBy(next, 'a1').clips.find(item => item.start < frame && end(item) > frame); + if (clip) next = split(next, clip.id, frame); + } + return next; +} export function planRange(record, {start, frames, parts, target = 'both', text = '', by = 'user', maxSeconds = RULES.maxSeconds, maxFollowSeconds = maxSeconds}) { check(TEXT_BY.includes(by), 'text'); check(whole(start) && whole(frames, 1), 'range'); check(['both', 'video', 'audio'].includes(target), 'target'); @@ -994,14 +1038,21 @@ export function planRange(record, {start, frames, parts, target = 'both', text = check(typeof maxSeconds === 'number' && Number.isFinite(maxSeconds) && maxSeconds >= RULES.minSeconds, 'range'); check(target === 'both', 'unsupported'); if (!record.tracks.some(track => track.clips.some(clip => overlaps(clip, start, frames)))) frames = Math.max(RULES.minSeconds, Math.round(frames / FPS)) * FPS; + // A range with no picture over imported sound is drawn to follow that sound: the sound is cut at the range. + const ledBySound = !!soundRange(record, start, frames); + if (ledBySound) record = cutSound(record, start, frames); const videoHits = trackBy(record, 'v1').clips.filter(clip => overlaps(clip, start, frames)); const audioHits = trackBy(record, 'a1').clips.filter(clip => overlaps(clip, start, frames)); const otherHits = record.tracks.filter(track => track.id === 'a2' || track.id === 'a3').flatMap(track => track.clips.filter(clip => overlaps(clip, start, frames))); const hits = [...videoHits, ...audioHits]; + const followsSound = ledBySound && !videoHits.length && audioHits.length > 0 && audioHits.every(isSound); if (hits.length || otherHits.length) { check(hits.length > 0, 'unsupported'); - check(videoHits.length && audioHits.length === videoHits.length, 'unsupported'); + check(followsSound || videoHits.length && audioHits.length === videoHits.length, 'unsupported'); let boundary = start; + if (followsSound) { + for (const clip of audioHits) { check(clip.start === boundary && end(clip) <= start + frames && !otherHits.length, 'unsupported'); boundary = end(clip); } + } for (const clip of videoHits) { const mate = find(record, clip.link); check(clip.start === boundary && end(clip) <= start + frames && mate && audioHits.includes(mate) diff --git a/web/editor_player.js b/web/editor_player.js index b10d2bfa..f5a546c7 100644 --- a/web/editor_player.js +++ b/web/editor_player.js @@ -54,8 +54,8 @@ export function createEditorPlayer(host, transport, {state, sources, api, t, err let seekGeneration = 0, forcePosition = false, pictureView = null, processedSound = false; const chainSeconds = (chain, position) => Math.max(0, (position - chain.start) / 24 + (chain.offset || 0)); const playback = createPlaybackClock(); - const gains = new WeakMap(), volumePreview = new Map(); - const audioPreview = createAudioPreview({state, sources, api, volumePreview, readSamples: readAudioSamples, error, speech}); + const gains = new WeakMap(), volumePreview = new Map(), sourcePreview = new Map(); + const audioPreview = createAudioPreview({state, sources, api, volumePreview, sourcePreview, readSamples: readAudioSamples, error, speech}); const clipVolume = clip => volumePreview.get(clip?.id) ?? clip?.volume ?? 1; function volume(media, value) { // HTMLMediaElement stops at 1. A GainNode implements the record's 0–2 range. @@ -457,6 +457,7 @@ export function createEditorPlayer(host, transport, {state, sources, api, t, err setTextRenderer(callback) { textRenderer = callback; }, previewVolume(id, value) { volumePreview.set(id, value); render(); }, clearVolumePreview() { volumePreview.clear(); render(); }, + previewClipSource(id, source) { if (source) sourcePreview.set(id, source); else sourcePreview.delete(id); render(); }, async previewSource(source) { if (disposed) return; const previousSize = previewSize || {width: state.record.width, height: state.record.height}; diff --git a/web/editor_store.js b/web/editor_store.js index a3d79eaa..30974eb6 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -256,7 +256,8 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener activateProject, get uploading() { return uploads.size > 0; }, async deleteProject(id) { - await sessionWrites; + // A lease that could not be renewed must not stop the delete: the server checks the leases itself. + await sessionWrites.catch(() => {}); const response = await request('/freevideo/editor/project/delete', post({id, session: session.session})); if (!response.ok) throw await saveFailure(response); memoryProjects.delete(id); diff --git a/web/editor_tracks.js b/web/editor_tracks.js index 3bd93347..5ab3c8a9 100644 --- a/web/editor_tracks.js +++ b/web/editor_tracks.js @@ -576,7 +576,8 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = `${name}，第 ${index + 1} 段，${seconds} 秒，${status}${seamText}`)); // Pending replacements can retain their old media; state styling // overlays it until the model reports that the new media is done. - item.media.hidden = visual !== 'done'; + // The imported sound under a sound-led range is kept as it is, so its waveform stays in view while the picture is made. + item.media.hidden = visual !== 'done' && !(track.kind === 'audio' && clip.source?.kind === 'import' && key); if (track.kind === 'video') { updatePictureBadge(item.element, clip, record, clipPictureSize(clip, sources), t); actions.blockNote?.(item, {noted, index, record}); diff --git a/web/editor_voice.css b/web/editor_voice.css new file mode 100644 index 00000000..e670c91d --- /dev/null +++ b/web/editor_voice.css @@ -0,0 +1,37 @@ +/* Dubbing lines in the generation bar. */ +.fv-ed-genbar[data-voice=true]{flex-wrap:wrap;align-items:center} +.fv-ed-genbar[data-voice=true] .fv-ed-description-box,.fv-ed-genbar[data-voice=true] .fv-ed-first-frame{display:none} +.fv-ed-genbar[data-voice=true] .fv-ed-gen-hint{flex:1} +.fv-ed-genbar[data-voice=true] .fv-ed-generate{margin-left:auto} +.fv-ed-voice{order:-1;flex:1 1 100%;min-width:0;display:flex;flex-direction:column;gap:8px} +.fv-ed-voice[hidden]{display:none} +.fv-ed-voice-top{display:flex;align-items:center;gap:12px;min-height:28px;font-size:13px;color:var(--fv-muted)} +.fv-ed-voice-name{flex:1;min-width:0} +.fv-ed-voice-compare{display:flex;border:1px solid var(--fv-border);border-radius:6px;overflow:hidden} +.fv-ed-voice-compare[hidden]{display:none} +.fv-editor .fv-ed-voice-compare button{height:26px;min-height:26px;padding:0 12px;border:0;border-radius:0;background:transparent;color:var(--fv-muted);font-size:12px} +.fv-editor .fv-ed-voice-compare button[aria-pressed=true]{background:var(--fv-accent,#6db8fa);color:#0f1722} +.fv-ed-voice-lines{display:flex;flex-direction:column;gap:6px;max-height:calc(6 * 44px);overflow-y:auto} +.fv-ed-voice-line{display:flex;flex-direction:column;gap:2px} +.fv-ed-voice-main{display:flex;align-items:flex-start;gap:8px;position:relative;padding-left:8px} +.fv-ed-voice-mark{position:absolute;left:0;top:4px;bottom:4px;width:2px;border-radius:1px;background:transparent} +.fv-ed-voice-line[data-changed=true] .fv-ed-voice-mark,.fv-ed-voice-line[data-active=true] .fv-ed-voice-mark{background:var(--fv-accent,#6db8fa)} +.fv-ed-voice-line[data-active=true] .fv-ed-voice-mark{animation:fv-ed-voice-run 1.2s ease-in-out infinite} +@keyframes fv-ed-voice-run{50%{opacity:.35}} +@media (prefers-reduced-motion:reduce){.fv-ed-voice-line[data-active=true] .fv-ed-voice-mark{animation:none}} +.fv-editor .fv-ed-voice-time{flex:none;width:44px;height:36px;min-height:36px;padding:0;border:0;background:transparent;color:var(--fv-muted);font-size:13px;font-variant-numeric:tabular-nums;font-family:ui-monospace,monospace;text-align:left} +.fv-editor .fv-ed-voice textarea.fv-ed-voice-text{flex:1;min-width:0;resize:none;overflow:hidden;box-sizing:border-box;min-height:36px;max-height:none;line-height:20px;padding:8px;border:1px solid var(--fv-border);border-radius:6px;background:transparent;color:var(--fv-ink);font:inherit;font-size:13px;line-height:18px} +.fv-ed-voice-extra{display:flex;align-items:center;gap:10px;padding-left:60px;font-size:12px;color:var(--fv-muted)} +.fv-ed-voice-extra[hidden]{display:none} +.fv-ed-voice-over{color:var(--fv-warn,#e0a84a)} +.fv-ed-voice-old,.fv-ed-voice-short{color:var(--fv-muted)} +.fv-ed-voice-line[data-failed=true] .fv-ed-voice-mark{background:var(--fv-warn,#e0a84a)} +.fv-editor .fv-ed-voice-link{height:auto;min-height:0;padding:0;border:0;background:transparent;color:var(--fv-accent,#6db8fa);font-size:12px} +.fv-ed-voice-placeholder{height:36px;border-radius:6px;background:var(--fv-border);opacity:.4} +.fv-ed-voice-note{font-size:13px;color:var(--fv-muted)} +.fv-ed-voice-note[hidden]{display:none} +@media (max-width:640px){ + .fv-editor .fv-ed-voice textarea.fv-ed-voice-text{font-size:16px} + .fv-ed-voice-extra{padding-left:60px;flex-wrap:wrap} +} +.fv-ed-voice-note:has(.fv-issue-button){display:flex;flex-wrap:wrap;align-items:center;gap:6px 8px} diff --git a/web/editor_voice.js b/web/editor_voice.js new file mode 100644 index 00000000..94319ec2 --- /dev/null +++ b/web/editor_voice.js @@ -0,0 +1,216 @@ +// Dubbing lines through the generation bar: a stretch of speech chosen there shows its lines in place, each one +// editable; only the lines changed are dubbed again. The server (or a stand-in) recognises and dubs; this keeps what +// the user typed, the progress, the original/new choice for playback and the lines that did not fit. +import * as voice from './editor_voice_model.js'; +import {isNetworkTypeError, connectionLostText} from './browser_extension.js'; +import {issueButton} from './issue_button.js'; +import {repairAction} from './repair_hint.js'; + +const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; + +// What a failed run says: the lines the user typed are kept, whatever went wrong; the code never reaches the words. +export function failureCopy(error, t) { + if (isNetworkTypeError(error)) return {text: connectionLostText(t), report: false}; + switch (error?.code) { + case 'out_of_memory': return {report: true, text: t('Not enough memory to finish dubbing. The edited lines are kept. Close other programs using the GPU, then redub.', '显存或内存不足，配音未完成，所改台词已保留。请关闭其他占用显卡的程序后重新配音。')}; + case 'storage_unavailable': return {report: false, text: t('Could not write to disk, so dubbing did not finish. The edited lines are kept. Check the free disk space, then redub.', '无法写入磁盘，配音未完成，所改台词已保留。请检查磁盘空间后重新配音。')}; + case 'voice_model_missing': return {report: false, text: t('The dubbing model is not fully installed. ', '配音模型没有装全。') + t(...repairAction())}; + default: return {report: true, text: t('Dubbing did not finish. The edited lines are kept. Redub them to try again.', '配音未完成，所改台词已保留，请重新配音。')}; + } +} + +export function createEditorVoice({state, store, t, onChange = () => {}, place = async () => {}, available = () => true, playLine = () => {}, comparePlayback = () => {}}) { + const sessions = new Map(); + let disposed = false, placing = false; + const target = (selection = state.selection) => available() && state.record && voice.voiceTarget(state.record, selection); + const keyOf = found => `${found.clipId}:${found.start}:${found.end}`; + const session = (found = target()) => found ? sessions.get(keyOf(found)) || null : null; + const emit = () => { if (!disposed) onChange(); }; + + // Recognition starts by itself when a stretch of speech is chosen. + async function open(found = target()) { + if (!found) return null; + const key = keyOf(found); + let current = sessions.get(key); + if (current) return current; + current = {key, found, phase: 'finding', lines: [], texts: new Map(), results: new Map(), shortened: new Map(), compare: 'new', + active: null, index: 0, count: 0, remaining: null, failed: 0, revision: state.record.revision}; + sessions.set(key, current); emit(); + try { + const result = await store.lines({found, revision: current.revision}); + if (disposed || sessions.get(key) !== current) return current; + current.phase = result.speech && result.lines.length ? 'ready' : 'none'; + current.lines = result.lines; current.separated = result.separated; current.recognition = result.recognition; + current.rate = new Map(result.lines.map(line => [line.id, line.rate])); + } catch (value) { + // A failed recognition is not kept: choosing the stretch again asks again. + current.phase = 'error'; current.error = value; + if (sessions.get(key) === current) sessions.delete(key); + if (!disposed) onChange(); return current; + } + emit(); return current; + } + const failedCount = current => [...current.results.values()].filter(value => value.state === 'failed').length; + const changed = current => voice.changedLines(current.lines, current.texts); + function edit(id, text, current = session()) { + if (!current || current.phase === 'dubbing') return; + // What the last run said about this line no longer fits its new words. + current.texts.set(id, text); current.results.delete(id); current.failed = failedCount(current); current.failure = null; emit(); + } + async function redub(current = session()) { + if (!current || current.phase === 'dubbing' || current.phase === 'finding') return null; + const lines = changed(current); + current.phase = 'dubbing'; current.index = 0; current.count = lines.length; current.remaining = null; current.failed = 0; current.results = new Map(); current.notice = null; current.failure = null; emit(); + try { + const send = () => store.replace({found: current.found, revision: current.revision, recognition: current.recognition, + lines: current.lines.map(line => ({id: line.id, start: line.start, end: line.end, text: current.texts.get(line.id) ?? line.text, original_text: line.text})), + progress: info => { current.index = info.index; current.count = info.count || current.count; current.active = info.line || null; current.remaining = info.remaining_seconds ?? current.remaining; emit(); }}); + let result; + try { result = await send(); } catch (value) { + // The service no longer keeps the recognition (restart, age): recognise the same stretch again and put the typed words back by line id. + if (value?.code !== 'invalid_request') throw value; + const again = await store.lines({found: current.found, revision: current.revision}); + const ids = new Set(again.lines.map(line => line.id)); + if (again.lines.length !== current.lines.length || current.lines.some(line => !ids.has(line.id))) throw value; + current.lines = again.lines; current.recognition = again.recognition; + result = await send(); + } + if (disposed) return result; + // The service did not put anything into the project: every line was too long, or the stretch was edited meanwhile. + if (result?.placed === false && (result.reason || !result.apply)) { + current.results = voice.lineResults(result); current.failed = result.failed || 0; current.phase = 'ready'; current.active = null; current.sources = null; + current.notice = result.reason === 'project_changed' ? 'changed' : null; current.revision = state.record.revision; + emit(); return result; + } + current.notice = null; current.failure = null; + placing = true; + try { current.voiceClip = (await place(result))?.voiceClip || current.found.clipId; } finally { placing = false; } + current.results = voice.lineResults(result); current.failed = result.failed || 0; current.sources = result.sources; + for (const line of current.lines) if (current.results.get(line.id)?.state === 'placed') line.text = current.texts.get(line.id) ?? line.text; + current.compare = 'new'; current.active = null; current.phase = 'done'; current.revision = state.record.revision; + emit(); return result; + } catch (value) { + current.phase = 'ready'; current.active = null; current.error = value; + // Said once, in the panel beside the lines: no pop-up, no second copy in the bar. + current.failure = value?.code === 'cancelled' ? null : value; emit(); + return null; + } + } + function cancel(current = session()) { if (current?.phase === 'dubbing') { current.failure = null; store.cancel?.(); } } + function setCompare(value, current = session()) { if (!current?.sources) return; current.compare = value; comparePlayback(current.voiceClip || current.found.clipId, value === 'original' ? current.sources.original : null); emit(); } + // The model's rewrite of a line that did not fit; the original wording stays beside it until the user lets go of it. + async function shorten(id, current = session()) { + const line = current?.lines.find(value => value.id === id), result = current?.results.get(id); + if (!line || current.phase === 'dubbing' || current.shortened.get(id)?.busy) return; + const entry = {busy: true, original: current.texts.get(id) ?? line.text}; current.shortened.set(id, entry); current.failure = null; emit(); + try { + const out = await store.shorten({line: {id, text: entry.original, seconds: result?.seconds}, target_seconds: Math.max(.1, (line.end - line.start))}); + entry.busy = false; current.texts.set(id, out.text); + } catch (value) { current.shortened.delete(id); if (value?.code === 'shorten_unavailable') current.noShorten = true; } + emit(); + } + function restore(id, current = session()) { + const entry = current?.shortened.get(id); if (!entry || entry.busy || current.phase === 'dubbing') return; + current.texts.set(id, entry.original); current.shortened.delete(id); current.failure = null; emit(); + } + // Undo takes the new sound out again; the lines then are the old ones. + const stillPlaced = current => { + const clip = state.record.tracks.flatMap(track => track.clips).find(value => value.id === (current.voiceClip || current.found.clipId)); + return !!clip && `generated:${clip.source?.id}` === String(current.sources?.new); + }; + // The original is only heard while its stretch stays chosen; a changed project (undo included) makes the lines stale. + function clearCompare(current) { if (current.compare === 'original') { current.compare = 'new'; comparePlayback(current.voiceClip || current.found.clipId, null); } } + const unsubscribe = state.subscribe(event => { + const type = event.type ?? event; + const here = target() ? keyOf(target()) : null; + for (const [key, current] of [...sessions]) { + if (key !== here) clearCompare(current); + if (type === 'record' && !placing && current.phase === 'done' && !stillPlaced(current)) { clearCompare(current); sessions.delete(key); } + } + if (type === 'selection' || type === 'record') { const found = target(); if (found) open(found); emit(); } + }); + // The entry appears once the engine says it can dub; a stretch already chosen is looked at then. + function refresh() { const found = target(); if (found) open(found); emit(); } + return {refresh, target, session, open, edit, redub, cancel, setCompare, shorten, restore, playLine, + dispose() { for (const current of sessions.values()) clearCompare(current); disposed = true; unsubscribe(); sessions.clear(); }, changed}; +} + +// The lines in the bar: a row each, time on the left, the words themselves to edit. +export function createVoicePanel(host, controller, {t}) { + const link = el('link'); link.rel = 'stylesheet'; link.href = new URL('./editor_voice.css', import.meta.url).href; document.head.append(link); + const element = el('div', 'fv-ed-voice'), top = el('div', 'fv-ed-voice-top'), voiceName = el('span', 'fv-ed-voice-name', t('Lines', '台词')); + const compare = el('div', 'fv-ed-voice-compare'); compare.setAttribute('role', 'group'); + const choices = [['original', t('Original', '原声')], ['new', t('New', '新声')]].map(([value, label]) => { + const button = el('button', null, label); button.type = 'button'; button.dataset.value = value; + button.onclick = () => controller.setCompare(value); compare.append(button); return button; + }); + const list = el('div', 'fv-ed-voice-lines'), note = el('div', 'fv-ed-voice-note'); note.setAttribute('role', 'status'); + top.append(voiceName, compare); element.append(top, list, note); element.hidden = true; host.append(element); + const rows = new Map(); + function row(line) { + let entry = rows.get(line.id); if (entry) return entry; + const wrap = el('div', 'fv-ed-voice-line'), main = el('div', 'fv-ed-voice-main'), time = el('button', 'fv-ed-voice-time', voice.clockText(line.start)), + text = el('textarea', 'fv-ed-voice-text'), mark = el('span', 'fv-ed-voice-mark'), extra = el('div', 'fv-ed-voice-extra'); + time.type = 'button'; time.setAttribute('aria-label', t('Play this line', '播放这一句')); + text.rows = 1; text.value = line.text; text.setAttribute('aria-label', t('Line', '台词')); + time.onclick = () => controller.playLine(line); text.oninput = () => { grow(text); controller.edit(line.id, text.value); }; + text.onkeydown = event => { if (event.key === 'Enter' && !event.shiftKey && !event.isComposing) event.preventDefault(); }; + main.append(mark, time, text); wrap.append(main, extra); entry = {wrap, text, time, mark, extra}; rows.set(line.id, entry); return entry; + } + // The box is border-box: the border is added to what the words need. Measured again when the bar's width or its style sheet changes. + const grow = area => { area.style.height = '0px'; area.style.height = (area.scrollHeight + area.offsetHeight - area.clientHeight) + 'px'; }; + const regrow = () => { if (!element.hidden) list.querySelectorAll('textarea').forEach(grow); }; + let seen = 0; + const observer = typeof ResizeObserver === 'function' ? new ResizeObserver(() => { if (list.clientWidth !== seen) { seen = list.clientWidth; regrow(); } }) : null; + observer?.observe(list); + link.onload = regrow; + function update() { + const found = controller.target(), current = found && controller.session(found); + element.hidden = !found || !current; + if (current && ['ready', 'done', 'dubbing'].includes(current.phase)) list.dataset.tourId = 'voice-lines'; else delete list.dataset.tourId; + if (element.hidden) return; + const dubbing = current.phase === 'dubbing', ready = ['ready', 'done', 'dubbing'].includes(current.phase); + compare.hidden = !current.sources; + for (const button of choices) button.setAttribute('aria-pressed', String(button.dataset.value === current.compare)); + voiceName.hidden = !ready; + const failure = current.failure && failureCopy(current.failure, t); + note.replaceChildren(); + if (failure) { + note.append(failure.text); + if (failure.report) note.append(issueButton(t, () => ({where: ['Editor generation', '剪辑器生成'], message: failure.text, kind: current.failure?.code || null}))); + } else note.textContent = current.notice === 'changed' ? t('This part changed during dubbing, so the new voice was not placed. Redub it.', '这一段在配音期间有改动，新声未放入。请重新配音。') : current.phase === 'none' ? t('No lines were found in this part.', '这一段没有识别到台词。') : current.phase === 'done' && current.failed ? voice.failedText(current.failed, t) : ''; + note.hidden = !note.textContent; + if (current.phase === 'finding') { + list.replaceChildren(...[0, 1, 2].map(() => el('div', 'fv-ed-voice-placeholder'))); rows.clear(); return; + } + if (!ready) { list.replaceChildren(); rows.clear(); return; } + const changedIds = new Set(controller.changed(current).map(line => line.id)); + list.replaceChildren(...current.lines.map(line => { + const entry = row(line), result = current.results.get(line.id), short = current.shortened.get(line.id); + const value = current.texts.get(line.id) ?? line.text; + if (entry.text.value !== value) { entry.text.value = value; } + entry.text.disabled = dubbing; entry.text.readOnly = dubbing; + entry.wrap.dataset.changed = String(changedIds.has(line.id)); entry.wrap.dataset.active = String(current.active === line.id); entry.wrap.dataset.failed = String(result?.state === 'failed'); + const extra = []; + if (short?.busy) extra.push(el('span', 'fv-ed-voice-shortening', t('Shortening', '正在改短'))); + else if (short) { + const old = el('s', 'fv-ed-voice-old', short.original), undo = el('button', 'fv-ed-voice-link', t('Restore line', '恢复原句')); undo.type = 'button'; + undo.onclick = () => controller.restore(line.id); extra.push(old, undo); + } else if (result?.state === 'kept' && result.overrun > 0) { + const over = el('span', 'fv-ed-voice-over', voice.overrunText(result.overrun, t, !current.noShorten)); + extra.push(over); + if (!current.noShorten) { + const cut = el('button', 'fv-ed-voice-link', t('Shorten this line', '改短这句')); cut.type = 'button'; + cut.onclick = () => controller.shorten(line.id); extra.push(cut); + } + } else if (result?.state === 'placed' && result.short) extra.push(el('span', 'fv-ed-voice-short', t('Shorter than the original, with a pause after it', '比原句短，句尾留有空白'))); + entry.extra.replaceChildren(...extra); entry.extra.hidden = !extra.length; + return entry.wrap; + })); + regrow(); + // A result that kept the original voice scrolls to its first line that did not fit; the list may hide it. + const over = current.lines.find(line => current.results.get(line.id)?.state === 'kept' && current.results.get(line.id).overrun > 0); + if (current.scrollOver !== current.results && over) { current.scrollOver = current.results; rows.get(over.id)?.wrap.scrollIntoView({block: 'nearest'}); } + } + return {element, update, dispose() { observer?.disconnect(); element.remove(); link.remove(); }}; +} diff --git a/web/editor_voice_model.js b/web/editor_voice_model.js new file mode 100644 index 00000000..7211135e --- /dev/null +++ b/web/editor_voice_model.js @@ -0,0 +1,71 @@ +// Dubbing lines of a chosen stretch of speech: which lines the user changed, what the bar says about them and which +// ones did not fit. Pure helpers; the server does the recognition and the dubbing. +import {FPS} from './editor_model.js'; + +export const SPEECH_TRACKS = ['a1', 'a4']; +const NBSP = ' '; +const clean = value => String(value ?? '').trim(); +const end = clip => clip.start + clip.frames; + +// What a chosen stretch asks to be dubbed: a sound clip of the original sound (or its separated voice), whole or the +// part a stretch covers, in seconds from the clip's start. Null for anything else. +export function voiceTarget(record, selection) { + if (!selection) return null; + let track, clip, from, to; + if (selection.kind === 'clip') { + // A sound clip is chosen together with the picture it is linked to; exactly one speech clip may be among them. + // Choosing a picture clip (which brings its linked sound along) is not choosing the voice. + if (record.tracks.some(row => row.kind !== 'audio' && row.clips.some(value => selection.ids.includes(value.id)))) return null; + const found = record.tracks.filter(row => SPEECH_TRACKS.includes(row.id)).flatMap(row => row.clips.filter(value => selection.ids.includes(value.id)).map(value => [row, value])); + if (found.length !== 1 || !found[0][1].source) return null; + [track, clip] = found[0]; + from = 0; to = clip.frames; + } else if (selection.kind === 'range') { + if (selection.tracks?.length !== 1 || !SPEECH_TRACKS.includes(selection.tracks[0])) return null; + track = record.tracks.find(row => row.id === selection.tracks[0]); + clip = track?.clips.find(value => value.source && value.start <= selection.start && end(value) >= selection.start + selection.frames); + if (!clip) return null; + from = selection.start - clip.start; to = from + selection.frames; + } else return null; + return {clipId: clip.id, track: track.id, separated: track.id !== 'a1', start: from / FPS, end: to / FPS, clip}; +} + +// The lines whose text differs from the recognised one; spaces at the ends do not count. +export const changedLines = (lines, texts) => lines.filter(line => clean(texts.get(line.id) ?? line.text) !== clean(line.text)); + +// Seconds as the bar shows them: m:ss. +export const clockText = seconds => { const whole = Math.max(0, Math.floor(seconds)); return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, '0')}`; }; + +// The button names what it does; the time is not shown: the figure the service has is the new sentence's length, not the wait. +export function redubLabel(count, t) { + if (!count) return t('Redub', '重新配音'); + return t(`Redub ${count} ${count === 1 ? 'line' : 'lines'}`, `重新配音 ${count} 句`); +} + +// One line needs no number; the time only once the service has given one. +export const dubbingText = (index, count, seconds, t) => { + const time = Number.isFinite(seconds) ? ` · ${t(`about ${Math.max(1, Math.ceil(seconds))}${NBSP}s`, `约 ${Math.max(1, Math.ceil(seconds))}${NBSP}秒`)}` : ''; + return count > 1 ? t(`Dubbing · line ${index} of ${count}${time}`, `正在配音 · 第 ${index}/${count} 句${time}`) + : t(`Dubbing${time}`, `正在配音${time}`); +}; + +// "2.6 s too long. Original voice kept.": one decimal, never "0.0". Without a way to shorten it, the row says what to do. +export const overrunText = (seconds, t, canShorten = true) => { + const value = Math.max(0.1, Math.round(seconds * 10) / 10).toFixed(1); + return canShorten ? t(`${value}${NBSP}s too long. Original voice kept.`, `超出 ${value}${NBSP}秒，已保留原声`) + : t(`${value}${NBSP}s too long. Original voice kept. Shorten this line.`, `超出 ${value}${NBSP}秒，已保留原声。请缩短这句。`); +}; + +export const failedText = (count, t) => count === 1 + ? t('1 line could not be dubbed. Redub it.', '有 1 句无法配音，请重新配音。') + : t(`${count} lines could not be dubbed. Redub them.`, `有 ${count} 句无法配音，请重新配音。`); + +// What a result says about each line it dubbed: kept the original voice because it did not fit, failed, or placed. +export function lineResults(result) { + const out = new Map(); + // A line the run was stopped before is simply not dubbed: no mark. + for (const line of (result?.lines || []).filter(value => value.code !== 'cancelled')) out.set(line.id, line.code ? {state: 'failed', code: line.code} + : line.replaced === false ? {state: 'kept', overrun: line.overrun || 0} + : {state: 'placed', overrun: 0, short: line.warning === 'too_short'}); + return out; +} diff --git a/web/editor_voice_store.js b/web/editor_voice_store.js new file mode 100644 index 00000000..2a654f21 --- /dev/null +++ b/web/editor_voice_store.js @@ -0,0 +1,64 @@ +// The dubbing calls of the editor page, through the engine's agent routes like the sound separation: each one is a job +// read until it ends. The job leaves the project alone (lay: false); the page puts the new sound in itself, so its own +// saves never clash with the job and the change can be undone. +import {agentGet, agentPost, retryAgentPost} from './agent_session.js'; + +const ROOT = '/freevideo/agent/v1'; +const ACTIVE = ['waiting', 'queued', 'running', 'cancelling']; +const wait = ms => new Promise(resolve => setTimeout(resolve, ms)); +const detail = progress => { try { return JSON.parse(progress?.detail || 'null') || {}; } catch { return {}; } }; +const failure = (value, fallback) => Object.assign(new Error(value?.code || value?.error?.code || fallback), {code: value?.code || value?.error?.code || fallback}); + +export async function voiceAvailable(api) { + try { + const response = await agentGet(api, `${ROOT}/capabilities`); + return !!(response.ok && (await response.json())?.features?.['projects.voice']); + } catch { return false; } +} + +export function createVoiceStore({api, state, queue}) { + let running = null; + const uuid = () => globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`; + const path = name => `${ROOT}/projects/${encodeURIComponent(state.record.id)}/voice/${name}`; + const selection = found => ({clip_id: found.clipId, start: found.start, end: found.end}); + async function saved() { + // A change made a moment ago is queued after its own event; let it be seen before asking. + for (let round = 0; round < 3; round++) { await new Promise(resolve => setTimeout(resolve, 0)); await queue.flush(); if (!queue.dirty) break; } + if (queue.dirty || !state.record.id) throw failure(null, 'unsaved'); + } + async function job(name, body, progress = () => {}) { + await saved(); + const project = state.record.id, tool = running = {job: null}; + try { + const response = await retryAgentPost(api, path(name), {revision: state.record.revision, ...body, ref: uuid()}); + const answer = await response?.json().catch(() => null); + let current = answer?.job || answer?.jobs?.[0]; + if (!response?.ok || !current?.id) throw failure(answer, 'voice_failed'); + tool.job = current.id; + while (ACTIVE.includes(current.status)) { + progress({phase: current.progress?.phase, status: current.status, ...detail(current.progress)}); + const reply = await agentGet(api, `${ROOT}/jobs/${encodeURIComponent(tool.job)}?wait=25`); + if (!reply.ok) { await wait(1000); continue; } + current = await reply.json(); + if (state.record.id !== project) throw failure(null, 'project_changed'); + } + if (current.status === 'succeeded' || (current.status === 'cancelled' && current.result)) return current.result; + throw failure(current.error, current.status === 'cancelled' ? 'cancelled' : 'voice_failed'); + } finally { if (running === tool) running = null; } + } + async function sync(name, body) { + const response = await agentPost(api, path(name), body); + const answer = await response.json().catch(() => null); + if (!response.ok) throw failure(answer, 'voice_failed'); + return answer; + } + return { + lines: ({found, language}) => job('lines', {selection: selection(found), ...(language ? {language} : {})}), + estimate: ({lines}) => sync('estimate', {lines}), + replace: ({found, lines, recognition, progress}) => job('replace', {selection: selection(found), lines, recognition, lay: false}, info => { + if (info.phase === 'dubbing') progress({index: info.index, count: info.count, line: info.line, remaining_seconds: info.remaining_seconds}); + }), + shorten: ({line, target_seconds}) => sync('shorten', {line, target_seconds}), + cancel: () => { if (running?.job) agentPost(api, `${ROOT}/jobs/${encodeURIComponent(running.job)}/cancel`, {}).catch(() => {}); }, + }; +} diff --git a/web/feature_tour.css b/web/feature_tour.css index fc646aa6..c586682c 100644 --- a/web/feature_tour.css +++ b/web/feature_tour.css @@ -43,3 +43,5 @@ .fv-feature-tour .fv-tour-close{position:relative;font-size:0} .fv-tour-close::before,.fv-tour-close::after{content:'';position:absolute;left:10px;top:17px;width:16px;height:1px;background:currentColor;transform:rotate(45deg)} .fv-tour-close::after{transform:rotate(-45deg)} +.fv-tour-book{background:none;border:0;padding:6px 4px;color:inherit;font:inherit;text-decoration:underline;opacity:.75;cursor:pointer;flex:none;margin-right:auto;margin-left:12px}.fv-tour-book:hover,.fv-tour-book:focus-visible{opacity:1} +@media (max-width:680px){.fv-tour-book{margin:0;align-self:center}} diff --git a/web/feature_tour.js b/web/feature_tour.js index b54fcec1..cf65dce3 100644 --- a/web/feature_tour.js +++ b/web/feature_tour.js @@ -3,9 +3,12 @@ import {readGuideRecord, saveGuideRecord, guideRecord} from './first_run_guide.j import {isolateDialogKeys, listenDialogKey, routeDialogKey} from './dialog_keys.js'; export const TOUR_TEXT = { - en: {examples: 'Feature examples', use: 'Use this example', next: 'Next', done: 'Done', close: 'Close'}, - zh: {examples: '功能示例', use: '用这个示例', next: '下一张', done: '完成', close: '关闭'}, + en: {examples: 'Feature examples', use: 'Use this example', next: 'Next', goto: 'Take me there', book: 'Read the tutorial', done: 'Done', close: 'Close'}, + zh: {examples: '功能示例', use: '用这个示例', next: '下一张', goto: '带我去', book: '看教程', done: '完成', close: '关闭'}, }; +// The chapter of the tutorial book that teaches the same thing as the card. +const BOOK_CHAPTER = {'first-video': 'quick-start', 'text-to-video': 'text-to-video', 'first-last-frame': 'first-last-frame', 'quality-tiers': 'quality-tiers', + 'reference-image': 'reference-video', 'replace-person': 'replace-person', character: 'cast-sheet', 'edit-view': 'editor', extend: 'extend-video', 'two-pass': 'refine', autopilot: 'autopilot'}; const asset = name => new URL(`./assets/tour/${name}`, import.meta.url).href; const el = (tag, cls, text) => { const node = document.createElement(tag); if (cls) node.className = cls; if (text != null) node.textContent = text; return node; }; const contexts = new Map(); @@ -160,7 +163,14 @@ export async function openFeatureTour({api = contextFor()?.api, cn = contextFor( const use = el('button', 'fv-tour-use fv-primary', text.use); const usable = card.tryit.status !== 'pending-assets' && contextFor()?.canTry?.(card.tryit) !== false; use.onclick = async () => { await end(); await tryIt(card.id); }; - usable ? actions.append(next, use) : actions.append(next); footer.append(dots, actions); content.append(footer); + if (card.goto?.length) { + const go = el('button', 'fv-tour-goto', text.goto); + go.onclick = async () => { await end(); const {guideTo} = await import('./tour_goto.js'); for (const id of card.goto) if (await guideTo(id)) break; }; + actions.append(go); + } + const chapter = BOOK_CHAPTER[card.id], read = chapter ? el('button', 'fv-tour-book', text.book) : null; + if (read) read.onclick = async () => { await end(); const {openTutorialBook} = await import('./tutorial_book.js'); openTutorialBook({chapter}); }; + usable ? actions.append(next, use) : actions.append(next); footer.append(dots, actions); if (read) dots.after(read); content.append(footer); for (const video of videos) video.onerror = () => { stop(); const failed = loaded.splice(index, 1); release(failed); if (!loaded.length) { end(); return; } diff --git a/web/feature_tour_content.js b/web/feature_tour_content.js index 25025416..d0e5a140 100644 --- a/web/feature_tour_content.js +++ b/web/feature_tour_content.js @@ -18,6 +18,12 @@ export const FEATURE_TOUR = [ {assets: [{file: 'fox-sheet.webp', role: 'reference'}], prompt: ['The fox in {image1} walks through a sunlit forest.', '{image1} 中的狐狸走过洒满阳光的森林。'], settings: {}}, {sequence: [clip('c5-forest-s1'), clip('c5-snow-s1')]}), card('replace-person', copy('Replace person', '替换人物'), copy('Put the person from an image into a video.', '把视频里的人换成图中的人物。'), [{src: 'c10-source.webp'}, {src: 'c10-person.webp'}], clip('c10-s1'), {status: 'pending-assets'}, {trigger: 'media', added: '0.4.28'}), + card('first-video', copy('Make your first video', '做出第一条视频'), copy('Write a description, generate, and find the video in Creations.', '写一段描述，生成，再到作品中找到它。'), + [{text: copy(...CASTLE_PROMPT)}], clip('c11-s1'), {assets: [], prompt: CASTLE_PROMPT, settings: {}}, {added: '0.5.0', goto: ['prompt-box', 'generate-btn', 'works-entry']}), + card('character', copy('Characters', '角色'), copy('Save a character once, then mention it with @ in any description.', '角色保存一次，之后在任意描述中用 @ 引用。'), + [{text: copy('@Leo walks along the shore.', '@小林 走在海边。')}], clip('c12-s1'), {status: 'pending-assets'}, {trigger: 'media', added: '0.5.0', goto: ['cast-add', 'prompt-box']}), + card('edit-view', copy('Edit view', '剪辑视图'), copy('Arrange clips on the timeline and adjust picture, sound and speed.', '在时间线上排列片段，调整画面、声音和速度。'), + [{src: 'c12-s1.webp'}], clip('c13-s1'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', added: '0.5.0', goto: ['editor-timeline', 'editor-inspector']}), card('extend', copy('Extend', '接着生成'), copy('Continue from where the last clip ends.', '从上一段的结尾接着生成下一段。'), [{src: 'c6-s1.webp'}], clip('c6-s1'), {status: 'pending-assets', view: 'editor'}, {trigger: 'editor', previewRange: [2, 8]}), card('two-pass', copy('Two-pass', '二次采样'), copy('The first pass sets the composition; the second adds detail.', '第一遍定构图，第二遍补细节。'), diff --git a/web/feedback.css b/web/feedback.css index effd5e4e..7890c83c 100644 --- a/web/feedback.css +++ b/web/feedback.css @@ -9,8 +9,11 @@ /* Keep the entry beside Settings; compact headers use the same labelled button with a line icon. */ .fv-header-actions>button{white-space:nowrap} .fv-feedback-entry>svg{display:none;flex:none;width:16px;height:16px;fill:none;stroke:currentColor;stroke-width:1.4;stroke-linecap:round;stroke-linejoin:round} +@media(max-width:1500px){.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-feedback-entry>svg{display:block}.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-feedback-label{display:none}} +@media(max-width:1040px){.fv-autopilot .fv-header:has(.fv-autopilot-now:not([hidden])) .fv-lan{display:none}} @media(max-width:1040px){.fv-autopilot .fv-feedback-entry>svg{display:block}.fv-autopilot .fv-feedback-label{display:none}} +@media(max-width:1040px){.fv-editor .fv-guide-entry>svg{display:block}.fv-editor .fv-guide-entry .fv-feedback-label{display:none}} @media(max-width:800px){.fv-editor .fv-feedback-entry>svg{display:block}.fv-editor .fv-feedback-label{display:none}} -@media(max-width:680px){.fv-studio .fv-feedback-entry>svg{display:block}.fv-studio .fv-feedback-label{display:none}} +@media(max-width:680px){.fv-studio .fv-feedback-entry>svg,.fv-view-navigation .fv-feedback-entry>svg{display:block}.fv-studio .fv-feedback-label,.fv-view-navigation .fv-feedback-label{display:none}} /* On a narrow screen the line gets the whole width above the button. */ @media(max-width:480px){.fv-feedback .fv-feedback-actions{flex-wrap:wrap;row-gap:10px}.fv-feedback .fv-feedback-status{flex:1 0 100%}.fv-feedback .fv-feedback-status:empty{display:none}} diff --git a/web/feedback.js b/web/feedback.js index b4aba8fa..32c5ebe9 100644 --- a/web/feedback.js +++ b/web/feedback.js @@ -228,3 +228,17 @@ export function feedbackButton(t, className = 'fv-quiet') { button.onclick = () => openFeedback(t); return button; } + +// The tutorial entry, left of Feedback: same style, icon only where Feedback is. +export function guideButton(t, className = 'fv-quiet') { + const label = t('Tutorial', '教程'); + const button = el('button', `${className} fv-feedback-entry fv-guide-entry`); button.type = 'button'; + button.setAttribute('aria-label', label); button.title = label; + const icon = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + icon.setAttribute('viewBox', '0 0 16 16'); icon.setAttribute('aria-hidden', 'true'); icon.setAttribute('focusable', 'false'); + const path = document.createElementNS(icon.namespaceURI, 'path'); + path.setAttribute('d', 'M2.5 3.2c1.9-.5 3.7-.3 5.5.8v9c-1.8-1.1-3.6-1.3-5.5-.8ZM13.5 3.2c-1.9-.5-3.7-.3-5.5.8v9c1.8-1.1 3.6-1.3 5.5-.8Z'); + icon.append(path); button.append(icon, el('span', 'fv-feedback-label', label)); + button.onclick = () => import('./tutorial_book.js').then(({openTutorialBook}) => openTutorialBook({chapter: 'quick-start'})).catch(() => {}); + return button; +} diff --git a/web/health.js b/web/health.js index 1e25a644..3a515a49 100644 --- a/web/health.js +++ b/web/health.js @@ -14,12 +14,12 @@ const t = (en, zh) => cn ? zh : en; const launchVisit = query.get('freevideo') === 'launch'; // Every script this folder ships; a test keeps the list in step with web/*.js. export const MODULES = ['audio_original.js', 'chat_data.js', 'phone_model.js', 'phone.js', 'lan_access.js', 'qrcodegen.js', 'agent_activity.js', 'agent_confirm_settings.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'chat_attachments.js', 'autopilot_conversations.js', 'browser_extension.js', 'caret_rect.js', 'chain_support.js', 'chain_rules.js', 'timed_prompt.js', 'capability_view.js', 'cast_generate.js', 'cast_mention.js', 'cast_overlay.js', 'cast_source.js', 'compatibility.js', 'entry_showcase.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', - 'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_duration.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_stems.js', 'editor_record.js', 'editor_record_model.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_music.js', 'editor_music_model.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js', + 'editor.js', 'editor_actions.js', 'editor_lifecycle.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_expand.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_media_references.js', 'editor_references.js', 'editor_genbar.js', 'editor_duration.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_stems.js', 'editor_record.js', 'editor_record_model.js', 'editor_voice.js', 'editor_voice_model.js', 'editor_voice_store.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_music.js', 'editor_music_model.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_failure.js', 'editor_header_fit.js', 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_toolbar_more.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'done_notify.js', 'failure_issue.js', 'feedback.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'image_generation.js', 'image_lora.js', 'launch_visit.js', 'launcher.js', 'length_plan.js', 'lora_triggers.js', 'lora_check.js', 'library.js', 'motion.js', 'native_errors.js', 'translate_guard.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'prompt_sections.js', 'prompt_terms.js', 'request_assets.js', 'repair_hint.js', 'report_issue.js', 'issue_button.js', 'studio_recent.js', 'task_groups.js', 'result_actions.js', 'result_reuse.js', 'sampling_effort.js', 'pass_mode.js','setup.js', 'share.js', 'share_layout.js', 'studio.js', 'studio_chain.js', 'studio_extend.js', 'studio_length.js', 'first_frame_row.js', 'replace_person.js', 'replace_state.js', 'optional_inputs.js', 'other_usage.js', 'studio_queue.js', 'studio_size.js', 'toolbar_position.js', 'updates.js', 'view_navigation.js', 'wordmark.js', 'workspace_restore.js', 'cast_library.js', 'media_upload.js', 'whitebox_encode.js', 'whitebox_mp4.js', 'whitebox_recording.js', 'whitebox_render.js', 'whitebox_scenes.js', 'whitebox_worker.js', - 'empty_prompt_notice.js', 'feature_tour.js', 'feature_tour_content.js', 'howto.js', 'swap_lora.js', 'tour_try.js', 'tutorial_book.js', 'tutorial_book_content.js']; + 'empty_prompt_notice.js', 'feature_tour.js', 'feature_tour_content.js', 'howto.js', 'swap_lora.js', 'tour_goto.js', 'tour_try.js', 'tutorial_book.js', 'tutorial_book_content.js']; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); let visibleAt = 0; async function waitForVisible() { diff --git a/web/studio.js b/web/studio.js index 93ff5bab..6b29f7de 100644 --- a/web/studio.js +++ b/web/studio.js @@ -6,7 +6,7 @@ import {showEmptyPrompt, dismissEmptyPrompt} from './empty_prompt_notice.js'; import { app } from '../../scripts/app.js'; import { api } from '../../scripts/api.js'; import { openSetup } from './setup.js'; -import { feedbackButton, feedbackIssueURL } from './feedback.js'; +import { feedbackButton, guideButton, feedbackIssueURL } from './feedback.js'; import { wordmark } from './wordmark.js'; import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, keepSoundText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { viewSwitch, viewChanged, agentTabReady, openAutopilot } from './view_navigation.js'; @@ -667,7 +667,7 @@ export function openStudio(node, {extendFrom = null} = {}) { const agents = createAgentActivity(t, api, dialog); tools.append(agents.element); cleanup.push(agents.dispose); const phoneAccess = createLanAccess(t, api); tools.append(phoneAccess.element); cleanup.push(phoneAccess.dispose); const versionInfo = createVersionInfo(cn); tools.append(versionInfo.element); cleanup.push(versionInfo.dispose); - tools.append(feedbackButton(t), button(t('Settings', '设置'), openSetup, 'fv-quiet')); + tools.append(guideButton(t), feedbackButton(t), button(t('Settings', '设置'), openSetup, 'fv-quiet')); // The Agent's page is the fourth tab, shown where the Agent interface is on; the guide's last step circles it. agentTabReady(agentInterfaceState(api), () => agentInterfaceState(api)); // The length hint links to the same two views, through the same actions. @@ -695,7 +695,7 @@ export function openStudio(node, {extendFrom = null} = {}) { const untouched = () => Boolean(starter) && value(node, 'text') === starter; const shown = text => starter && text === starter ? '' : String(text || ''); const prompt = el('textarea'); prompt.value = shown(value(node, 'text')); prompt.setAttribute('aria-label', t('Prompt', '提示词')); - prompt.dataset.fvGuide = 'describe'; + prompt.dataset.fvGuide = 'describe'; prompt.dataset.tourId = 'prompt-box'; const placeholder = () => { prompt.placeholder = extension.source ? extendText(t, 'prompt') : untouched() ? starterShown : t('Describe the scene… Type @ to reference media', '描述画面… 输入 @ 引用素材'); }; placeholder(); prompt.disabled = linked(node, 'text'); prompt.oninput = () => { set(node, 'text', prompt.value); placeholder(); }; @@ -1270,10 +1270,10 @@ export function openStudio(node, {extendFrom = null} = {}) { seedFields.append(field(t('Seed', '种子'), seed), field(t('After generation', '生成后'), mode)); advanced.append(seedFields); const previewHead = el('div', null, 'fv-preview-head'); previewHead.append(el('strong', t('Preview', '预览'))); const previewTools = el('div', null, 'fv-preview-tools'); - previewTools.append(button(t('Creations', '作品'), () => { + const worksButton = button(t('Creations', '作品'), () => { stage.querySelector('video')?.pause(); openLibrary(t, {upscale: value => node.freevideoUpscaleResult(value), extend: (saved, options) => openExtendEditor(saved, node, {studio: dialog, ...options})}); - })); previewHead.append(previewTools); output.append(previewHead); + }); worksButton.dataset.tourId = 'works-entry'; previewTools.append(worksButton); previewHead.append(previewTools); output.append(previewHead); const previewSpace = el('div', null, 'fv-preview-space'); output.append(previewSpace); // The preview keeps the size it has once the list is in; an opened part grows the column, which scrolls. const partList = createChainPartList(t, (block, text) => chainController.setText(block, composePartText(chainController.state, block, text)), { @@ -1653,7 +1653,7 @@ export function openStudio(node, {extendFrom = null} = {}) { }; runOptions.append(optionsToggle, optionsPanel); const generate = button('', submitDraft, 'fv-primary'); - generate.dataset.fvGuide = 'generate'; + generate.dataset.fvGuide = 'generate'; generate.dataset.tourId = 'generate-btn'; runrow.append(generate, runOptions); runButtonReady = true; const queueActions = el('div', null, 'fv-queue-actions'); diff --git a/web/tour_goto.css b/web/tour_goto.css new file mode 100644 index 00000000..d2b1b8e3 --- /dev/null +++ b/web/tour_goto.css @@ -0,0 +1,4 @@ +.fv-tour-ring{position:fixed;z-index:2147483000;box-sizing:border-box;border:2px solid var(--fv-accent,#6db8fa);border-radius:var(--fv-r-md,10px);pointer-events:none;box-shadow:0 0 0 0 color-mix(in srgb,var(--fv-accent,#6db8fa) 25%,transparent);animation:fv-tour-ring-pulse .4s ease-out 1 forwards;transition:opacity .3s} +.fv-tour-ring-out{opacity:0} +@keyframes fv-tour-ring-pulse{to{box-shadow:0 0 0 6px color-mix(in srgb,var(--fv-accent,#6db8fa) 25%,transparent)}} +@media(prefers-reduced-motion:reduce){.fv-tour-ring{animation:none;box-shadow:0 0 0 6px color-mix(in srgb,var(--fv-accent,#6db8fa) 25%,transparent);transition:none}} diff --git a/web/tour_goto.js b/web/tour_goto.js new file mode 100644 index 00000000..d1e5056a --- /dev/null +++ b/web/tour_goto.js @@ -0,0 +1,60 @@ +// "Take me there": ring the real control a tutorial scene is about. Targets carry data-tour-id; +// a collapsed section or a hidden panel is opened first, and a missing target only opens the view. +if (!document.head.querySelector('link[href$="tour_goto.css"]')) { + const link = document.createElement('link'); link.rel = 'stylesheet'; link.href = new URL('./tour_goto.css', import.meta.url).href; + document.head.append(link); +} +const RING_MS = 3000, FADE_MS = 300; +let current = null; +const visible = node => { const r = node?.getBoundingClientRect(); return !!r && r.width > 0 && r.height > 0; }; +const find = id => document.querySelector(`[data-tour-id="${CSS.escape(id)}"]`); +function reveal(node) { + for (let parent = node.parentElement; parent; parent = parent.parentElement) if (parent.tagName === 'DETAILS' && !parent.open) parent.open = true; +} +export function clearTourRing() { current?.(); } +/** Resolve with true when the ring was drawn, false when the target is absent or hidden. */ +export async function guideTo(id) { + clearTourRing(); + let node = find(id); + if (!node) return false; + reveal(node); + if (!visible(node)) { + // A narrow layout keeps some panels behind a toggle that names the panel it opens. + document.querySelector(`[data-tour-toggle="${CSS.escape(id)}"]`)?.click(); + await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); + node = find(id); + if (!node || !visible(node)) return false; + } + const reduced = matchMedia('(prefers-reduced-motion: reduce)').matches; + node.scrollIntoView({block: 'center', inline: 'nearest', behavior: reduced ? 'auto' : 'smooth'}); + await new Promise(resolve => setTimeout(resolve, reduced ? 0 : 320)); + if (!visible(node)) return false; + const ring = document.createElement('div'); ring.className = 'fv-tour-ring'; ring.setAttribute('aria-hidden', 'true'); + // A transformed dialog becomes the containing block of fixed children, so the ring corrects by what it measures. + const place = () => { + const r = node.getBoundingClientRect(); + Object.assign(ring.style, {left: `${r.left - 4}px`, top: `${r.top - 4}px`, width: `${r.width + 8}px`, height: `${r.height + 8}px`}); + if (!ring.isConnected) return; + const g = ring.getBoundingClientRect(); + Object.assign(ring.style, {left: `${r.left - 4 - (g.left - (r.left - 4))}px`, top: `${r.top - 4 - (g.top - (r.top - 4))}px`}); + }; + // Smooth scrolling and a section that just opened can still be moving: wait until the target holds still. + for (let last = null, i = 0; i < 30; i++) { + const r = node.getBoundingClientRect(), key = `${r.top}|${r.left}`; + if (key === last) break; last = key; await new Promise(resolve => requestAnimationFrame(resolve)); + } + (node.closest('dialog[open]') || document.body).append(ring); place(); + const follow = () => { if (node.isConnected) place(); else done(); }; + addEventListener('scroll', follow, true); addEventListener('resize', follow); + let timer = 0, gone = false; + function done() { + if (gone) return; gone = true; clearTimeout(timer); + removeEventListener('scroll', follow, true); removeEventListener('resize', follow); + document.removeEventListener('pointerdown', done, true); + ring.classList.add('fv-tour-ring-out'); setTimeout(() => ring.remove(), reduced ? 0 : FADE_MS); + if (current === done) current = null; + } + current = done; timer = setTimeout(done, RING_MS); + document.addEventListener('pointerdown', done, true); + return true; +} diff --git a/web/tutorial_book.js b/web/tutorial_book.js index 425f4811..98cad791 100644 --- a/web/tutorial_book.js +++ b/web/tutorial_book.js @@ -1,8 +1,8 @@ import {TUTORIAL_CHAPTERS} from './tutorial_book_content.js'; const COPY = { - zh: {book: '教程', contents: '目录', close: '关闭', limits: '限制', start: '开始演示', tryIt: '试一试'}, - en: {book: 'Tutorial', contents: 'Contents', close: 'Close', limits: 'Limits', start: 'Start walkthrough', tryIt: 'Try it'}, + zh: {book: '教程', contents: '目录', close: '关闭', limits: '限制', start: '开始演示', tryIt: '试一试', goto: '带我去'}, + en: {book: 'Tutorial', contents: 'Contents', close: 'Close', limits: 'Limits', start: 'Start walkthrough', tryIt: 'Try it', goto: 'Take me there'}, }; const el = (tag, cls, text) => { const node = document.createElement(tag); @@ -118,7 +118,16 @@ export function openTutorialBook({chapter = TUTORIAL_CHAPTERS[0].id} = {}) { window.freevideoHowTo.tryIt(item.example, {chapter: item.id}); }; } - article.append(steps, ...(tryButton ? [tryButton] : []), start, limits); + let gotoButton = null; + if (item.goto?.length) { + gotoButton = el('button', 'fv-book-start fv-book-goto', text.goto); gotoButton.type = 'button'; + gotoButton.onclick = async () => { + dialog.close(); + const {guideTo} = await import('./tour_goto.js'); + for (const id of item.goto) if (await guideTo(id)) break; + }; + } + article.append(steps, ...(tryButton ? [tryButton] : []), ...(gotoButton ? [gotoButton] : []), start, limits); syncHowTo(); syncTry(); article.scrollTop = 0; diff --git a/web/tutorial_book_content.js b/web/tutorial_book_content.js index 2642c276..df730df5 100644 --- a/web/tutorial_book_content.js +++ b/web/tutorial_book_content.js @@ -3,6 +3,7 @@ export const TUTORIAL_CHAPTERS = [ { id: 'quick-start', + goto: ['prompt-box', 'generate-btn'], title: {zh: '做出第一条视频', en: 'Make your first video'}, summary: {zh: '只写一句描述，其余使用默认设置。', en: 'Write one sentence and keep the default settings.'}, media: { @@ -26,6 +27,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'text-to-video', + goto: ['prompt-box'], title: {zh: '文字生成视频', en: 'Make a video from text'}, summary: {zh: '用文字描述镜头，并选择画幅、清晰度和时长。', en: 'Describe the shot, then choose the shape, resolution and length.'}, media: { @@ -118,6 +120,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'cast-sheet', + goto: ['cast-add'], title: {zh: '角色库和角色设定图', en: 'Characters and character sheets'}, summary: {zh: '把人物保存为角色，之后在描述中用 @ 引用。', en: 'Save a person as a character, then mention it with @ in later prompts.'}, media: { @@ -274,6 +277,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'editor', + goto: ['editor-timeline'], title: {zh: '剪辑视图：裁剪、描述、接续', en: 'The Edit view: crop, describe, continue'}, summary: {zh: '在时间线上选中一段，描述它，并接着生成。', en: 'Select a part on the timeline, describe it and continue.'}, media: { @@ -338,6 +342,7 @@ export const TUTORIAL_CHAPTERS = [ }, { id: 'library-export', + goto: ['works-entry'], title: {zh: '作品库和导出', en: 'Creations and export'}, summary: {zh: '查找生成过的视频，并导出为文件。', en: 'Find earlier videos and export them as files.'}, media: { diff --git a/web/view_navigation.css b/web/view_navigation.css index 1507acca..7f6632cf 100644 --- a/web/view_navigation.css +++ b/web/view_navigation.css @@ -7,7 +7,7 @@ .fv-view-switch button[hidden],.fv-studio .fv-view-switch button[hidden]{display:none} .fv-view-navigation .fv-view-settings{font-size:12px;padding:6px} .fv-studio .fv-header:has(.fv-view-switch){grid-template-columns:1fr auto 1fr}.fv-studio .fv-header>.fv-header-actions{justify-self:end;min-width:0}.fv-studio .fv-view-switch button{padding:5px 14px;min-height:var(--fv-h-sm)} -@media(max-width:680px){.fv-view-navigation{top:68px;max-width:calc(100vw - 16px)}.fv-view-row{gap:6px}.fv-view-row>.fv-wordmark{width:90px}.fv-view-switch button{padding:7px 10px}.fv-studio .fv-header:has(.fv-view-switch){grid-template-columns:1fr auto 1fr;gap:8px}.fv-studio .fv-header:has(.fv-view-switch) .fv-wordmark{width:90px}.fv-studio .fv-view-switch{justify-self:center}.fv-studio .fv-view-switch button{padding:6px 9px}} +@media(max-width:680px){.fv-view-navigation{top:68px;max-width:calc(100vw - 16px)}.fv-view-navigation .fv-view-settings{padding:6px 3px}.fv-view-row{gap:4px}.fv-view-row>.fv-wordmark{width:90px}.fv-view-switch button{padding:7px 10px}.fv-studio .fv-header:has(.fv-view-switch){grid-template-columns:1fr auto 1fr;gap:8px}.fv-studio .fv-header:has(.fv-view-switch) .fv-wordmark{width:90px}.fv-studio .fv-view-switch{justify-self:center}.fv-studio .fv-view-switch button{padding:6px 9px}} .fv-view-navigation{animation:fv-panel-enter .25s ease} @media(max-width:380px){.fv-view-row>.fv-wordmark{width:76px}.fv-view-navigation .fv-view-switch button{font-size:12px;padding:7px 8px}} diff --git a/web/view_navigation.js b/web/view_navigation.js index 1d3f84f0..344205a2 100644 --- a/web/view_navigation.js +++ b/web/view_navigation.js @@ -2,7 +2,7 @@ import {createLanAccess} from './lan_access.js'; import {api} from '../../scripts/api.js'; import { app } from '../../scripts/app.js'; import { openSetup } from './setup.js'; -import { feedbackButton } from './feedback.js'; +import { feedbackButton, guideButton } from './feedback.js'; import { wordmark } from './wordmark.js'; import { createUpdateNotice, createVersionInfo } from './updates.js'; import { makeMovable } from './toolbar_position.js'; @@ -169,7 +169,7 @@ export function installNavigation(openStudio, agent = false, {openWorkflow = nul const handle = await openEditorView(node); if (created) handle?.notify(t('Created an editing project for you.', '已新建一个剪辑工程')); return handle; - }, true), () => choose(node => openAutopilot(node), true)), createLanAccess(t, api).element, createVersionInfo(cn).element, feedbackButton(t, 'fv-view-settings'), settings); toolbar.append(row); + }, true), () => choose(node => openAutopilot(node), true)), createLanAccess(t, api).element, createVersionInfo(cn).element, guideButton(t, 'fv-view-settings'), feedbackButton(t, 'fv-view-settings'), settings); toolbar.append(row); toolbar.append(createUpdateNotice(cn).element); document.body.append(toolbar); refreshNavigation(); makeMovable(toolbar);