diff --git a/.github/workflows/model_validation.yml b/.github/workflows/model_validation.yml new file mode 100644 index 00000000..33234f2f --- /dev/null +++ b/.github/workflows/model_validation.yml @@ -0,0 +1,71 @@ +name: Remote Model Validation + +on: + workflow_dispatch: + inputs: + model_id: + description: "Optional single model id from model_registry.json" + required: false + type: string + schedule: + - cron: "0 8 * * 1" + +concurrency: + group: remote-model-validation + cancel-in-progress: false + +env: + BUILD_TYPE: Release + BUILD_DIR: Builds + DISPLAY: :0 + HARP_MODEL_VALIDATION_TIMEOUT_MS: 180000 + HARP_MODEL_VALIDATION_RETRIES: 1 + HARP_MODEL_VALIDATION_RETRY_DELAY_MS: 30000 + HARP_MODEL_VALIDATION_REPORT_DIR: artifacts/model_validation/remote + HARP_HUGGINGFACE_TOKEN: ${{ secrets.HARP_HUGGINGFACE_TOKEN }} + HARP_STABILITY_API_KEY: ${{ secrets.HARP_STABILITY_API_KEY }} + +jobs: + validate-remote-models: + runs-on: ubuntu-22.04 + timeout-minutes: 90 + + steps: + - name: Install JUCE Linux dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + ninja-build \ + libasound2-dev \ + libx11-dev \ + libxinerama-dev \ + libxext-dev \ + libfreetype6-dev \ + libwebkit2gtk-4.0-dev \ + libglu1-mesa-dev \ + libcurl4-openssl-dev \ + xvfb + sudo /usr/bin/Xvfb "$DISPLAY" & + + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Configure + run: cmake -B "$BUILD_DIR" -G Ninja -DCMAKE_BUILD_TYPE="$BUILD_TYPE" . + + - name: Build model validation runner + run: cmake --build "$BUILD_DIR" --config "$BUILD_TYPE" --target HARPRemoteModelTests --parallel 4 + + - name: Run remote model validation + env: + HARP_MODEL_VALIDATION_ID: ${{ inputs.model_id }} + run: ctest --test-dir "$BUILD_DIR" --output-on-failure -R HARPRemoteModelTests + + - name: Upload validation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: remote-model-validation-report + path: artifacts/model_validation/remote diff --git a/.gitignore b/.gitignore index b41816c2..db6b6996 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,8 @@ temp cache dist +artifacts/model_validation/ + testproj.RPP # Ignore all the *.md files in website/HARP diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b05d238..e08b9462 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,133 +1,117 @@ -# The first line of any CMake project should be a call to `cmake_minimum_required`, which checks -# that the installed CMake will be able to understand the following CMakeLists, and ensures that -# CMake's behaviour is compatible with the named version. This is a standard CMake command, so more -# information can be found in the CMake docs. cmake_minimum_required(VERSION 3.21) -set(PROJECT_NAME "HARP") - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) file(STRINGS VERSION CURRENT_VERSION) -# add_definitions(-DAPP_VERSION="${CURRENT_VERSION}") -# add_definitions(-DAPP_NAME="${PROJECT_NAME}") -# add_definitions(-DAPP_COPYRIGHT="TEAMuP") - - - -# The top-level CMakeLists.txt file for a project must contain a literal, direct call to the -# `project()` command. `project()` semats up some helpful variables that describe source/binary -# directories, and the current project version. This is a standard CMake command. -project(${PROJECT_NAME} VERSION ${CURRENT_VERSION}) +project(HARP VERSION ${CURRENT_VERSION}) -# Add these lines to suppress warnings for JUCE -# ------------------------------------------ -# For macOS Xcode deprecation warnings +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(JUCE_SILENCE_XCODE_DEPRECATION_WARNINGS ON) -add_subdirectory(JUCE) # If you've put JUCE in a subdirectory called JUCE +add_subdirectory(JUCE) -# Reset warning levels if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang|GNU") add_compile_options(-Wall -Wextra) elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") add_compile_options(/W4) endif() -# `juce_add_gui_app` adds an executable target with the name passed as the first argument -# (${PROJECT_NAME} here). This target is a normal CMake target, but has a lot of extra properties set -# up by default. This function accepts many optional arguments. Check the readme at -# `docs/CMake API.md` in the JUCE repo for the full list. -# include(cmake/gradio_client.cmake) +set(HARP_COMPANY_NAME "TEAMuP") +set(HARP_ICON "${CMAKE_SOURCE_DIR}/resources/icons/harp_logo_1.png") + +set(HARP_COMMON_SOURCES + src/Model.h + src/clients/Client.h + src/clients/GradioClient.h + src/utils/Logging.h + src/utils/Settings.h + src/utils/Interface.h + src/utils/Controls.h + src/utils/Labels.h + src/utils/Clients.h + src/utils/Errors.h + src/utils/Enums.h +) + +set(HARP_APP_SOURCES + src/Main.cpp + src/Application.cpp + src/MainComponent.cpp + src/ModelTab.h + src/windows/AboutWindow.h + src/windows/WelcomeWindow.h + src/windows/settings/SettingsWindow.h + src/windows/settings/GeneralSettingsTab.cpp + src/windows/settings/LoginTab.cpp + src/widgets/ModelSelectionWidget.h + src/widgets/ModelInfoWidget.h + src/widgets/ControlAreaWidget.h + src/widgets/TrackAreaWidget.h + src/widgets/StatusAreaWidget.h + src/widgets/MediaClipboardWidget.h + src/gui/MultiButton.cpp + src/gui/HoverableLabel.h + src/gui/TextBoxWithLabel.h + src/gui/SliderWithLabel.h + src/gui/ComboBoxWithLabel.h + src/gui/HoverHandler.h + src/gui/ControlComponent.h + src/gui/ToggleWithLabel.h + src/media/MediaDisplayComponent.cpp + src/media/AudioDisplayComponent.cpp + src/media/MidiDisplayComponent.cpp + src/media/OutputLabelComponent.cpp + src/media/pianoroll/KeyboardComponent.cpp + src/media/pianoroll/NoteGridComponent.cpp + src/media/pianoroll/PianoRollComponent.cpp + src/media/pianoroll/SynthAudioSource.h + src/external/magic_enum.hpp + src/external/fontawesome/src/FontAwesome.h + src/external/fontawesome/src/FontAwesome.cpp + src/external/fontawesome/data/FontAwesomeData.h + src/external/fontawesome/data/FontAwesomeData.cpp + src/external/fontawesome/data/FontAwesomeIcons.h + src/external/fontaudio/src/FontAudio.h + src/external/fontaudio/src/FontAudio.cpp + src/external/fontaudio/data/FontAudioData.h + src/external/fontaudio/data/FontAudioData.cpp + src/external/fontaudio/data/FontAudioIcons.h +) + +set(HARP_JUCE_MODULES + juce::juce_audio_basics + juce::juce_audio_devices + juce::juce_audio_formats + juce::juce_audio_processors + juce::juce_audio_utils + juce::juce_core + juce::juce_data_structures + juce::juce_dsp + juce::juce_events + juce::juce_gui_basics + juce::juce_gui_extra +) + +set(HARP_JUCE_FLAGS + juce::juce_recommended_config_flags + juce::juce_recommended_lto_flags + juce::juce_recommended_warning_flags +) juce_add_gui_app(${PROJECT_NAME} - # VERSION ... # Set this if the app version is different to the project version - # ICON_* arguments specify a path to an image file to use as an icon - ICON_BIG "${CMAKE_SOURCE_DIR}/resources/icons/harp_logo_1.png" # Specify a big icon for the app - ICON_SMALL "${CMAKE_SOURCE_DIR}/resources/icons/harp_logo_1.png" # Specify a small icon for the app - - DOCUMENT_EXTENSIONS wav mp3 aiff # Specify file extensions that should be associated with this app - COMPANY_NAME "TEAMuP" # Specify the name of the app's author - PRODUCT_NAME "HARP") # The name of the final executable, which can differ from the target name - -# `juce_generate_juce_header` will create a JuceHeader.h for a given target, which will be generated -# into your build tree. This should be included with `#include `. The include path for -# this header will be automatically added to the target. The main function of the JuceHeader is to -# include all your JUCE module headers; if you're happy to include module headers directly, you -# probably don't need to call this. + ICON_BIG "${HARP_ICON}" + ICON_SMALL "${HARP_ICON}" + DOCUMENT_EXTENSIONS wav mp3 aiff + COMPANY_NAME "${HARP_COMPANY_NAME}" + PRODUCT_NAME "HARP" +) juce_generate_juce_header(${PROJECT_NAME}) -# `target_sources` adds source files to a target. We pass the target that needs the sources as the -# first argument, then a visibility parameter for the sources which should normally be PRIVATE. -# Finally, we supply a list of source files that will be built into the target. This is a standard -# CMake command. - target_sources(${PROJECT_NAME} PRIVATE - src/Main.cpp - src/Application.cpp - src/MainComponent.cpp - src/ModelTab.h - src/Model.h - - src/windows/AboutWindow.h - src/windows/WelcomeWindow.h - src/windows/settings/SettingsWindow.h - src/windows/settings/GeneralSettingsTab.cpp - src/windows/settings/LoginTab.cpp - - src/widgets/ModelSelectionWidget.h - src/widgets/ModelInfoWidget.h - src/widgets/ControlAreaWidget.h - src/widgets/TrackAreaWidget.h - src/widgets/StatusAreaWidget.h - src/widgets/MediaClipboardWidget.h - - src/gui/MultiButton.cpp - src/gui/HoverableLabel.h - src/gui/TextBoxWithLabel.h - src/gui/SliderWithLabel.h - src/gui/ComboBoxWithLabel.h - src/gui/HoverHandler.h - src/gui/ControlComponent.h - src/gui/ToggleWithLabel.h - - src/media/MediaDisplayComponent.cpp - src/media/AudioDisplayComponent.cpp - src/media/MidiDisplayComponent.cpp - src/media/OutputLabelComponent.cpp - - src/media/pianoroll/KeyboardComponent.cpp - src/media/pianoroll/NoteGridComponent.cpp - src/media/pianoroll/PianoRollComponent.cpp - src/media/pianoroll/SynthAudioSource.h - - src/clients/Client.h - src/clients/GradioClient.h - src/clients/GradioClient.h - - src/utils/Logging.h - src/utils/Settings.h - src/utils/Interface.h - src/utils/Controls.h - src/utils/Labels.h - src/utils/Clients.h - src/utils/Errors.h - src/utils/Enums.h - - src/external/magic_enum.hpp - src/external/fontawesome/src/FontAwesome.h - src/external/fontawesome/src/FontAwesome.cpp - src/external/fontawesome/data/FontAwesomeData.h - src/external/fontawesome/data/FontAwesomeData.cpp - src/external/fontawesome/data/FontAwesomeIcons.h - src/external/fontaudio/src/FontAudio.h - src/external/fontaudio/src/FontAudio.cpp - src/external/fontaudio/data/FontAudioData.h - src/external/fontaudio/data/FontAudioData.cpp - src/external/fontaudio/data/FontAudioIcons.h + ${HARP_APP_SOURCES} + ${HARP_COMMON_SOURCES} ) if(APPLE) @@ -138,18 +122,10 @@ elseif(LINUX) target_sources(${PROJECT_NAME} PRIVATE src/utils/copy/CopyLinux.cpp) endif() -# `target_compile_definitions` adds some preprocessor definitions to our target. In a Projucer -# project, these might be passed in the 'Preprocessor Definitions' field. JUCE modules also make use -# of compile definitions to switch certain features on/off, so if there's a particular feature you -# need that's not on by default, check the module header for the correct flag to set here. These -# definitions will be visible both to your code, and also the JUCE module code, so for new -# definitions, pick unique names that are unlikely to collide! This is a standard CMake command. - target_compile_definitions(${PROJECT_NAME} PRIVATE - # JUCE_WEB_BROWSER and JUCE_USE_CURL would be on by default, but you might not need them. - JUCE_WEB_BROWSER=0 # If you remove this, add `NEEDS_WEB_BROWSER TRUE` to the `juce_add_gui_app` call - JUCE_USE_CURL=1 # only needed in Linux + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=1 JUCE_LOAD_CURL_SYMBOLS_LAZILY=1 JUCE_APPLICATION_NAME_STRING="$" JUCE_APPLICATION_VERSION_STRING="$" @@ -157,51 +133,25 @@ target_compile_definitions(${PROJECT_NAME} JUCE_USE_OGGVORBIS=1 JUCE_USE_MP3AUDIOFORMAT=1 JUCE_USE_WINDOWS_MEDIA_FORMAT=1 - PUBLIC APP_VERSION="${CURRENT_VERSION}" - APP_COMPANY="TEAMuP" + APP_COMPANY="${HARP_COMPANY_NAME}" APP_NAME="HARP" - APP_COPYRIGHT="Copyright 2026 TEAMuP. All rights reserved." + APP_COPYRIGHT="Copyright 2026 TEAMuP. All rights reserved." ) -# If your target needs extra binary assets, you can add them here. The first argument is the name of -# a new static library target that will include all the binary resources. There is an optional -# `NAMESPACE` argument that can specify the namespace of the generated binary data class. Finally, -# the SOURCES argument should be followed by a list of source files that should be built into the -# static library. These source files can be of any kind (wav data, images, fonts, icons etc.). -# Conversion to binary-data will happen when your target is built. - juce_add_binary_data(stability_controls SOURCES - ${CMAKE_SOURCE_DIR}/src/clients/providers/stability/models/text-to-audio.json - ${CMAKE_SOURCE_DIR}/src/clients/providers/stability/models/audio-to-audio.json + "${CMAKE_SOURCE_DIR}/src/clients/providers/stability/models/text-to-audio.json" + "${CMAKE_SOURCE_DIR}/src/clients/providers/stability/models/audio-to-audio.json" + "${CMAKE_SOURCE_DIR}/resources/models/model_registry.json" ) -# `target_link_libraries` links libraries and JUCE modules to other libraries or executables. Here, -# we're linking our executable target to the `juce::juce_gui_extra` module. Inter-module -# dependencies are resolved automatically, so `juce_core`, `juce_events` and so on will also be -# linked automatically. If we'd generated a binary data target above, we would need to link to it -# here too. This is a standard CMake command. - target_link_libraries(${PROJECT_NAME} PRIVATE - juce::juce_gui_extra - juce::juce_audio_basics - juce::juce_audio_devices - juce::juce_audio_formats - juce::juce_audio_processors - juce::juce_audio_utils - juce::juce_core - juce::juce_data_structures - juce::juce_dsp - juce::juce_events - juce::juce_gui_basics - juce::juce_gui_extra + ${HARP_JUCE_MODULES} stability_controls PUBLIC - juce::juce_recommended_config_flags - juce::juce_recommended_lto_flags - juce::juce_recommended_warning_flags + ${HARP_JUCE_FLAGS} ) if(LINUX) @@ -209,7 +159,7 @@ if(LINUX) target_link_libraries(${PROJECT_NAME} PRIVATE X11::X11) endif() -if (WIN32) +if(WIN32) install(TARGETS ${PROJECT_NAME} RUNTIME_DEPENDENCIES PRE_EXCLUDE_REGEXES "api-ms-" "ext-ms-" @@ -219,26 +169,45 @@ if (WIN32) ) endif() -# copy the pyinstaller tools to the bundle -# if (APPLE) -# add_custom_command(TARGET ${PROJECT_NAME} -# POST_BUILD -# COMMAND ${CMAKE_COMMAND} -E copy_directory -# ${CMAKE_SOURCE_DIR}/py/client/dist/ -# "$/../Resources") -# else() -# add_custom_command(TARGET ${PROJECT_NAME} -# POST_BUILD -# COMMAND ${CMAKE_COMMAND} -E copy_directory -# ${CMAKE_SOURCE_DIR}/py/client/dist/ -# "$/Resources") -# endif() - -# this fixes the RPATH to be relative to the executable -# in MacOS. Now, all we need to do is copy the -# dynamic libraries to the executable directories -if (APPLE) - set_property(TARGET ${PROJECT_NAME} PROPERTY BUILD_RPATH "@loader_path/../Frameworks" ) +include(CTest) + +juce_add_console_app(HARPRemoteModelTests + PRODUCT_NAME "HARPRemoteModelTests" +) + +juce_generate_juce_header(HARPRemoteModelTests) + +target_sources(HARPRemoteModelTests + PRIVATE + tests/ModelValidationMain.cpp + ${HARP_COMMON_SOURCES} +) +target_include_directories(HARPRemoteModelTests + PRIVATE + "${CMAKE_SOURCE_DIR}/src" +) + +target_compile_definitions(HARPRemoteModelTests + PRIVATE + JUCE_USE_CURL=1 + JUCE_LOAD_CURL_SYMBOLS_LAZILY=1 + HARP_SOURCE_DIR="${CMAKE_SOURCE_DIR}" +) + +target_link_libraries(HARPRemoteModelTests + PRIVATE + ${HARP_JUCE_MODULES} + stability_controls + PUBLIC + ${HARP_JUCE_FLAGS} +) + +if(BUILD_TESTING) + add_test(NAME HARPRemoteModelTests COMMAND HARPRemoteModelTests) +endif() + +if(APPLE) + set_property(TARGET ${PROJECT_NAME} PROPERTY BUILD_RPATH "@loader_path/../Frameworks") set(CMAKE_SKIP_RPATH "NO" CACHE INTERNAL "") -endif(APPLE) +endif() diff --git a/resources/media/test.wav b/resources/media/test.wav index 6ac2cd4d..c1af7c93 100644 Binary files a/resources/media/test.wav and b/resources/media/test.wav differ diff --git a/resources/media/test_old.wav b/resources/media/test_old.wav new file mode 100644 index 00000000..6ac2cd4d Binary files /dev/null and b/resources/media/test_old.wav differ diff --git a/resources/models/model_registry.json b/resources/models/model_registry.json new file mode 100644 index 00000000..4c9d4c21 --- /dev/null +++ b/resources/models/model_registry.json @@ -0,0 +1,285 @@ +{ + "version": 1, + "models": [ + { + "id": "custom-path", + "name": "Custom Path", + "path": "click here to enter a custom path...", + "provider": "custom", + "source": "ui", + "featured": true, + "validation": { + "enabled": false, + "mode": "manual", + "reason": "UI placeholder entry used to add custom model paths." + } + }, + { + "id": "stability-text-to-audio", + "name": "Stable Audio Text to Audio", + "path": "stability/text-to-audio", + "provider": "stability", + "source": "stability_api", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_api", + "requires_network": true, + "requires_env": "HARP_STABILITY_API_KEY" + } + }, + { + "id": "stability-audio-to-audio", + "name": "Stable Audio Audio to Audio", + "path": "stability/audio-to-audio", + "provider": "stability", + "source": "stability_api", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_api", + "requires_network": true, + "requires_env": "HARP_STABILITY_API_KEY" + } + }, + { + "id": "text2midi-symbolic-music-generation", + "name": "Text2MIDI Symbolic Music Generation", + "path": "teamup-tech/text2midi-symbolic-music-generation", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "demucs-source-separation", + "name": "Demucs Source Separation", + "path": "teamup-tech/demucs-source-separation", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "solo-piano-audio-to-midi-transcription", + "name": "Solo Piano Audio to MIDI Transcription", + "path": "teamup-tech/solo-piano-audio-to-midi-transcription", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "transkun", + "name": "Transkun", + "path": "teamup-tech/transkun", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "tria", + "name": "TRIA", + "path": "teamup-tech/TRIA", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "anticipatory-music-transformer", + "name": "Anticipatory Music Transformer", + "path": "teamup-tech/anticipatory-music-transformer", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "vampnet-conditional-music-generation", + "name": "VampNet Conditional Music Generation", + "path": "teamup-tech/vampnet-conditional-music-generation", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "harmonic-percussive-separation", + "name": "Harmonic Percussive Separation", + "path": "teamup-tech/harmonic-percussive-separation", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "kokoro-tts", + "name": "Kokoro TTS", + "path": "teamup-tech/Kokoro-TTS", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "megatts3-voice-cloning", + "name": "MegaTTS3 Voice Cloning", + "path": "teamup-tech/MegaTTS3-Voice-Cloning", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "midi-synthesizer", + "name": "MIDI Synthesizer", + "path": "teamup-tech/midi-synthesizer", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "audioseal", + "name": "AudioSeal", + "path": "teamup-tech/audioseal", + "provider": "huggingface", + "source": "huggingface_space", + "featured": true, + "validation": { + "enabled": true, + "mode": "remote_gradio", + "requires_network": true + } + }, + { + "id": "pitch-shifter-example", + "name": "Pitch Shifter Example", + "path": null, + "provider": "pyharp", + "source": "local_example", + "featured": false, + "validation": { + "enabled": true, + "mode": "local_pyharp_example", + "app_path": "pyharp/examples/pitch_shifter/app.py", + "python_modules": [ + "gradio", + "audiotools", + "torch", + "torchaudio" + ], + "inputs": [ + { + "kind": "audio_fixture", + "path": "resources/media/test.wav" + }, + { + "kind": "literal", + "value": 7 + } + ], + "expected_outputs": [ + { + "kind": "audio_file", + "extension": ".wav", + "min_size_bytes": 128 + } + ] + } + }, + { + "id": "midi-pitch-shifter-example", + "name": "MIDI Pitch Shifter Example", + "path": null, + "provider": "pyharp", + "source": "local_example", + "featured": false, + "validation": { + "enabled": true, + "mode": "local_pyharp_example", + "app_path": "pyharp/examples/midi_pitch_shifter/app.py", + "python_modules": [ + "gradio", + "symusic" + ], + "inputs": [ + { + "kind": "midi_fixture", + "path": "resources/media/test.mid" + }, + { + "kind": "literal", + "value": 7 + } + ], + "expected_outputs": [ + { + "kind": "midi_file", + "extension": ".mid", + "min_size_bytes": 32 + } + ] + } + }, + { + "id": "ui-tester-example", + "name": "UI Tester Example", + "path": null, + "provider": "pyharp", + "source": "local_example", + "featured": false, + "validation": { + "enabled": false, + "mode": "local_pyharp_example", + "reason": "Downloads external fixture files and is better covered by a dedicated integration test." + } + } + ] +} \ No newline at end of file diff --git a/src/utils/ModelRegistry.h b/src/utils/ModelRegistry.h new file mode 100644 index 00000000..bc4bb423 --- /dev/null +++ b/src/utils/ModelRegistry.h @@ -0,0 +1,96 @@ +/** + * @file ModelRegistry.h + * @brief Accessors for the bundled model registry. + */ + +#pragma once + +#include + +#include +#include + +#include "Logging.h" + +using namespace juce; + +namespace ModelRegistry +{ +inline std::vector getFallbackFeaturedModelPaths() +{ + return { + "click here to enter a custom path...", + "stability/text-to-audio", + "stability/audio-to-audio", + "teamup-tech/text2midi-symbolic-music-generation", + "teamup-tech/demucs-source-separation", + "teamup-tech/solo-piano-audio-to-midi-transcription", + "teamup-tech/transkun", // TODO - more intuitive name + "teamup-tech/TRIA", // TODO - more intuitive name: (The Rhythm In Anything) conditional drum generation + "teamup-tech/anticipatory-music-transformer", + "teamup-tech/vampnet-conditional-music-generation", + "teamup-tech/harmonic-percussive-separation", + "teamup-tech/Kokoro-TTS", + "teamup-tech/MegaTTS3-Voice-Cloning", + "teamup-tech/midi-synthesizer", + "teamup-tech/audioseal", // TODO - more intuitive name + // "xribene/HARP-UI-TEST-v3" + }; +} + +inline std::vector getFeaturedModelPaths() +{ + const String registryJson = + String::fromUTF8(BinaryData::model_registry_json, BinaryData::model_registry_jsonSize); + + var parsedRegistry; + const Result parseResult = JSON::parse(registryJson, parsedRegistry); + + if (parseResult.failed() || ! parsedRegistry.isObject()) + { + DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Failed to parse bundled registry. " + "Using fallback list."); + + return getFallbackFeaturedModelPaths(); + } + + auto* root = parsedRegistry.getDynamicObject(); + + if (root == nullptr || ! root->hasProperty("models") || ! root->getProperty("models").isArray()) + { + DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Registry is missing a valid models " + "array. Using fallback list."); + + return getFallbackFeaturedModelPaths(); + } + + std::vector featuredPaths; + + for (const auto& modelVar : *root->getProperty("models").getArray()) + { + if (! modelVar.isObject()) + continue; + + auto* model = modelVar.getDynamicObject(); + + if (model == nullptr) + continue; + + const bool isFeatured = static_cast(model->getProperty("featured")); + const String path = model->getProperty("path").toString(); + + if (isFeatured && path.isNotEmpty()) + featuredPaths.push_back(path.toStdString()); + } + + if (featuredPaths.empty()) + { + DBG_AND_LOG("ModelRegistry::getFeaturedModelPaths: Registry did not yield any featured " + "paths. Using fallback list."); + + return getFallbackFeaturedModelPaths(); + } + + return featuredPaths; +} +} // namespace ModelRegistry \ No newline at end of file diff --git a/src/utils/Settings.h b/src/utils/Settings.h index ea6069fb..267dc2c5 100644 --- a/src/utils/Settings.h +++ b/src/utils/Settings.h @@ -7,6 +7,7 @@ #pragma once #include +#include using namespace juce; diff --git a/src/widgets/ModelSelectionWidget.h b/src/widgets/ModelSelectionWidget.h index bba39217..900c5fc8 100644 --- a/src/widgets/ModelSelectionWidget.h +++ b/src/widgets/ModelSelectionWidget.h @@ -19,11 +19,17 @@ #include "../utils/Errors.h" #include "../utils/Interface.h" #include "../utils/Logging.h" +#include "../utils/ModelRegistry.h" using namespace juce; struct SharedChoices : public ChangeBroadcaster { + SharedChoices() + : savedModelPaths(ModelRegistry::getFeaturedModelPaths()) + { + } + int getIndexForPath(const std::string& p) { int idx = -1; @@ -56,24 +62,7 @@ struct SharedChoices : public ChangeBroadcaster sendSynchronousChangeMessage(); } - std::vector savedModelPaths = { - "click here to enter a custom path...", - "stability/text-to-audio", - "stability/audio-to-audio", - "teamup-tech/text2midi-symbolic-music-generation", - "teamup-tech/demucs-source-separation", - "teamup-tech/solo-piano-audio-to-midi-transcription", - "teamup-tech/transkun", // TODO - more intuitive name - "teamup-tech/TRIA", // TODO - more intuitive name: (The Rhythm In Anything) conditional drum generation - "teamup-tech/anticipatory-music-transformer", - "teamup-tech/vampnet-conditional-music-generation", - "teamup-tech/harmonic-percussive-separation", - "teamup-tech/Kokoro-TTS", - "teamup-tech/MegaTTS3-Voice-Cloning", - "teamup-tech/midi-synthesizer", - "teamup-tech/audioseal", // TODO - more intuitive name - // "xribene/HARP-UI-TEST-v3" - }; + std::vector savedModelPaths; }; class CustomPathComponent : public Component @@ -550,4 +539,4 @@ class ModelSelectionWidget : public Component, public ChangeBroadcaster, public MultiButton::Mode loadButtonActiveInfo; SharedResourcePointer instructionsMessage; -}; +}; \ No newline at end of file diff --git a/tests/ModelValidationMain.cpp b/tests/ModelValidationMain.cpp new file mode 100644 index 00000000..6986ffde --- /dev/null +++ b/tests/ModelValidationMain.cpp @@ -0,0 +1,745 @@ +#include +#include +#include +#include + +#include + +#include "Model.h" +#include "utils/Logging.h" + +JUCE_IMPLEMENT_SINGLETON(HARPLogger) + +using namespace juce; + +namespace +{ +constexpr auto registryRelativePath = "resources/models/model_registry.json"; +constexpr auto audioFixtureRelativePath = "resources/media/test.wav"; +constexpr auto midiFixtureRelativePath = "resources/media/test.mid"; +constexpr int defaultPerModelTimeoutMs = 120000; + +struct ValidationEntry +{ + String id; + String name; + String path; + String mode; + String requiredEnv; +}; + +struct ValidationResultRow +{ + String id; + String name; + String path; + String outcome; + String reason; +}; + +File repoRoot() +{ + return File(HARP_SOURCE_DIR); +} + +String getEnvValue(const char* name) +{ + if (const char* value = std::getenv(name)) + { + return String::fromUTF8(value); + } + + return {}; +} + +String firstNonEmptyEnv(std::initializer_list names) +{ + for (const auto* name : names) + { + const auto value = getEnvValue(name); + + if (value.isNotEmpty()) + { + return value; + } + } + + return {}; +} + +String getSelectedModelId() +{ + return getEnvValue("HARP_MODEL_VALIDATION_ID"); +} + +bool isChildMode() +{ + return getEnvValue("HARP_MODEL_VALIDATION_CHILD") == "1"; +} + +int getPerModelTimeoutMs() +{ + const auto value = getEnvValue("HARP_MODEL_VALIDATION_TIMEOUT_MS"); + + if (value.isEmpty()) + { + return defaultPerModelTimeoutMs; + } + + const auto parsed = value.getIntValue(); + return parsed > 0 ? parsed : defaultPerModelTimeoutMs; +} + +constexpr int defaultMaxRetries = 0; +constexpr int defaultRetryDelayMs = 15000; + +int getMaxRetries() +{ + const auto value = getEnvValue("HARP_MODEL_VALIDATION_RETRIES"); + + if (value.isEmpty()) + { + return defaultMaxRetries; + } + + const auto parsed = value.getIntValue(); + return parsed >= 0 ? parsed : defaultMaxRetries; +} + +int getRetryDelayMs() +{ + const auto value = getEnvValue("HARP_MODEL_VALIDATION_RETRY_DELAY_MS"); + + if (value.isEmpty()) + { + return defaultRetryDelayMs; + } + + const auto parsed = value.getIntValue(); + return parsed > 0 ? parsed : defaultRetryDelayMs; +} + +File getReportDir() +{ + const auto envValue = getEnvValue("HARP_MODEL_VALIDATION_REPORT_DIR"); + + if (envValue.isNotEmpty()) + { + return File::isAbsolutePath(envValue) ? File(envValue) : repoRoot().getChildFile(envValue); + } + + return repoRoot().getChildFile("artifacts/model_validation/remote"); +} + +void seedProviderTokens(const ValidationEntry& entry) +{ + SharedResourcePointer sharedTokens; + + if (entry.mode == "remote_api") + { + const auto token = firstNonEmptyEnv({ "HARP_STABILITY_API_KEY", "STABILITY_API_KEY" }); + + if (token.isNotEmpty()) + { + sharedTokens->savedTokens[Provider::Stability] = token; + } + } + else + { + const auto token = firstNonEmptyEnv({ "HARP_HUGGINGFACE_TOKEN", "HF_TOKEN" }); + + if (token.isNotEmpty()) + { + sharedTokens->savedTokens[Provider::HuggingFace] = token; + } + } +} + +var parseJsonFile(const File& file) +{ + const auto parsed = JSON::parse(file.loadFileAsString()); + + if (parsed.isVoid()) + { + throw std::runtime_error(("Failed to parse " + file.getFullPathName()).toStdString()); + } + + return parsed; +} + +std::vector loadRemoteValidationEntries() +{ + const auto parsedRegistry = parseJsonFile(repoRoot().getChildFile(registryRelativePath)); + + if (! parsedRegistry.isObject()) + { + throw std::runtime_error("Model registry root must be a JSON object."); + } + + const auto* root = parsedRegistry.getDynamicObject(); + + if (root == nullptr || ! root->hasProperty("models") || ! root->getProperty("models").isArray()) + { + throw std::runtime_error("Model registry must contain a models array."); + } + + const auto selectedModelId = getSelectedModelId(); + std::vector entries; + std::unordered_set seenIds; + + for (const auto& modelVar : *root->getProperty("models").getArray()) + { + if (! modelVar.isObject()) + continue; + + const auto* model = modelVar.getDynamicObject(); + + if (model == nullptr || ! model->hasProperty("validation")) + continue; + + const auto validationVar = model->getProperty("validation"); + + if (! validationVar.isObject()) + continue; + + const auto* validation = validationVar.getDynamicObject(); + + if (validation == nullptr || ! static_cast(validation->getProperty("enabled"))) + continue; + + const auto mode = validation->getProperty("mode").toString(); + + if (mode != "remote_gradio" && mode != "remote_api") + continue; + + const auto id = model->getProperty("id").toString(); + + if (! seenIds.insert(id).second) + { + throw std::runtime_error(("Duplicate remote validation model id: " + id).toStdString()); + } + + if (selectedModelId.isNotEmpty() && id != selectedModelId) + continue; + + entries.push_back({ + id, + model->getProperty("name").toString(), + model->getProperty("path").toString(), + mode, + validation->getProperty("requires_env").toString(), + }); + } + + return entries; +} + +void seedModelInputs(Model& model) +{ + const auto audioFixture = repoRoot().getChildFile(audioFixtureRelativePath); + const auto midiFixture = repoRoot().getChildFile(midiFixtureRelativePath); + + for (const auto& input : model.getInputTracks()) + { + if (auto* audio = dynamic_cast(input.get())) + { + if (audio->required) + { + audio->path = audioFixture.getFullPathName().toStdString(); + } + } + else if (auto* midi = dynamic_cast(input.get())) + { + if (midi->required) + { + midi->path = midiFixture.getFullPathName().toStdString(); + } + } + } + + for (const auto& control : model.getControls()) + { + if (auto* text = dynamic_cast(control.get())) + { + if (text->value.empty()) + { + text->value = "Short validation prompt"; + } + } + } +} + +String validateOutputFiles(const std::vector& outputFiles, const LabelList& labels) +{ + for (const auto& outputFile : outputFiles) + { + if (! outputFile.existsAsFile()) + { + return "missing output file"; + } + + const auto extension = outputFile.getFileExtension().toLowerCase(); + + if (extension == ".mid" || extension == ".midi") + { + std::ifstream midiStream(outputFile.getFullPathName().toStdString(), std::ios::binary); + char header[4] {}; + midiStream.read(header, 4); + + if (String::fromUTF8(header, 4) != "MThd") + { + return "invalid MIDI output"; + } + } + else if (outputFile.getSize() <= 0) + { + return "empty output file"; + } + } + + for (const auto& label : labels) + { + if (label == nullptr) + { + return "null label output"; + } + } + + return {}; +} + +String classifyFailureMessage(const String& message) +{ + if (message.containsIgnoreCase("status code 503")) + return "503 Service Unavailable"; + if (message.containsIgnoreCase("timed out")) + return "remote Gradio timeout"; + if (message.containsIgnoreCase("runtime error occurred at endpoint")) + return "remote Gradio runtime error"; + if (message.containsIgnoreCase("valid API key")) + return "missing or invalid API key"; + + return message.isNotEmpty() ? message : "unknown failure"; +} + +String summarizeReason(const String& reason) +{ + auto lines = StringArray::fromLines(reason); + + for (int index = lines.size() - 1; index >= 0; --index) + { + const auto line = lines[index].trim(); + + if (line.isNotEmpty()) + { + return line.replace("|", "\\|"); + } + } + + return {}; +} + +String renderMarkdownReport(const std::vector& results) +{ + int passed = 0; + int failed = 0; + int skipped = 0; + + for (const auto& result : results) + { + if (result.outcome == "passed") + passed += 1; + else if (result.outcome == "failed") + failed += 1; + else if (result.outcome == "skipped") + skipped += 1; + } + + String markdown; + markdown << "# HARP Remote Model Validation\n\n"; + markdown << "- Total: " << results.size() << "\n"; + markdown << "- Passed: " << passed << "\n"; + markdown << "- Failed: " << failed << "\n"; + markdown << "- Skipped: " << skipped << "\n\n"; + markdown << "## Dashboard\n\n"; + markdown << "| Model Path | Outcome | Detail |\n"; + markdown << "| --- | --- | --- |\n"; + + for (const auto& result : results) + { + markdown << "| `" << result.path << "` | " << result.outcome << " | " + << summarizeReason(result.reason) << " |\n"; + } + + return markdown; +} + +void writeReport(const File& reportDir, const std::vector& results) +{ + reportDir.createDirectory(); + + int passed = 0; + int failed = 0; + int skipped = 0; + Array rows; + + for (const auto& result : results) + { + if (result.outcome == "passed") + passed += 1; + else if (result.outcome == "failed") + failed += 1; + else if (result.outcome == "skipped") + skipped += 1; + + auto* row = new DynamicObject(); + row->setProperty("id", result.id); + row->setProperty("name", result.name); + row->setProperty("path", result.path); + row->setProperty("outcome", result.outcome); + row->setProperty("reason", result.reason); + rows.add(var(row)); + } + + auto* summary = new DynamicObject(); + summary->setProperty("total", static_cast(results.size())); + summary->setProperty("passed", passed); + summary->setProperty("failed", failed); + summary->setProperty("skipped", skipped); + + auto* report = new DynamicObject(); + report->setProperty("generated_at", Time::getCurrentTime().toISO8601(true)); + report->setProperty("registry_path", repoRoot().getChildFile(registryRelativePath).getFullPathName()); + report->setProperty("summary", var(summary)); + report->setProperty("results", rows); + + const auto reportJson = JSON::toString(var(report), true); + const auto reportMd = renderMarkdownReport(results); + + reportDir.getChildFile("latest.json").replaceWithText(reportJson); + reportDir.getChildFile("latest.md").replaceWithText(reportMd); + reportDir.getChildFile("status.json").replaceWithText(reportJson); + reportDir.getChildFile("dashboard.md").replaceWithText(reportMd); +} + +ValidationResultRow validateEntry(const ValidationEntry& entry) +{ + if (entry.requiredEnv.isNotEmpty() && getEnvValue(entry.requiredEnv.toRawUTF8()).isEmpty()) + { + return { entry.id, entry.name, entry.path, "skipped", + "Required environment variable " + entry.requiredEnv + " is not set." }; + } + + seedProviderTokens(entry); + + Model model; + const auto loadResult = model.load(entry.path); + + if (loadResult.failed()) + { + return { entry.id, + entry.name, + entry.path, + "failed", + classifyFailureMessage(toUserMessage(loadResult.getError())) }; + } + + seedModelInputs(model); + + std::map inputFiles; + + for (const auto& input : model.getInputTracks()) + { + if (auto* audio = dynamic_cast(input.get())) + { + if (audio->required) + { + inputFiles[input->id] = File(audio->path); + } + } + else if (auto* midi = dynamic_cast(input.get())) + { + if (midi->required) + { + inputFiles[input->id] = File(midi->path); + } + } + } + + std::vector outputFiles; + LabelList labels; + const auto processResult = model.process(inputFiles, outputFiles, labels); + + if (processResult.failed()) + { + return { entry.id, + entry.name, + entry.path, + "failed", + classifyFailureMessage(toUserMessage(processResult.getError())) }; + } + + if (static_cast(outputFiles.size()) != static_cast(model.getOutputTracks().size())) + { + return { entry.id, entry.name, entry.path, "failed", "unexpected number of output files" }; + } + + const auto outputError = validateOutputFiles(outputFiles, labels); + + for (const auto& outputFile : outputFiles) + { + ignoreUnused(outputFile.deleteFile()); + } + + if (outputError.isNotEmpty()) + { + return { entry.id, entry.name, entry.path, "failed", outputError }; + } + + return { entry.id, entry.name, entry.path, "passed", {} }; +} + +String serializeResultRow(const ValidationResultRow& result) +{ + DynamicObject::Ptr object = new DynamicObject(); + object->setProperty("id", result.id); + object->setProperty("name", result.name); + object->setProperty("path", result.path); + object->setProperty("outcome", result.outcome); + object->setProperty("reason", result.reason); + return JSON::toString(var(object), false).replaceCharacters("\r\n", " "); +} + +bool parseResultRowFromOutput(const String& output, ValidationResultRow& result) +{ + auto lines = StringArray::fromLines(output); + + for (int index = lines.size() - 1; index >= 0; --index) + { + const auto line = lines[index].trim(); + + if (! line.startsWith("RESULT_JSON:")) + { + continue; + } + + const auto parsed = JSON::parse(line.fromFirstOccurrenceOf("RESULT_JSON:", false, false).trim()); + + if (! parsed.isObject()) + { + return false; + } + + const auto* object = parsed.getDynamicObject(); + + if (object == nullptr) + { + return false; + } + + result.id = object->getProperty("id").toString(); + result.name = object->getProperty("name").toString(); + result.path = object->getProperty("path").toString(); + result.outcome = object->getProperty("outcome").toString(); + result.reason = object->getProperty("reason").toString(); + return true; + } + + return false; +} + +bool isRetryableFailure(const ValidationResultRow& result) +{ + if (result.outcome != "failed") + return false; + + const auto& reason = result.reason; + return reason.containsIgnoreCase("sleeping") + || reason.containsIgnoreCase("restarting") + || reason.containsIgnoreCase("try again") + || reason.containsIgnoreCase("timed out") + || reason.containsIgnoreCase("503") + || reason.containsIgnoreCase("Unable to make POST request"); +} + +ValidationResultRow runEntryInChildProcessOnce(const ValidationEntry& entry) +{ + StringArray command; + command.add("env"); + command.add("HARP_MODEL_VALIDATION_CHILD=1"); + command.add("HARP_MODEL_VALIDATION_ID=" + entry.id); + command.add("HARP_MODEL_VALIDATION_TIMEOUT_MS=" + String(getPerModelTimeoutMs())); + + const auto reportDir = getReportDir(); + command.add("HARP_MODEL_VALIDATION_REPORT_DIR=" + reportDir.getFullPathName()); + + const auto hfToken = firstNonEmptyEnv({ "HARP_HUGGINGFACE_TOKEN", "HF_TOKEN" }); + if (hfToken.isNotEmpty()) + { + command.add("HARP_HUGGINGFACE_TOKEN=" + hfToken); + } + + const auto stabilityKey = firstNonEmptyEnv({ "HARP_STABILITY_API_KEY", "STABILITY_API_KEY" }); + if (stabilityKey.isNotEmpty()) + { + command.add("HARP_STABILITY_API_KEY=" + stabilityKey); + } + + command.add(File::getSpecialLocation(File::currentExecutableFile).getFullPathName()); + + ChildProcess child; + + if (! child.start(command)) + { + return { entry.id, entry.name, entry.path, "failed", "failed to launch child process" }; + } + + if (! child.waitForProcessToFinish(getPerModelTimeoutMs())) + { + child.kill(); + return { entry.id, entry.name, entry.path, "failed", "model validation timed out" }; + } + + const auto output = child.readAllProcessOutput(); + ValidationResultRow result; + + if (parseResultRowFromOutput(output, result)) + { + return result; + } + + const auto exitCode = child.getExitCode(); + const auto trimmedOutput = output.trim(); + const auto fallbackReason = trimmedOutput.isNotEmpty() + ? summarizeReason(trimmedOutput) + : "child process exited with code " + String(exitCode); + + return { entry.id, entry.name, entry.path, "failed", fallbackReason }; +} + +ValidationResultRow runEntryInChildProcess(const ValidationEntry& entry) +{ + const auto maxRetries = getMaxRetries(); + const auto retryDelayMs = getRetryDelayMs(); + + auto result = runEntryInChildProcessOnce(entry); + + for (int attempt = 1; attempt <= maxRetries && isRetryableFailure(result); ++attempt) + { + std::cout << " retry " << attempt << "/" << maxRetries + << " (waiting " << retryDelayMs / 1000 << "s)...\n"; + + Thread::sleep(retryDelayMs); + result = runEntryInChildProcessOnce(entry); + } + + return result; +} +} // namespace + +int main(int argc, char* argv[]) +{ + ignoreUnused(argc, argv); + + ScopedJuceInitialiser_GUI scopedJuce; + + try + { + const auto entries = loadRemoteValidationEntries(); + const auto reportDir = getReportDir(); + + if (entries.empty()) + { + std::cerr << "No enabled remote validation entries found.\n"; + return 1; + } + + if (isChildMode()) + { + const auto result = validateEntry(entries.front()); + std::cout << "RESULT_JSON: " << serializeResultRow(result) << '\n'; + return result.outcome == "failed" ? 1 : 0; + } + + const auto totalEntries = static_cast(entries.size()); + + std::cout << "\n"; + std::cout << "========================================\n"; + std::cout << " HARP Remote Model Validation\n"; + std::cout << "========================================\n"; + std::cout << " Models to validate: " << totalEntries << "\n"; + std::cout << " Timeout per model: " << getPerModelTimeoutMs() / 1000 << "s\n"; + std::cout << " Max retries: " << getMaxRetries() << "\n"; + std::cout << "----------------------------------------\n\n"; + + std::vector results; + results.reserve(entries.size()); + + for (int i = 0; i < totalEntries; ++i) + { + const auto& entry = entries[static_cast(i)]; + + std::cout << "[" << (i + 1) << "/" << totalEntries << "] " + << entry.path << " ... " << std::flush; + + const auto result = runEntryInChildProcess(entry); + results.push_back(result); + + if (result.outcome == "passed") + std::cout << "PASSED\n"; + else if (result.outcome == "skipped") + std::cout << "SKIPPED (" << result.reason << ")\n"; + else + std::cout << "FAILED\n"; + } + + int passed = 0; + int failed = 0; + int skipped = 0; + std::vector failedResults; + + for (const auto& result : results) + { + if (result.outcome == "passed") + passed += 1; + else if (result.outcome == "skipped") + skipped += 1; + else + { + failed += 1; + failedResults.push_back(&result); + } + } + + std::cout << "\n"; + std::cout << "========================================\n"; + std::cout << " Summary\n"; + std::cout << "========================================\n"; + std::cout << " Passed: " << passed << " / " << totalEntries << "\n"; + std::cout << " Failed: " << failed << " / " << totalEntries << "\n"; + std::cout << " Skipped: " << skipped << " / " << totalEntries << "\n"; + std::cout << "----------------------------------------\n"; + + if (! failedResults.empty()) + { + std::cout << "\n Failed models:\n\n"; + + for (const auto* result : failedResults) + { + std::cout << " " << result->path << "\n"; + std::cout << " Reason: " << result->reason << "\n\n"; + } + } + + std::cout << "========================================\n\n"; + + writeReport(reportDir, results); + + return failed > 0 ? 1 : 0; + } + catch (const std::exception& exception) + { + std::cerr << exception.what() << '\n'; + return 1; + } +}