Fine-grained plugins dependendy upload

.github/workflows/build.yml

@@ -14,6 +14,9 @@ env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
   BUILD_TYPE: Release
   MACENABLED: ${{ inputs.forceMacBuild }}
+  USERNAME: ManiVaultStudio
+  FEED_URL: https://nuget.pkg.github.com/ManiVaultStudio/index.json
+  VCPKG_BINARY_SOURCES: "clear;nuget,https://nuget.pkg.github.com/ManiVaultStudio/index.json,readwrite"
 
 # for matrix check https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners
 jobs:
@@ -42,14 +45,14 @@ jobs:
 
       - name: Checkout the source
         if: github.event_name != 'pull_request'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: recursive
           fetch-depth: 0
 
       - name: Checkout the source - pull request
         if: github.event_name == 'pull_request'
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: recursive
           fetch-depth: 0
@@ -76,6 +79,85 @@ jobs:
           key: ${{ secrets.RULESSUPPORT_DEPLOY_KEY }}
           known_hosts: github.com AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
 
+      # Install vcpkg
+      - name: Clone vcpkg
+        run: |
+          cd ${{ github.workspace }}
+          git clone --branch 2024.08.23 --single-branch https://github.com/microsoft/vcpkg.git
+
+      # Bootstrap vcpkg
+      - name: Bootstrap vcpkg 
+        if: startsWith(matrix.os, 'windows')
+        shell: pwsh
+        run:  |
+          ${{ github.workspace }}\vcpkg\bootstrap-vcpkg.bat
+          echo "VCPKG_ROOT=${{ github.workspace }}\vcpkg" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "VCPKG_EXE=${{ github.workspace }}\vcpkg\vcpkg.exe" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "VCPKG_LIBRARY_LINKAGE=dynamic" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
+      - name: Bootstrap vcpkg
+        if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
+        shell: bash
+        run: |
+          ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh 
+          echo "VCPKG_ROOT=${{ github.workspace }}/vcpkg" >> $GITHUB_ENV
+          echo "VCPKG_EXE=${{ github.workspace }}/vcpkg/vcpkg" >> $GITHUB_ENV
+          echo "VCPKG_LIBRARY_LINKAGE=dynamic" >> $GITHUB_ENV
+
+      # Use cached vcpkg packges if possible
+      - name: Add NuGet sources
+        if: startsWith(matrix.os, 'windows')
+        shell: pwsh
+        run: |
+          .$(${{ env.VCPKG_EXE }} fetch nuget) `
+            sources add `
+            -Source "${{ env.FEED_URL }}" `
+            -StorePasswordInClearText `
+            -Name GitHubPackages `
+            -UserName "${{ env.USERNAME }}" `
+            -Password "${{ secrets.GH_VCPKG_PACKAGES }}"
+          .$(${{ env.VCPKG_EXE }} fetch nuget) `
+            setapikey "${{ secrets.GH_VCPKG_PACKAGES }}" `
+            -Source "${{ env.FEED_URL }}"
+
+      - name: Add NuGet sources
+        if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
+        shell: bash
+        run: |
+          mono `${{ env.VCPKG_EXE }} fetch nuget | tail -n 1` \
+            sources add \
+            -Source "${{ env.FEED_URL }}" \
+            -StorePasswordInClearText \
+            -Name GitHubPackages \
+            -UserName "${{ env.USERNAME }}" \
+            -Password "${{ secrets.GH_VCPKG_PACKAGES }}"
+          mono `${{ env.VCPKG_EXE }} fetch nuget | tail -n 1` \
+            setapikey "${{ secrets.GH_VCPKG_PACKAGES }}" \
+            -Source "${{ env.FEED_URL }}"
+
+      # Install vcpkg dependencies
+      - name: Install vcpkg dependencies 
+        if: startsWith(matrix.os, 'windows')
+        shell: pwsh
+        run:  |
+          cd ${{ github.workspace }}
+          ${{ env.VCPKG_EXE }} install --triplet x64-windows
+
+      - name: Install vcpkg dependencies 
+        if: startsWith(matrix.os, 'ubuntu')
+        shell: bash
+        run:  |
+          cd ${{ github.workspace }}
+          ${{ env.VCPKG_EXE }} install --triplet x64-linux
+
+      - name: Install vcpkg dependencies 
+        if: startsWith(matrix.os, 'macos')
+        shell: bash
+        run:  |
+          cd ${{ github.workspace }}
+          ${{ env.VCPKG_EXE }} install --triplet x64-osx --allow-unsupported
+
+      # Build the package
       - name: Windows build
         if: startsWith(matrix.os, 'windows')
         uses: ManiVaultStudio/github-actions/conan_windows_build@main

.gitignore

@@ -3,3 +3,4 @@ build
 x64
 *.dir
 *.bak
+ExampleDependencies/cmake/install_dependencies.cmake

CMakeLists.txt

@@ -1,11 +1,22 @@
 cmake_minimum_required(VERSION 3.17)
 
+set(MV_EXAMPLES_USE_VCPKG OFF)
+if(DEFINED CMAKE_TOOLCHAIN_FILE AND CMAKE_TOOLCHAIN_FILE MATCHES "vcpkg")
+    set(MV_EXAMPLES_USE_VCPKG ON)
+    set(VCPKG_LIBRARY_LINKAGE "dynamic" CACHE STRING "Link vcpkg libraries dynamically")
+endif()
+
 # -----------------------------------------------------------------------------
 # ExamplePlugins
 # -----------------------------------------------------------------------------
 set(PROJECT "ExamplePlugins")
 PROJECT(${PROJECT})
 
+# vcpkg is used together with conan on ci
+if(DEFINED VCPKG_TARGET_TRIPLET)
+    set(MV_EXAMPLES_USE_VCPKG ON)
+endif()
+
 add_subdirectory(ExampleView)
 add_subdirectory(ExampleViewJS)
 add_subdirectory(ExampleViewOpenGL)
@@ -14,3 +25,4 @@ add_subdirectory(ExampleTransformation)
 add_subdirectory(ExampleLoader)
 add_subdirectory(ExampleWriter)
 add_subdirectory(ExampleData)
+add_subdirectory(ExampleDependencies)

ExampleAnalysis/CMakeLists.txt

@@ -26,7 +26,7 @@ endif(MSVC)
 # -----------------------------------------------------------------------------
 find_package(Qt6 COMPONENTS Widgets WebEngineWidgets REQUIRED)
 
-find_package(ManiVault COMPONENTS Core PointData CONFIG)
+find_package(ManiVault COMPONENTS Core PointData CONFIG QUIET)
 
 # -----------------------------------------------------------------------------
 # Source files
@@ -40,10 +40,6 @@ set(PLUGIN_SOURCES
     src/ExampleAnalysisPlugin.json
 )
 
-set(PLUGIN_MOC_HEADERS
-    src/ExampleAnalysisPlugin.h
-)
-
 source_group( Plugin FILES ${PLUGIN_SOURCES})
 
 # -----------------------------------------------------------------------------
@@ -83,7 +79,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE ManiVault::PointData)
 # -----------------------------------------------------------------------------
 # Target installation
 # -----------------------------------------------------------------------------
-# Install the shared plugin libary to the "Plugins" folder in the ManiVault install directory
+# Install the shared plugin library to the "Plugins" folder in the ManiVault install directory
 install(TARGETS ${PROJECT_NAME}
     RUNTIME DESTINATION Plugins COMPONENT PLUGINS # Windows .dll
     LIBRARY DESTINATION Plugins COMPONENT PLUGINS # Linux/Mac .so

ExampleData/CMakeLists.txt

@@ -29,7 +29,7 @@ include(GenerateExportHeader)
 # -----------------------------------------------------------------------------
 find_package(Qt6 COMPONENTS Widgets WebEngineWidgets REQUIRED)
 
-find_package(ManiVault COMPONENTS Core PointData CONFIG)
+find_package(ManiVault COMPONENTS Core PointData CONFIG QUIET)
 
 # -----------------------------------------------------------------------------
 # Source files

ExampleDependencies/CMakeLists.txt

@@ -0,0 +1,143 @@
+cmake_minimum_required(VERSION 3.21)
+
+option(MV_UNITY_BUILD "Combine target source files into batches for faster compilation" OFF)
+
+# -----------------------------------------------------------------------------
+# ExampleAnalysis Plugin
+# -----------------------------------------------------------------------------
+PROJECT("ExampleDependenciesPlugin")
+
+# -----------------------------------------------------------------------------
+# CMake Options
+# -----------------------------------------------------------------------------
+set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_AUTOMOC ON)
+
+if(MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /DWIN32 /EHsc /MP /permissive- /Zc:__cplusplus")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /NODEFAULTLIB:LIBCMT")
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd")
+    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD")
+endif(MSVC)
+
+include(cmake/get_cpm.cmake)
+
+# -----------------------------------------------------------------------------
+# Dependencies
+# -----------------------------------------------------------------------------
+find_package(Qt6 COMPONENTS Widgets WebEngineWidgets REQUIRED)
+
+find_package(ManiVault COMPONENTS Core PointData CONFIG QUIET)
+
+CPMAddPackage(
+  NAME highway
+  URL https://github.com/google/highway/archive/refs/tags/1.2.0.tar.gz
+  URL_HASH SHA256=7e0be78b8318e8bdbf6fa545d2ecb4c90f947df03f7aadc42c1967f019e63343
+  PATCHES "cmake/highway.patch" # see https://github.com/conan-io/conan-center-index/pull/24197/files, fixes https://github.com/google/highway/issues/2225
+  OPTIONS "HWY_ENABLE_EXAMPLES OFF" "HWY_ENABLE_INSTALL OFF" "HWY_ENABLE_TESTS OFF" "HWY_ENABLE_CONTRIB ON" "BUILD_SHARED_LIBS ON" 
+)
+
+if(${MV_EXAMPLES_USE_VCPKG})
+    find_package(blake3 CONFIG REQUIRED)
+
+    if(NOT APPLE)
+        include(cmake/ci_fixes.cmake)
+        find_package(faiss CONFIG REQUIRED)
+    endif()
+endif()
+
+# -----------------------------------------------------------------------------
+# Source files
+# -----------------------------------------------------------------------------
+# Define the plugin sources
+set(PLUGIN_SOURCES
+    src/ExampleDependenciesPlugin.h
+    src/ExampleDependenciesPlugin.cpp
+    src/SettingsAction.h
+    src/SettingsAction.cpp
+    src/ExampleDependenciesPlugin.json
+)
+
+source_group( Plugin FILES ${PLUGIN_SOURCES})
+
+# -----------------------------------------------------------------------------
+# CMake Target
+# -----------------------------------------------------------------------------
+# Create dynamic library for the plugin
+add_library(${PROJECT_NAME} SHARED ${PLUGIN_SOURCES})
+
+# -----------------------------------------------------------------------------
+# Target include directories
+# -----------------------------------------------------------------------------
+# Include ManiVault headers, including system data plugins
+target_include_directories(${PROJECT_NAME} PRIVATE "${ManiVault_INCLUDE_DIR}")
+
+# -----------------------------------------------------------------------------
+# Target properties
+# -----------------------------------------------------------------------------
+# Request C++17
+target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17)
+
+# Enable unity build
+if(MV_UNITY_BUILD)
+    set_target_properties(${PROJECT_NAME} PROPERTIES UNITY_BUILD ON)
+endif()
+
+# -----------------------------------------------------------------------------
+# Target library linking
+# -----------------------------------------------------------------------------
+# Link to Qt libraries
+target_link_libraries(${PROJECT_NAME} PRIVATE Qt6::Widgets)
+target_link_libraries(${PROJECT_NAME} PRIVATE Qt6::WebEngineWidgets)
+
+# Link to ManiVault and data plugins
+target_link_libraries(${PROJECT_NAME} PRIVATE ManiVault::Core)
+target_link_libraries(${PROJECT_NAME} PRIVATE ManiVault::PointData)
+
+target_link_libraries(${PROJECT_NAME} PRIVATE hwy hwy_contrib)
+
+if(${MV_EXAMPLES_USE_VCPKG})
+    target_link_libraries(${PROJECT_NAME} PRIVATE BLAKE3::blake3)
+
+    if(NOT APPLE)
+    target_link_libraries(${PROJECT_NAME} PRIVATE faiss)
+    endif()
+
+    target_compile_definitions (${PROJECT_NAME} PRIVATE __USE_VCPKG__)
+endif()
+
+# Not used here, but you can use this in your plugin
+# automatically available when using find_package(ManiVault)
+mv_check_and_set_AVX(${PROJECT_NAME} OFF)
+
+# -----------------------------------------------------------------------------
+# Target installation
+# -----------------------------------------------------------------------------
+# Install the shared plugin library to the "Plugins" folder in the ManiVault install directory
+install(TARGETS ${PROJECT_NAME}
+    RUNTIME DESTINATION Plugins COMPONENT PLUGINS # Windows .dll
+    LIBRARY DESTINATION Plugins COMPONENT PLUGINS # Linux/Mac .so
+)
+
+add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E echo "Installing: Plugin files"
+    COMMAND "${CMAKE_COMMAND}"
+        --install ${CMAKE_CURRENT_BINARY_DIR}
+        --config $<CONFIGURATION>
+        --component PLUGINS
+        --prefix ${ManiVault_INSTALL_DIR}/$<CONFIGURATION>
+        --verbose
+)
+
+# Automatically available when using find_package(ManiVault)
+mv_install_dependencies(${PROJECT_NAME} "hwy" "hwy_contrib")
+
+# -----------------------------------------------------------------------------
+# Miscellaneous
+# -----------------------------------------------------------------------------
+# Automatically set the debug environment (command + working directory) for MSVC
+if(MSVC)
+    set_property(TARGET ${PROJECT_NAME} PROPERTY VS_DEBUGGER_WORKING_DIRECTORY $<IF:$<CONFIG:DEBUG>,${ManiVault_INSTALL_DIR}/debug,${ManiVault_INSTALL_DIR}/release>)
+    set_property(TARGET ${PROJECT_NAME} PROPERTY VS_DEBUGGER_COMMAND $<IF:$<CONFIG:DEBUG>,${ManiVault_INSTALL_DIR}/debug/ManiVault\ Studio.exe,${ManiVault_INSTALL_DIR}/release/ManiVault\ Studio.exe>)
+endif()

ExampleDependencies/README.md

@@ -0,0 +1,70 @@
+# ExampleDependencies
+
+Exemplary setup for automatically installing runtime dependencies.
+
+A plugin developer can use `mv_install_dependencies` to set up automatic plugin runtime dependency gathering and installation with this command:
+```cmake
+mv_install_dependencies(${PROJECT_NAME} "hwy" "hwy_contrib")
+```
+Dependencies that are linked to our plugin `${PROJECT_NAME}` and found with `find_package` are automatically resolve (given that their respective cmake export files are nicely setup). This even works with dependency chains, as shown in this example: we link to Faiss, which in turn depends on Lapack and OpenBlas which in turn depends on other libraries.
+Otherwise, we need to pass all targets that we build in our project but do not set up with `find_package` to `mv_install_dependencies`, which in this example is the plugin itself and two dependencies, "hwy" and "hwy_contrib". 
+
+The `mv_install_dependencies` is automatically available when using `find_package(ManiVault ... CONFIG)`. The important bit here is **CONFIG**.
+
+There might be cases in which not all dependencies are automatically resolved, i.e. some libraries are located in folders that `mv_install_dependencies` does not search in. We can provide additional search paths like this:
+```cmake
+set(DEPENDENCIES_FOLDERS "${FREEIMAGE_ROOT_DIR}/bin")
+mv_install_dependencies(${PROJECT_NAME})
+```
+All paths listed in `DEPENDENCIES_FOLDERS` will now also be searched for runtime dependencies.
+
+All installed dependencies are listed like this (current CI output on windows):
+```bash
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/blake3.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/faiss.dll
+-- Resolved: D:/.conan/bab46b/1/_deps/highway-build/Release/hwy.dll
+-- Resolved: D:/.conan/bab46b/1/_deps/highway-build/Release/hwy_contrib.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/libgcc_s_seh-1.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/libgfortran-5.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/liblapack.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/libquadmath-0.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/libwinpthread-1.dll
+-- Resolved: D:/.conan/bab46b/1/ExampleDependencies/Release/openblas.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/blake3.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/faiss.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/hwy.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/hwy_contrib.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/libgcc_s_seh-1.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/libgfortran-5.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/liblapack.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/libquadmath-0.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/libwinpthread-1.dll
+-- Installing: D:\.conan\bab46b\1\package\Release/PluginDependencies/ExampleDependenciesPlugin/openblas.dll
+```
+
+## Managing dependencies with vcpkg
+[vcpkg](https://github.com/microsoft/vcpkg/) will be used in this project if `CMAKE_TOOLCHAIN_FILE` is set to `YOUR_LOCAL_PATH_TO/vcpkg/scripts/buildsystems/vcpkg.cmake`. It is used to install [faiss](https://github.com/facebookresearch/faiss) and [blake3](https://github.com/BLAKE3-team/BLAKE3). If not set, this repository will only build [highway](https://github.com/google/highway).
+
+This plugin also illustrates how you can set up vcpkg on CI runs, see the main `conanfile.py` and `build.yml`. Caching vcpkg dependencies on the CI requires setting up a secret as shown [here](https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-packages), in our example referred to as `secrets.GH_VCPKG_PACKAGES`.
+
+> Important: when building a project using `add_subdirectory` all vcpkg dependencies defined in the manifest file `vcpkg.json` must be uplifted to the main project. vcpkg is not recursively scanning for `vcpkg.json`. In CMake, vcpkg only uses the toplevel `vcpkg.json` since all dependencies need to be installed before the very first project() call.
+
+## ManiVault's runtime dependency handling
+When starting ManiVault, the application will load all dynamic libraries located in `PluginDependencies/plugin_a` before loading `plugin_a`. The folder structure of your ManiVault installation should look look like this:
+```
+ManiVault Studio/
+├─ ManiVault Studio.exe
+├─ plugin_c_dependency.dll
+├─ Plugins/
+│  ├─ plugin_a.dll
+│  ├─ plugin_b.dll
+│  ├─ plugin_c.dll
+├─ PluginDependencies/
+│  ├─ plugin_a/
+│  │  ├─ plugin_a_dependency.dll
+│  ├─ plugin_b/
+│  │  ├─ plugin_b_dependency_1.dll
+│  │  ├─ plugin_b_dependency_2.dll
+│  │  ├─ dependency_of_plugin_b_dependency_2.dll
+````
+You can always manually copy runtime dependencies into the specific `PluginDependencies/YourPlugin` subfolder (or next to the ManiVault executable on Windows), but we recommend doing so during the installation step of your build (as done with `mv_install_dependencies`).