From f79e9b02e7c0fe1c9d259f82e13a71ecbc4feab4 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 7 Mar 2024 17:58:43 +0100 Subject: [PATCH 01/63] implement query of sample-type-connections --- .github.settings.xml | 14 +++ .github/CODEOWNERS | 1 + .github/pr-labels.yml | 3 + .github/release.yml | 20 ++++ .github/workflows/build-package.yml | 29 +++++ .github/workflows/codeql-analysis.yml | 83 +++++++++++++ .github/workflows/create-release.yml | 96 +++++++++++++++ .github/workflows/label-pull-requests.yml | 15 +++ .github/workflows/nexus-publish-snapshots.yml | 46 +++++++ .github/workflows/run-tests.yml | 31 +++++ LICENSE | 21 ++++ pom.xml | 113 ++++++++++++++++++ src/main/java/META-INF/MANIFEST.MF | 3 + src/main/java/life/qbic/App.java | 75 ++++++++++++ .../io/commandline/AuthenticationOptions.java | 79 ++++++++++++ .../io/commandline/CommandLineOptions.java | 30 +++++ .../commandline/ManifestVersionProvider.java | 15 +++ .../io/commandline/OpenBISPasswordParser.java | 30 +++++ .../commandline/SampleHierarchyCommand.java | 37 ++++++ .../java/life/qbic/model/Configuration.java | 13 ++ .../life/qbic/model/SampleTypeConnection.java | 69 +++++++++++ .../qbic/model/download/Authentication.java | 61 ++++++++++ .../download/AuthenticationException.java | 23 ++++ .../model/download/ConnectionException.java | 20 ++++ .../qbic/model/download/FileSystemWriter.java | 39 ++++++ .../qbic/model/download/ModelReporter.java | 10 ++ .../qbic/model/download/OpenbisConnector.java | 108 +++++++++++++++++ .../qbic/model/download/OutputPathFinder.java | 83 +++++++++++++ src/main/java/life/qbic/util/StringUtil.java | 9 ++ src/main/resources/log4j2.xml | 17 +++ 30 files changed, 1193 insertions(+) create mode 100644 .github.settings.xml create mode 100644 .github/CODEOWNERS create mode 100644 .github/pr-labels.yml create mode 100644 .github/release.yml create mode 100644 .github/workflows/build-package.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/create-release.yml create mode 100644 .github/workflows/label-pull-requests.yml create mode 100644 .github/workflows/nexus-publish-snapshots.yml create mode 100644 .github/workflows/run-tests.yml create mode 100644 LICENSE create mode 100644 pom.xml create mode 100644 src/main/java/META-INF/MANIFEST.MF create mode 100644 src/main/java/life/qbic/App.java create mode 100644 src/main/java/life/qbic/io/commandline/AuthenticationOptions.java create mode 100644 src/main/java/life/qbic/io/commandline/CommandLineOptions.java create mode 100644 src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java create mode 100644 src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java create mode 100644 src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java create mode 100644 src/main/java/life/qbic/model/Configuration.java create mode 100644 src/main/java/life/qbic/model/SampleTypeConnection.java create mode 100644 src/main/java/life/qbic/model/download/Authentication.java create mode 100644 src/main/java/life/qbic/model/download/AuthenticationException.java create mode 100644 src/main/java/life/qbic/model/download/ConnectionException.java create mode 100644 src/main/java/life/qbic/model/download/FileSystemWriter.java create mode 100644 src/main/java/life/qbic/model/download/ModelReporter.java create mode 100644 src/main/java/life/qbic/model/download/OpenbisConnector.java create mode 100644 src/main/java/life/qbic/model/download/OutputPathFinder.java create mode 100644 src/main/java/life/qbic/util/StringUtil.java create mode 100644 src/main/resources/log4j2.xml diff --git a/.github.settings.xml b/.github.settings.xml new file mode 100644 index 0000000..5c65a2e --- /dev/null +++ b/.github.settings.xml @@ -0,0 +1,14 @@ + + + + nexus-snapshots + ${env.MAVEN_REPO_USERNAME} + ${env.MAVEN_REPO_PASSWORD} + + + nexus-releases + ${env.MAVEN_REPO_USERNAME} + ${env.MAVEN_REPO_PASSWORD} + + + \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..8d7e69b --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @qbicsoftware/itss diff --git a/.github/pr-labels.yml b/.github/pr-labels.yml new file mode 100644 index 0000000..f95cf6e --- /dev/null +++ b/.github/pr-labels.yml @@ -0,0 +1,3 @@ +feature: ['feature/*', 'feat/*'] +fix: ['fix/*', 'hotfix'] +chore: ['chore/*', 'documentation/*', 'docs/*', 'ci/*', 'refactor/*'] diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..1a928f0 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,20 @@ +changelog: + exclude: + labels: + - ignore-for-release + authors: + - JohnnyQ5 + - github-actions + categories: + - title: New Features πŸš€ + labels: + - feature + - title: Bugfixes πŸͺ² + labels: + - fix + - title: Documentation & CI πŸͺ‚ + labels: + - chore + - title: Others πŸ§ƒ + labels: + - "*" diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml new file mode 100644 index 0000000..d616186 --- /dev/null +++ b/.github/workflows/build-package.yml @@ -0,0 +1,29 @@ +name: Build Maven Package + +on: + push: + branches: + - '**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ main, master ] + +jobs: + package: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Load local Maven repository cache + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run mvn package + run: mvn -B package --file pom.xml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..76c1d11 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,83 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main, master, development, release/*, hotfix/* ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main, master ] + schedule: + - cron: '21 1 * * 4' + +jobs: + analyze: + name: analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'java' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + settings-path: ${{ github.workspace }} + + - name: Load local Maven repository cache + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # πŸ“š https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 0000000..82efca7 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,96 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + versionTag: + description: 'Version Tag (semantic version)' + required: true + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + settings-path: ${{ github.workspace }} + + - name: Load local Maven repository cache + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Set up git + run: | + git config --global user.email "support@qbic.zendesk.com" + git config --global user.name "JohnnyQ5" + + - name: Set version in Maven project + run: mvn versions:set -DnewVersion=${{ github.event.inputs.versionTag }} + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Create Release Notes + if: ${{ !startsWith(github.ref, 'refs/tags/') + && !( contains(github.event.inputs.versionTag, 'alpha') + || contains(github.event.inputs.versionTag, 'beta') + || contains(github.event.inputs.versionTag, 'rc')) }} + uses: actions/github-script@v4.0.2 + with: + github-token: ${{secrets.JOHNNY_Q5_REPORTS_TOKEN}} + script: | + await github.request(`POST /repos/${{ github.repository }}/releases`, { + tag_name: "${{ github.event.inputs.versionTag }}", + generate_release_notes: true + }); + + - name: Create Pre-Release Notes + if: ${{ !startsWith(github.ref, 'refs/tags/') + && ( contains(github.event.inputs.versionTag, 'alpha') + || contains(github.event.inputs.versionTag, 'beta') + || contains(github.event.inputs.versionTag, 'rc')) }} + uses: actions/github-script@v4.0.2 + with: + github-token: ${{secrets.JOHNNY_Q5_REPORTS_TOKEN}} + script: | + await github.request(`POST /repos/${{ github.repository }}/releases`, { + tag_name: "${{ github.event.inputs.versionTag }}", + generate_release_notes: true, + prerelease: true + }); + + - name: Publish artefact to QBiC Nexus Repository + run: mvn --quiet --settings $GITHUB_WORKSPACE/.github.settings.xml deploy + env: + MAVEN_REPO_USERNAME: ${{ secrets.NEXUS_USERNAME }} + MAVEN_REPO_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + + - name: Switch to new branch + run: git checkout -b release/set-version-to-${{ github.event.inputs.versionTag }} + + - name: Set remote branch + run: git push --set-upstream origin release/set-version-to-${{ github.event.inputs.versionTag }} + + - name: Checkin commit + run: git commit . -m 'Set version to ${{ github.event.inputs.versionTag }}' + + - name: Push to Github + run: git push + + - name: Open PR with version bump + uses: actions/github-script@v4.0.2 + with: + github-token: ${{secrets.JOHNNY_Q5_REPORTS_TOKEN}} + script: | + await github.request(`POST /repos/${{ github.repository }}/pulls`, { + title: 'Update version to ${{ github.event.inputs.versionTag }}', + head: 'release/set-version-to-${{ github.event.inputs.versionTag }}', + base: 'master' + }); diff --git a/.github/workflows/label-pull-requests.yml b/.github/workflows/label-pull-requests.yml new file mode 100644 index 0000000..b03eee1 --- /dev/null +++ b/.github/workflows/label-pull-requests.yml @@ -0,0 +1,15 @@ +name: Label Pull Requests + +on: + pull_request: + types: [ opened, edited ] + +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: TimonVS/pr-labeler-action@v3 + with: + configuration-path: .github/pr-labels.yml # optional, .github/pr-labeler.yml is the default value + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nexus-publish-snapshots.yml b/.github/workflows/nexus-publish-snapshots.yml new file mode 100644 index 0000000..4fc649b --- /dev/null +++ b/.github/workflows/nexus-publish-snapshots.yml @@ -0,0 +1,46 @@ +# This workflow will build a package using Maven and then publish it to +# qbic-repo.qbic.uni-tuebingen.de packages when a release is created +# For more information see: https://github.com/actions/setup-java#apache-maven-with-a-settings-path + +name: Deploy Snapshot + +on: + push: + branches: + - development + +jobs: + publish_snapshots: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + settings-path: ${{ github.workspace }} + + - name: Load local Maven repository cache + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + # Remove existing snapshot tags which are not supposed to be present + - name: Remove snapshot tags + run: mvn versions:set -DremoveSnapshot + # Set the SNAPSHOT for this build and deployment + - name: Set version in Maven project + run: mvn versions:set -DnewVersion='${project.version}-SNAPSHOT' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Publish artefact to QBiC Nexus Repository + run: mvn --settings $GITHUB_WORKSPACE/.github.settings.xml deploy + env: + MAVEN_REPO_USERNAME: ${{ secrets.NEXUS_USERNAME }} + MAVEN_REPO_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..5708a25 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,31 @@ +name: Run Maven Tests + +on: + push: + branches: + - '**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ main, master ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Load local Maven repository cache + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Run tests + run: mvn clean verify diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d167600 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2024 University of TΓΌbingen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..59840c3 --- /dev/null +++ b/pom.xml @@ -0,0 +1,113 @@ + + + 4.0.0 + life.qbic + openbis-scripts + 1.0.0 + Postman cli + https://github.com/qbicsoftware/openbis20-scripts + A client software written in Java to query openBIS + jar + + 11 + 11 + UTF-8 + 5.3.31 + + + + + + + + central + https://repo.maven.apache.org/maven2 + + false + + + + nexus-releases + QBiC Releases + + fail + true + always + + + false + + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases + + + + + + + org.apache.logging.log4j + log4j-core + 2.20.0 + + + org.slf4j + slf4j-api + 2.0.12 + + + org.slf4j + slf4j-simple + 2.0.12 + + + info.picocli + picocli + 4.6.2 + + + jline + jline + 2.14.6 + + + life.qbic + openbis-api + 20.10.7.3 + r1700646105 + + + life.qbic + openbis-api-core + 20.10.7.3 + r1700646105 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + package + + single + + + + + life.qbic.App + true + + + + jar-with-dependencies + + + + + + + + diff --git a/src/main/java/META-INF/MANIFEST.MF b/src/main/java/META-INF/MANIFEST.MF new file mode 100644 index 0000000..4657ac2 --- /dev/null +++ b/src/main/java/META-INF/MANIFEST.MF @@ -0,0 +1,3 @@ +Manifest-Version: 1.0 +Main-Class: life.qbic.App + diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java new file mode 100644 index 0000000..6ef2b8a --- /dev/null +++ b/src/main/java/life/qbic/App.java @@ -0,0 +1,75 @@ +package life.qbic; + +import life.qbic.io.commandline.OpenBISPasswordParser; +import life.qbic.io.commandline.CommandLineOptions; +import life.qbic.model.Configuration; +import life.qbic.model.download.Authentication; +import life.qbic.model.download.AuthenticationException; +import life.qbic.model.download.ConnectionException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; + +import java.io.File; +import java.util.Arrays; +import java.util.Optional; + +/** + * postman for staging data from openBIS + */ +public class App { + + private static final Logger LOG = LogManager.getLogger(App.class); + + public static void main(String[] args) { + LOG.debug("command line arguments: " + Arrays.deepToString(args)); + CommandLine cmd = new CommandLine(new CommandLineOptions()); + int exitCode = cmd.execute(args); + System.exit(exitCode); + } + + /** + * checks if the commandline parameter for reading out the password from the environment variable + * is correctly provided + */ + private static Boolean isNotNullOrEmpty(String envVariableCommandLineParameter) { + return envVariableCommandLineParameter != null && !envVariableCommandLineParameter.isEmpty(); + } + + /** + * Logs into OpenBIS asks for and verifies password. + * + * @return An instance of the Authentication class. + */ + public static Authentication loginToOpenBIS( + char[] password, String user, String as_url) { + + // Ensure 'logs' folder is created + File logFolder = new File(Configuration.LOG_PATH.toAbsolutePath().toString()); + if (!logFolder.exists()) { + boolean logFolderCreated = logFolder.mkdirs(); + if (!logFolderCreated) { + LOG.error("Could not create log folder '" + logFolder.getAbsolutePath() + "'"); + System.exit(1); + } + } + + Authentication authentication = + new Authentication( + user, + new String(password), + as_url); + try { + authentication.login(); + } catch (ConnectionException e) { + LOG.error(e.getMessage(), e); + LOG.error("Could not connect to QBiC's data source. Have you requested access to the " + + "server? If not please write to support@qbic.zendesk.com"); + System.exit(1); + } catch (AuthenticationException e) { + LOG.error(e.getMessage()); + System.exit(1); + } + return authentication; + } +} diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java new file mode 100644 index 0000000..8f49d7a --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -0,0 +1,79 @@ +package life.qbic.io.commandline; + +import static java.util.Objects.nonNull; +import static picocli.CommandLine.ArgGroup; + +import java.util.StringJoiner; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +public class AuthenticationOptions { + private static final Logger log = LogManager.getLogger(AuthenticationOptions.class); + + @Option( + names = {"-u", "--user"}, + required = true, + description = "openBIS user name") + public String user; + @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. + PasswordOptions passwordOptions; + + @Option( + names = {"-as", "-as_url"}, + description = "ApplicationServer URL", + scope = CommandLine.ScopeType.INHERIT) + public String as_url; + + public char[] getPassword() { + return passwordOptions.getPassword(); + } + + /** + * official picocli documentation example + */ + static class PasswordOptions { + @Option(names = "--password:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--password:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--password", arity = "0", description = "please provide your password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } + + @Override + public String toString() { + return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") + .add("user='" + user + "'") + .toString(); + //ATTENTION: do not expose the password here! + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java new file mode 100644 index 0000000..315dd85 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -0,0 +1,30 @@ +package life.qbic.io.commandline; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +// main command with format specifiers for the usage help message +@Command(name = "openbis-scripts", + subcommands = { SampleHierarchyCommand.class }, + description = "A client software for querying openBIS.", + mixinStandardHelpOptions = true) +public class CommandLineOptions { + private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); + + @Option(names = {"-V", "--version"}, + versionHelp = true, + description = "print version information", + scope = CommandLine.ScopeType.INHERIT) + boolean versionRequested; + + @Option( + names = {"-h", "--help"}, + usageHelp = true, + description = "display a help message and exit", + scope = CommandLine.ScopeType.INHERIT) + public boolean helpRequested = false; + +} diff --git a/src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java b/src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java new file mode 100644 index 0000000..afa48a3 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/ManifestVersionProvider.java @@ -0,0 +1,15 @@ +package life.qbic.io.commandline; + +import picocli.CommandLine; + +public class ManifestVersionProvider implements CommandLine.IVersionProvider { + @Override + public String[] getVersion() { + String implementationVersion = getClass().getPackage().getImplementationVersion(); + return new String[]{ + "version: " + implementationVersion, + "JVM: ${java.version} (${java.vendor} ${java.vm.name} ${java.vm.version})", + "OS: ${os.name} ${os.version} ${os.arch}" + }; + } +} diff --git a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java b/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java new file mode 100644 index 0000000..55923f7 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java @@ -0,0 +1,30 @@ +package life.qbic.io.commandline; + +import java.io.Console; +import java.util.Optional; + +public class OpenBISPasswordParser { + + + /** + * Retrieve the password from the system console + * + * @return the password read from the system console input + */ + public static String readPasswordFromConsole() { + Console console = System.console(); + char[] passwordChars = console.readPassword(); + return String.valueOf(passwordChars); + } + + /** + * @param variableName Name of given environment variable + * @return the password read from the environment variable + */ + public static Optional readPasswordFromEnvVariable(String variableName) { + + return Optional.ofNullable(System.getenv(variableName)); + + } + +} diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java new file mode 100644 index 0000000..c0b1a92 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -0,0 +1,37 @@ +package life.qbic.io.commandline; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import life.qbic.App; +import life.qbic.model.SampleTypeConnection; +import life.qbic.model.download.Authentication; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +@Command(name = "sample-types", + description = "lists sample types with children sample types and how often they are found in the openbis instance") +public class SampleHierarchyCommand implements Runnable { + + @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) + private String space; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + List spaces = new ArrayList<>(); + if(space!=null) { + spaces.add(space); + } + Authentication authentication = App.loginToOpenBIS(auth.getPassword(), auth.user, auth.as_url); + OpenbisConnector openbis = new OpenbisConnector(auth.as_url, authentication.getSessionToken(), "/Users/afriedrich/Downloads/"); + Map hierarchy = openbis.queryFullSampleHierarchy(spaces); + hierarchy.entrySet().stream() + .sorted(Entry.comparingByValue()) + .forEach(entry -> System.err.println(entry.getKey()+" ("+entry.getValue()+")")); + } +} diff --git a/src/main/java/life/qbic/model/Configuration.java b/src/main/java/life/qbic/model/Configuration.java new file mode 100644 index 0000000..cb567bd --- /dev/null +++ b/src/main/java/life/qbic/model/Configuration.java @@ -0,0 +1,13 @@ +package life.qbic.model; + +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Global configuration container + */ +public class Configuration { + + public static final long MAX_DOWNLOAD_ATTEMPTS = 3; + public static final Path LOG_PATH = Paths.get(System.getProperty("user.dir"),"logs"); +} diff --git a/src/main/java/life/qbic/model/SampleTypeConnection.java b/src/main/java/life/qbic/model/SampleTypeConnection.java new file mode 100644 index 0000000..807abce --- /dev/null +++ b/src/main/java/life/qbic/model/SampleTypeConnection.java @@ -0,0 +1,69 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.util.Objects; + +public class SampleTypeConnection { + + private SampleType parent; + private SampleType child; + + public SampleTypeConnection(SampleType parentType, SampleType childType) { + this.parent = parentType; + this.child = childType; + } + + public SampleTypeConnection(SampleType parentType) { + this(parentType, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SampleTypeConnection)) { + return false; + } + + SampleTypeConnection that = (SampleTypeConnection) o; + if(child == null || that.child == null) { + boolean a = Objects.equals(child, that.child); + if(parent == null || that.parent == null) { + return a && Objects.equals(parent, that.parent); + } else { + return a && Objects.equals(parent.getCode(), that.parent.getCode()); + } + } + if(parent == null || that.parent == null) { + boolean a = Objects.equals(parent, that.parent); + if(child == null || that.child == null) { + return a && Objects.equals(child, that.child); + } else { + return a && Objects.equals(child.getCode(), that.child.getCode()); + } + } + + if (!Objects.equals(parent.getCode(), that.parent.getCode())) { + return false; + } + return Objects.equals(child.getCode(), that.child.getCode()); + } + + @Override + public int hashCode() { + int result = parent != null ? parent.getCode().hashCode() : 0; + result = 31 * result + (child != null ? child.getCode().hashCode() : 0); + return result; + } + + @Override + public String toString() { + String parentCode = parent.getCode(); + if(child==null) { + return parentCode; + } else { + return parentCode+" -> "+child.getCode(); + } + } +} diff --git a/src/main/java/life/qbic/model/download/Authentication.java b/src/main/java/life/qbic/model/download/Authentication.java new file mode 100644 index 0000000..f151fd7 --- /dev/null +++ b/src/main/java/life/qbic/model/download/Authentication.java @@ -0,0 +1,61 @@ +package life.qbic.model.download; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.systemsx.cisd.common.spring.HttpInvokerUtils; + +public class Authentication { + + private static String sessionToken; + private String user; + private String password; + private final IApplicationServerApi applicationServer; + + + public Authentication( + String user, + String password, + String AppServerUri) { + this.user = user; + this.password = password; + if (!AppServerUri.isEmpty()) { + this.applicationServer = + HttpInvokerUtils.createServiceStub( + IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); + } else { + this.applicationServer = null; + } + this.setCredentials(user, password); + } + + /** + * Login method for openBIS authentication + * + * @throws AuthenticationException in case the authentication failed + */ + public void login() throws ConnectionException, AuthenticationException { + try { + sessionToken = applicationServer.login(user, password); + } catch (Exception e) { + throw new ConnectionException("Connection to openBIS server failed.", e); + } + if (sessionToken == null || sessionToken.isEmpty()) { + throw new AuthenticationException("Authentication failed. Are you using the correct " + + "credentials?"); + } + } + + public String getSessionToken() { + return sessionToken; + } + + /** + * Setter for user and password credentials + * + * @param user The openBIS user + * @param password The openBIS user's password + */ + public void setCredentials(String user, String password) { + this.user = user; + this.password = password; + } +} diff --git a/src/main/java/life/qbic/model/download/AuthenticationException.java b/src/main/java/life/qbic/model/download/AuthenticationException.java new file mode 100644 index 0000000..6738853 --- /dev/null +++ b/src/main/java/life/qbic/model/download/AuthenticationException.java @@ -0,0 +1,23 @@ +package life.qbic.model.download; + +/** + * Exception to indicate failed authentication against openBIS. + *

+ * This exception shall be thrown, when the returned session token of openBIS is empty, after the + * client tried to authenticate against the openBIS application server via its Java API. + */ +public class AuthenticationException extends RuntimeException { + + AuthenticationException() { + super(); + } + + AuthenticationException(String msg) { + super(msg); + } + + AuthenticationException(String msg, Throwable t) { + super(msg, t); + } + +} diff --git a/src/main/java/life/qbic/model/download/ConnectionException.java b/src/main/java/life/qbic/model/download/ConnectionException.java new file mode 100644 index 0000000..73bf9af --- /dev/null +++ b/src/main/java/life/qbic/model/download/ConnectionException.java @@ -0,0 +1,20 @@ +package life.qbic.model.download; + +/** + * ConnectionException indicates issues when a client wants to connect with the openBIS API. + */ +public class ConnectionException extends RuntimeException { + + ConnectionException() { + super(); + } + + ConnectionException(String msg) { + super(msg); + } + + ConnectionException(String msg, Throwable t) { + super(msg, t); + } + +} diff --git a/src/main/java/life/qbic/model/download/FileSystemWriter.java b/src/main/java/life/qbic/model/download/FileSystemWriter.java new file mode 100644 index 0000000..7a09c63 --- /dev/null +++ b/src/main/java/life/qbic/model/download/FileSystemWriter.java @@ -0,0 +1,39 @@ +package life.qbic.model.download; + +import java.io.File; +import java.nio.file.Path; +import java.util.List; + +/** + * File system implementation of the ModelReporter interface. + * + * Provides methods to write the queried openBIS model to a file on the local file system. + * + * @author: Sven Fillinger, Andreas Friedrich + */ +class FileSystemWriter implements ModelReporter { + + /** + * File that stores the summary report content for valid checksums. + */ + final private File matchingSummaryFile; + + + /** + * FileSystemWriter constructor with the paths for the summary files. * + * + * @param matchingSummaryFile The path where to write the summary + */ + FileSystemWriter(Path matchingSummaryFile) { + this.matchingSummaryFile = new File(matchingSummaryFile.toString()); + } + + /** + * {@inheritDoc} + */ + @Override + public void reportSummary(List summary) { + //TODO + } + +} diff --git a/src/main/java/life/qbic/model/download/ModelReporter.java b/src/main/java/life/qbic/model/download/ModelReporter.java new file mode 100644 index 0000000..2e453eb --- /dev/null +++ b/src/main/java/life/qbic/model/download/ModelReporter.java @@ -0,0 +1,10 @@ +package life.qbic.model.download; + +import java.net.URL; +import java.nio.file.Path; +import java.util.List; + +interface ModelReporter { + + void reportSummary(List summary); +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java new file mode 100644 index 0000000..e5dc841 --- /dev/null +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -0,0 +1,108 @@ +package life.qbic.model.download; + +import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; +import ch.systemsx.cisd.common.spring.HttpInvokerUtils; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import life.qbic.App; +import life.qbic.model.Configuration; +import life.qbic.model.SampleTypeConnection; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class OpenbisConnector { + + private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); + private final IApplicationServerApi applicationServer; + private final String sessionToken; + private final String outputPath; + + private final ModelReporter modelReporter = new FileSystemWriter( + Paths.get(Configuration.LOG_PATH.toString(), "summary_model.txt")); + + /** + * Constructor for a QBiCDataDownloader instance + * + * @param AppServerUri The openBIS application server URL (AS) + * @param sessionToken The session token for the datastore & application servers + */ + public OpenbisConnector( + String AppServerUri, + String sessionToken, + String outputPath) { + this.sessionToken = sessionToken; + this.outputPath = outputPath; + + if (!AppServerUri.isEmpty()) { + applicationServer = + HttpInvokerUtils.createServiceStub( + IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); + } else { + applicationServer = null; + } + } + + public List getSpaces() { + SpaceSearchCriteria criteria = new SpaceSearchCriteria(); + SpaceFetchOptions options = new SpaceFetchOptions(); + return applicationServer.searchSpaces(sessionToken, criteria, options).getObjects() + .stream().map(Space::getCode).collect(Collectors.toList()); + } + + public Map queryFullSampleHierarchy(List spaces) { + Map hierarchy = new HashMap<>(); + if(spaces.isEmpty()) { + spaces = getSpaces(); + } + for(String space : spaces) { + SampleFetchOptions fetchType = new SampleFetchOptions(); + fetchType.withType(); + SampleFetchOptions withDescendants = new SampleFetchOptions(); + withDescendants.withChildrenUsing(fetchType); + withDescendants.withType(); + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCode().thatEquals(space.toUpperCase()); + SearchResult result = applicationServer.searchSamples( + sessionToken, criteria, withDescendants); + for (Sample s : result.getObjects()) { + SampleType parentType = s.getType(); + List children = s.getChildren(); + if (children.isEmpty()) { + SampleTypeConnection leaf = new SampleTypeConnection(parentType); + if (hierarchy.containsKey(leaf)) { + int count = hierarchy.get(leaf) + 1; + hierarchy.put(leaf, count); + } else { + hierarchy.put(leaf, 1); + } + } else { + for (Sample c : children) { + SampleType childType = c.getType(); + SampleTypeConnection connection = new SampleTypeConnection(parentType, childType); + if (hierarchy.containsKey(connection)) { + int count = hierarchy.get(connection) + 1; + hierarchy.put(connection, count); + } else { + hierarchy.put(connection, 1); + } + } + } + } + } + return hierarchy; + } + + +} diff --git a/src/main/java/life/qbic/model/download/OutputPathFinder.java b/src/main/java/life/qbic/model/download/OutputPathFinder.java new file mode 100644 index 0000000..2dbb977 --- /dev/null +++ b/src/main/java/life/qbic/model/download/OutputPathFinder.java @@ -0,0 +1,83 @@ +package life.qbic.model.download; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Methods to determine the final path for the output directory. + * The requested data will be downloaded into this directory. + */ +public class OutputPathFinder { + + private static final Logger LOG = LogManager.getLogger(OutputPathFinder.class); + + /** + * @param path to be shortened + * @return path that has no parents (top directory) + */ + private static Path getTopDirectory(Path path) { + Path currentPath = Paths.get(path.toString()); + Path parentPath; + while (currentPath.getParent() != null) { + parentPath = currentPath.getParent(); + currentPath = parentPath; + } + return currentPath; + } + + /** + * @param possiblePath: string that could be an existing Path to a directory + * @return true if path exists, false otherwise + */ + private static boolean isPathValid(String possiblePath){ + Path path = Paths.get(possiblePath); + return Files.isDirectory(path); + } + + /** + * @param file to download + * @param conservePaths if true, directory structure will be conserved + * @return final path to file itself + */ + private static Path determineFinalPathFromDataset(DataSetFile file, Boolean conservePaths ) { + Path finalPath; + if (conservePaths) { + finalPath = Paths.get(file.getPath()); + // drop top parent directory name in the openBIS DSS (usually "/origin") + Path topDirectory = getTopDirectory(finalPath); + finalPath = topDirectory.relativize(finalPath); + } else { + finalPath = Paths.get(file.getPath()).getFileName(); + } + return finalPath; + } + + /** + * @param outputPath provided by user + * @param prefix sample code + * @param file to download + * @param conservePaths provided by user + * @return output directory path + */ + public static Path determineOutputDirectory(String outputPath, Path prefix, DataSetFile file, boolean conservePaths){ + Path filePath = determineFinalPathFromDataset(file, conservePaths); + String path = File.separator + prefix.toString() + File.separator + filePath.toString(); + Path finalPath = Paths.get(""); + if (outputPath != null && !outputPath.isEmpty()) { + if(isPathValid(outputPath)) { + finalPath = Paths.get(outputPath + path); + } else{ + LOG.error("The path you provided does not exist."); + System.exit(1); + } + } else { + finalPath = Paths.get(System.getProperty("user.dir") + path); + } + return finalPath; + } +} diff --git a/src/main/java/life/qbic/util/StringUtil.java b/src/main/java/life/qbic/util/StringUtil.java new file mode 100644 index 0000000..a2c8fa4 --- /dev/null +++ b/src/main/java/life/qbic/util/StringUtil.java @@ -0,0 +1,9 @@ +package life.qbic.util; + +public class StringUtil { + + public static boolean endsWithIgnoreCase(String input, String suffix) { + int suffixLength = suffix.length(); + return input.regionMatches(true, input.length() - suffixLength, suffix, 0, suffixLength); + } +} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 0000000..282f112 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + From 1f00c1727f53718f4487ad7c84c2484e87491f73 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 7 Mar 2024 18:08:43 +0100 Subject: [PATCH 02/63] update wfs --- .github/workflows/build-package.yml | 4 ++-- .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/create-release.yml | 4 ++-- .github/workflows/nexus-publish-snapshots.yml | 4 ++-- .github/workflows/run-tests.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index d616186..15172ce 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Load local Maven repository cache uses: actions/cache@v2 with: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 76c1d11..551579f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,10 +39,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 82efca7..058427c 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/.github/workflows/nexus-publish-snapshots.yml b/.github/workflows/nexus-publish-snapshots.yml index 4fc649b..979529c 100644 --- a/.github/workflows/nexus-publish-snapshots.yml +++ b/.github/workflows/nexus-publish-snapshots.yml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 settings-path: ${{ github.workspace }} - name: Load local Maven repository cache diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 5708a25..174cef2 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -14,10 +14,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Load local Maven repository cache uses: actions/cache@v2 From ac10137b2f7f472b7c195ee768dacda2e0bfa5e4 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 8 Mar 2024 13:59:07 +0100 Subject: [PATCH 03/63] adress review --- pom.xml | 2 +- src/main/java/life/qbic/App.java | 4 +--- src/main/resources/log4j2.xml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 59840c3..edf7d3f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ life.qbic openbis-scripts 1.0.0 - Postman cli + OpenBis Scripts https://github.com/qbicsoftware/openbis20-scripts A client software written in Java to query openBIS jar diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 6ef2b8a..95b79a8 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -1,6 +1,5 @@ package life.qbic; -import life.qbic.io.commandline.OpenBISPasswordParser; import life.qbic.io.commandline.CommandLineOptions; import life.qbic.model.Configuration; import life.qbic.model.download.Authentication; @@ -12,10 +11,9 @@ import java.io.File; import java.util.Arrays; -import java.util.Optional; /** - * postman for staging data from openBIS + * Scripts to perform different openBIS queries */ public class App { diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 282f112..750a990 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -4,7 +4,7 @@ - + From 16cc00b796117f823200cb836fc01d91bd28fcbd Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 11 Mar 2024 18:44:01 +0100 Subject: [PATCH 04/63] add more user friendly reporting --- .../commandline/SampleHierarchyCommand.java | 46 +++++++++++++++++-- .../qbic/model/download/FileSystemWriter.java | 30 ++++++++---- .../qbic/model/download/ModelReporter.java | 5 +- .../qbic/model/download/OpenbisConnector.java | 13 +----- 4 files changed, 68 insertions(+), 26 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index c0b1a92..b36af10 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -1,12 +1,22 @@ package life.qbic.io.commandline; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import life.qbic.App; +import life.qbic.model.Configuration; import life.qbic.model.SampleTypeConnection; import life.qbic.model.download.Authentication; +import life.qbic.model.download.FileSystemWriter; +import life.qbic.model.download.ModelReporter; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -18,20 +28,48 @@ public class SampleHierarchyCommand implements Runnable { @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) private String space; + @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) + private String outpath; @Mixin AuthenticationOptions auth = new AuthenticationOptions(); @Override public void run() { - List spaces = new ArrayList<>(); + List summary = new ArrayList<>(); + List spaces = new ArrayList<>(); if(space!=null) { + summary.add("Querying samples in space: "+space+"...\n"); spaces.add(space); + } else { + summary.add("Querying samples in all available spaces...\n"); } Authentication authentication = App.loginToOpenBIS(auth.getPassword(), auth.user, auth.as_url); - OpenbisConnector openbis = new OpenbisConnector(auth.as_url, authentication.getSessionToken(), "/Users/afriedrich/Downloads/"); - Map hierarchy = openbis.queryFullSampleHierarchy(spaces); + OpenbisConnector openbis = new OpenbisConnector(auth.as_url, authentication.getSessionToken()); + Map hierarchy = openbis.queryFullSampleHierarchy(spaces); + hierarchy.entrySet().stream() .sorted(Entry.comparingByValue()) - .forEach(entry -> System.err.println(entry.getKey()+" ("+entry.getValue()+")")); + .forEach(entry -> summary.add(entry.getKey()+" ("+entry.getValue()+")")); + + for(String s : summary) { + System.out.println(s); + } + Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), + "summary_model_"+getTimeStamp()+".txt"); + if(outpath!=null) { + outputPath = Paths.get(outpath); + } + ModelReporter modelReporter = new FileSystemWriter(outputPath); + try { + modelReporter.reportSummary(summary); + } catch (IOException e) { + throw new RuntimeException("Could not write summary file."); + } + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter).toString(); } } diff --git a/src/main/java/life/qbic/model/download/FileSystemWriter.java b/src/main/java/life/qbic/model/download/FileSystemWriter.java index 7a09c63..4cf0cb9 100644 --- a/src/main/java/life/qbic/model/download/FileSystemWriter.java +++ b/src/main/java/life/qbic/model/download/FileSystemWriter.java @@ -1,6 +1,9 @@ package life.qbic.model.download; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -11,29 +14,40 @@ * * @author: Sven Fillinger, Andreas Friedrich */ -class FileSystemWriter implements ModelReporter { +public class FileSystemWriter implements ModelReporter { /** * File that stores the summary report content for valid checksums. */ - final private File matchingSummaryFile; + final private File summaryFile; /** - * FileSystemWriter constructor with the paths for the summary files. * + * FileSystemWriter constructor with the paths for the summary files. * - * @param matchingSummaryFile The path where to write the summary + * @param summaryFile The path where to write the summary */ - FileSystemWriter(Path matchingSummaryFile) { - this.matchingSummaryFile = new File(matchingSummaryFile.toString()); + public FileSystemWriter(Path summaryFile) { + this.summaryFile = new File(summaryFile.toString()); } /** * {@inheritDoc} */ @Override - public void reportSummary(List summary) { - //TODO + public void reportSummary(List summary) throws IOException { + if (!summaryFile.exists()) { + summaryFile.createNewFile(); + //file exists or could not be created + if (!summaryFile.exists()) { + throw new IOException("The file " + summaryFile.getAbsoluteFile() + " could not be created."); + } + } + BufferedWriter writer = new BufferedWriter(new FileWriter(summaryFile, true)); + for(String line : summary) { + writer.append(line+"\n"); + } + writer.close(); } } diff --git a/src/main/java/life/qbic/model/download/ModelReporter.java b/src/main/java/life/qbic/model/download/ModelReporter.java index 2e453eb..875ffb4 100644 --- a/src/main/java/life/qbic/model/download/ModelReporter.java +++ b/src/main/java/life/qbic/model/download/ModelReporter.java @@ -1,10 +1,11 @@ package life.qbic.model.download; +import java.io.IOException; import java.net.URL; import java.nio.file.Path; import java.util.List; -interface ModelReporter { +public interface ModelReporter { - void reportSummary(List summary); + void reportSummary(List summary) throws IOException; } diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index e5dc841..f91bf8e 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -10,14 +10,10 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; import ch.systemsx.cisd.common.spring.HttpInvokerUtils; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import life.qbic.App; -import life.qbic.model.Configuration; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -27,11 +23,6 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); private final IApplicationServerApi applicationServer; private final String sessionToken; - private final String outputPath; - - private final ModelReporter modelReporter = new FileSystemWriter( - Paths.get(Configuration.LOG_PATH.toString(), "summary_model.txt")); - /** * Constructor for a QBiCDataDownloader instance * @@ -40,10 +31,8 @@ public class OpenbisConnector { */ public OpenbisConnector( String AppServerUri, - String sessionToken, - String outputPath) { + String sessionToken) { this.sessionToken = sessionToken; - this.outputPath = outputPath; if (!AppServerUri.isEmpty()) { applicationServer = From 757a4c8dacf1d771b99113e0e1950a33e5258e6a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 12 Mar 2024 10:41:45 +0100 Subject: [PATCH 05/63] fix codeql and deployment --- .github/workflows/codeql-analysis.yml | 78 +++++++++++++-------------- pom.xml | 76 ++++++++++++++++++-------- 2 files changed, 92 insertions(+), 62 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 551579f..7951f13 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -22,7 +22,6 @@ on: jobs: analyze: - name: analyze runs-on: ubuntu-latest permissions: actions: read @@ -37,47 +36,48 @@ jobs: # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - settings-path: ${{ github.workspace }} + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '11' + settings-path: ${{ github.workspace }} - - name: Load local Maven repository cache - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + - name: Load local Maven repository cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # ℹ️ Command-line programs to run using the OS shell. - # πŸ“š https://git.io/JvXDl + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # ℹ️ Command-line programs to run using the OS shell. + # πŸ“š https://git.io/JvXDl - #- run: | - # make bootstrap - # make release + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/pom.xml b/pom.xml index edf7d3f..28f0176 100644 --- a/pom.xml +++ b/pom.xml @@ -17,31 +17,61 @@ 5.3.31 - - - - - central - https://repo.maven.apache.org/maven2 - - false - - - - nexus-releases - QBiC Releases - - fail - true - always - - - false - - https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases - + + + true + always + fail + + + false + + maven-central + Maven central + https://repo.maven.apache.org/maven2 + + + + false + + + true + always + fail + + nexus-snapshots + QBiC Snapshots + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots + + + + true + always + fail + + + false + + nexus-releases + QBiC Releases + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases + + + + true + nexus-releases + QBiC Releases + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-releases + + + false + nexus-snapshots + QBiC Snapshots + https://qbic-repo.qbic.uni-tuebingen.de/repository/maven-snapshots + + From 7656e04aeb9cfd6fb1c5a7a3035d837f6cdf6729 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Sun, 24 Mar 2024 16:21:21 +0100 Subject: [PATCH 06/63] use openbis-internal wrapper class --- src/main/java/life/qbic/App.java | 14 ++++++-------- .../io/commandline/SampleHierarchyCommand.java | 5 +++-- .../qbic/model/download/OpenbisConnector.java | 18 +++++++++++------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 95b79a8..aa142b5 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -1,5 +1,6 @@ package life.qbic; +import ch.ethz.sis.openbis.generic.OpenBIS; import life.qbic.io.commandline.CommandLineOptions; import life.qbic.model.Configuration; import life.qbic.model.download.Authentication; @@ -39,8 +40,8 @@ private static Boolean isNotNullOrEmpty(String envVariableCommandLineParameter) * * @return An instance of the Authentication class. */ - public static Authentication loginToOpenBIS( - char[] password, String user, String as_url) { + public static OpenBIS loginToOpenBIS( + char[] password, String user, String url) { // Ensure 'logs' folder is created File logFolder = new File(Configuration.LOG_PATH.toAbsolutePath().toString()); @@ -52,13 +53,10 @@ public static Authentication loginToOpenBIS( } } - Authentication authentication = - new Authentication( - user, - new String(password), - as_url); + OpenBIS authentication = + new OpenBIS(url); try { - authentication.login(); + authentication.login(user, new String(password)); } catch (ConnectionException e) { LOG.error(e.getMessage(), e); LOG.error("Could not connect to QBiC's data source. Have you requested access to the " diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index b36af10..1b01349 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -1,5 +1,6 @@ package life.qbic.io.commandline; +import ch.ethz.sis.openbis.generic.OpenBIS; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -43,8 +44,8 @@ public void run() { } else { summary.add("Querying samples in all available spaces...\n"); } - Authentication authentication = App.loginToOpenBIS(auth.getPassword(), auth.user, auth.as_url); - OpenbisConnector openbis = new OpenbisConnector(auth.as_url, authentication.getSessionToken()); + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.user, auth.as_url); + OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); hierarchy.entrySet().stream() diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index f91bf8e..3536415 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -1,6 +1,6 @@ package life.qbic.model.download; -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; +import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; @@ -9,7 +9,6 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; -import ch.systemsx.cisd.common.spring.HttpInvokerUtils; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -21,8 +20,7 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); - private final IApplicationServerApi applicationServer; - private final String sessionToken; + OpenBIS openBIS; /** * Constructor for a QBiCDataDownloader instance * @@ -32,6 +30,7 @@ public class OpenbisConnector { public OpenbisConnector( String AppServerUri, String sessionToken) { + /* this.sessionToken = sessionToken; if (!AppServerUri.isEmpty()) { @@ -41,12 +40,18 @@ public OpenbisConnector( } else { applicationServer = null; } + + */ + } + + public OpenbisConnector(OpenBIS authentication) { + this.openBIS = authentication; } public List getSpaces() { SpaceSearchCriteria criteria = new SpaceSearchCriteria(); SpaceFetchOptions options = new SpaceFetchOptions(); - return applicationServer.searchSpaces(sessionToken, criteria, options).getObjects() + return openBIS.searchSpaces(criteria, options).getObjects() .stream().map(Space::getCode).collect(Collectors.toList()); } @@ -63,8 +68,7 @@ public Map queryFullSampleHierarchy(List withDescendants.withType(); SampleSearchCriteria criteria = new SampleSearchCriteria(); criteria.withSpace().withCode().thatEquals(space.toUpperCase()); - SearchResult result = applicationServer.searchSamples( - sessionToken, criteria, withDescendants); + SearchResult result = openBIS.searchSamples(criteria, withDescendants); for (Sample s : result.getObjects()) { SampleType parentType = s.getType(); List children = s.getChildren(); From 2f525bd4ab38e49060c086d79be5248a526ef4d4 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 27 Jun 2024 18:01:20 +0200 Subject: [PATCH 07/63] add data search and upload --- pom.xml | 8 ++ src/main/java/life/qbic/App.java | 46 ++++-- .../io/commandline/AuthenticationOptions.java | 82 ++++++++++- .../io/commandline/CommandLineOptions.java | 6 +- .../commandline/DownloadDatasetCommand.java | 68 +++++++++ .../io/commandline/FindDatasetsCommand.java | 70 +++++++++ .../commandline/SampleHierarchyCommand.java | 3 +- .../io/commandline/UploadDatasetCommand.java | 81 +++++++++++ .../qbic/model/download/OpenbisConnector.java | 136 ++++++++++++++++++ 9 files changed, 479 insertions(+), 21 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java create mode 100644 src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java create mode 100644 src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java diff --git a/pom.xml b/pom.xml index 28f0176..0cbf3be 100644 --- a/pom.xml +++ b/pom.xml @@ -138,6 +138,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 11 + 11 + + diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index aa142b5..a100f95 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -3,7 +3,6 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import life.qbic.io.commandline.CommandLineOptions; import life.qbic.model.Configuration; -import life.qbic.model.download.Authentication; import life.qbic.model.download.AuthenticationException; import life.qbic.model.download.ConnectionException; import org.apache.logging.log4j.LogManager; @@ -36,25 +35,34 @@ private static Boolean isNotNullOrEmpty(String envVariableCommandLineParameter) } /** - * Logs into OpenBIS asks for and verifies password. + * Logs into OpenBIS, asks for and verifies password. * * @return An instance of the Authentication class. */ public static OpenBIS loginToOpenBIS( char[] password, String user, String url) { + //setupLog(); - // Ensure 'logs' folder is created - File logFolder = new File(Configuration.LOG_PATH.toAbsolutePath().toString()); - if (!logFolder.exists()) { - boolean logFolderCreated = logFolder.mkdirs(); - if (!logFolderCreated) { - LOG.error("Could not create log folder '" + logFolder.getAbsolutePath() + "'"); - System.exit(1); - } - } + OpenBIS authentication = new OpenBIS(url); + + return tryLogin(authentication, user, password); + } - OpenBIS authentication = - new OpenBIS(url); + /** + * Logs into OpenBIS, asks for and verifies password, includes Datastore Server connection. + * + * @return An instance of the Authentication class. + */ + public static OpenBIS loginToOpenBIS( + char[] password, String user, String url, String dssUrl) { + //setupLog(); + + OpenBIS authentication = new OpenBIS(url, dssUrl); + + return tryLogin(authentication, user, password); + } + + private static OpenBIS tryLogin(OpenBIS authentication, String user, char[] password) { try { authentication.login(user, new String(password)); } catch (ConnectionException e) { @@ -68,4 +76,16 @@ public static OpenBIS loginToOpenBIS( } return authentication; } + + private static void setupLog() { + // Ensure 'logs' folder is created + File logFolder = new File(Configuration.LOG_PATH.toAbsolutePath().toString()); + if (!logFolder.exists()) { + boolean logFolderCreated = logFolder.mkdirs(); + if (!logFolderCreated) { + LOG.error("Could not create log folder '" + logFolder.getAbsolutePath() + "'"); + System.exit(1); + } + } + } } diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 8f49d7a..795fc33 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -3,7 +3,13 @@ import static java.util.Objects.nonNull; import static picocli.CommandLine.ArgGroup; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; import java.util.StringJoiner; +import java.util.TreeMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import picocli.CommandLine; @@ -14,9 +20,8 @@ public class AuthenticationOptions { @Option( names = {"-u", "--user"}, - required = true, description = "openBIS user name") - public String user; + private String user; @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. PasswordOptions passwordOptions; @@ -24,7 +29,40 @@ public class AuthenticationOptions { names = {"-as", "-as_url"}, description = "ApplicationServer URL", scope = CommandLine.ScopeType.INHERIT) - public String as_url; + private String as_url; + + @Option( + names = {"-dss", "-dss_url"}, + description = "DatastoreServer URL", + scope = CommandLine.ScopeType.INHERIT) + private String dss_url; + + @Option( + names = {"-config", "-config_file"}, + description = "Config file path to provide openbis server information.", + scope = CommandLine.ScopeType.INHERIT) + public String configPath; + + public String getUser() { + if(user == null & configPath!=null && !configPath.isBlank()) { + user = ReadProperties.getProperties(configPath).get("user"); + } + return user; + } + + public String getDSS() { + if(dss_url == null & configPath!=null && !configPath.isBlank()) { + dss_url = ReadProperties.getProperties(configPath).get("dss"); + } + return dss_url; + } + + public String getAS() { + if(as_url == null & configPath!=null && !configPath.isBlank()) { + as_url = ReadProperties.getProperties(configPath).get("as"); + } + return as_url; + } public char[] getPassword() { return passwordOptions.getPassword(); @@ -76,4 +114,42 @@ public String toString() { .toString(); //ATTENTION: do not expose the password here! } + + public static class ReadProperties { + + public static TreeMap getProperties(String infile) { + final int lhs = 0; + final int rhs = 1; + + TreeMap map = new TreeMap(); + BufferedReader bfr = null; + try { + bfr = new BufferedReader(new FileReader(new File(infile))); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + + String line; + while (true) { + try { + if (!((line = bfr.readLine()) != null)) + break; + } catch (IOException e) { + throw new RuntimeException(e); + } + if (!line.startsWith("#") && !line.isEmpty()) { + String[] pair = line.trim().split("="); + map.put(pair[lhs].trim(), pair[rhs].trim()); + } + } + + try { + bfr.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return(map); + } + } } \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index 315dd85..3acf00a 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,9 +8,9 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class }, + subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, UploadDatasetCommand.class }, description = "A client software for querying openBIS.", - mixinStandardHelpOptions = true) + mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); @@ -18,7 +18,7 @@ public class CommandLineOptions { versionHelp = true, description = "print version information", scope = CommandLine.ScopeType.INHERIT) - boolean versionRequested; + boolean versionRequested = false; @Option( names = {"-h", "--help"}, diff --git a/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java new file mode 100644 index 0000000..885a517 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java @@ -0,0 +1,68 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import life.qbic.App; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "list-data", + description = "lists datasets and their details for a given experiment code") +public class DownloadDatasetCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "experiment", description = "The code of the experiment data is attached to") + private String experimentCode; + @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter results", names = {"-s", "--space"}) + private String space; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + List spaces = new ArrayList<>(); + if (space != null) { + System.out.println("Querying experiment in space: " + space + "..."); + spaces.add(space); + } else { + System.out.println("Querying experiment in all available spaces..."); + } + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() + .sorted(Comparator.comparing( + (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( + Collectors.toList()); + int i = 0; + for (DataSet dataSet : datasets) { + i++; + System.out.println("["+i+"]"); + System.out.println(dataSet.getExperiment().getIdentifier()); + System.out.println(dataSet.getCode()); + System.out.println(dataSet.getType().getCode()); + System.out.println(dataSet.getRegistrationDate()); + System.out.println(new SimpleDateFormat("MM-dd-yyyy").format(dataSet.getRegistrationDate())); + Person person = dataSet.getRegistrator(); + System.out.println(person.getFirstName() + " " + person.getLastName()); + System.out.println(); + } + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); + } +} diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java new file mode 100644 index 0000000..ab08afe --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -0,0 +1,70 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import life.qbic.App; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "list-data", + description = "lists datasets and their details for a given experiment code") +public class FindDatasetsCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "experiment", description = "The code of the experiment data is attached to") + private String experimentCode; + @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter samples", names = {"-s", "--space"}) + private String space; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + List spaces = new ArrayList<>(); + if (space != null) { + System.out.println("Querying experiment in space: " + space + "..."); + spaces.add(space); + } else { + System.out.println("Querying experiment in all available spaces..."); + } + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() + .sorted(Comparator.comparing( + (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( + Collectors.toList()); + int i = 0; + System.out.println(); + System.out.printf("Found %s datasets for experiment %s:%n", datasets.size(), experimentCode); + for (DataSet dataSet : datasets) { + i++; + System.out.println("["+i+"]"); + System.out.printf("ID: %s (%s)%n", dataSet.getCode(), dataSet.getExperiment().getIdentifier()); + System.out.println("Type: "+dataSet.getType().getCode()); + Person person = dataSet.getRegistrator(); + String simpleTime = new SimpleDateFormat("MM-dd-yy HH:mm:ss").format(dataSet.getRegistrationDate()); + String name = person.getFirstName() +" "+ person.getLastName(); + String uploadedBy = "Uploaded by "+name+" ("+simpleTime+")"; + System.out.println(uploadedBy); + System.out.println(); + } + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); + } +} diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 1b01349..e9668d6 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -15,7 +15,6 @@ import life.qbic.App; import life.qbic.model.Configuration; import life.qbic.model.SampleTypeConnection; -import life.qbic.model.download.Authentication; import life.qbic.model.download.FileSystemWriter; import life.qbic.model.download.ModelReporter; import life.qbic.model.download.OpenbisConnector; @@ -44,7 +43,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces...\n"); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.user, auth.as_url); + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java new file mode 100644 index 0000000..a3cd462 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -0,0 +1,81 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import java.io.File; +import java.nio.file.Path; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import life.qbic.App; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "upload-data", + description = "uploads a dataset and attaches it to an experiment and (optionally) other datasets") +public class UploadDatasetCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") + private String dataPath; + @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. " + + "The identifier must be of the format: /space/project/experiment") + private String experimentID; + @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) + private List parents = new ArrayList<>(); + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + private OpenbisConnector openbis; + + @Override + public void run() { + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + openbis = new OpenbisConnector(authentication); + + if(!pathValid(dataPath)) { + System.out.printf("Path %s could not be found%n", dataPath); + return; + } + if(!experimentExists(experimentID)) { + System.out.printf("Experiment %s could not be found%n", experimentID); + return; + } + if(!datasetsExist(parents)) { + System.out.printf("One or more datasets %s could not be found%n", parents); + return; + } + System.out.println(); + System.out.println("Parameters verified, uploading dataset..."); + System.out.println(); + DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); + System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + } + + private boolean datasetsExist(List parents) { + if(parents.isEmpty()) { + return true; + } + return openbis.findDataSets(parents).size() == parents.size(); + } + + private boolean experimentExists(String experimentID) { + return openbis.experimentExists(experimentID); + } + + private boolean pathValid(String dataPath) { + return new File(dataPath).exists(); + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 3536415..844aa9b 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -2,6 +2,15 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; @@ -9,6 +18,20 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.dataset.create.UploadedDataSetCreation; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownload; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.DataSetFilePermId; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,6 +78,107 @@ public List getSpaces() { .stream().map(Space::getCode).collect(Collectors.toList()); } + public DataSetPermId registerDataset(Path uploadPath, String experimentID, List parentCodes) { + final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); + + final UploadedDataSetCreation creation = new UploadedDataSetCreation(); + creation.setUploadId(uploadId); + creation.setExperimentId(new ExperimentIdentifier(experimentID)); + creation.setParentIds(parentCodes.stream().map(DataSetPermId::new).collect( + Collectors.toList())); + creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); + + try + { + return openBIS.createUploadedDataSet(creation); + } catch (final Exception e) + { + e.printStackTrace(); + } + return null; + } + + public static void main(String[] args) throws IOException { + String as = ""; + String dss = ""; + String user = ""; + String pass = ""; + + OpenBIS authentication = + new OpenBIS(as, dss); + authentication.login(user, pass); + OpenbisConnector c = new OpenbisConnector(authentication); + + String space = "TEMP_PLAYGROUND"; + String project = "TEMP_PLAYGROUND"; + String experiment = "E123"; + String expID = "/TEMP_PLAYGROUND/TEMP_PLAYGROUND/E123"; + Path toUpload = Path.of("/Users/afriedrich/Downloads/cats"); + //c.registerDataset(toUpload, expID, List.of("20240512182451335-425180", "20240512191248297-425181")); + + String basePath = "/Users/afriedrich/Downloads/downloaded_cats"; + //c.downloadDataset(basePath, "20240503175717814-263844"); + c.listDatasetsOfExperiment(List.of(), experiment); + + } + + +private static void copyInputStreamToFile(InputStream inputStream, File file) + throws IOException { + try (FileOutputStream outputStream = new FileOutputStream(file, false)) { + int read; + byte[] bytes = new byte[8192]; + while ((read = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + } +} + +public List listDatasetsOfExperiment(List spaces, String experiment) { + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withExperiment().withCode().thatEquals(experiment); + if(!spaces.isEmpty()) { + criteria.withAndOperator(); + criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + } + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withType(); + options.withRegistrator(); + options.withExperiment().withProject().withSpace(); + return openBIS.searchDataSets(criteria, options).getObjects(); +} + + public void downloadDataset(String targetPath, String datasetID) throws IOException { + DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); + IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), + "");//TODO test this path + + // Setting recursive flag to true will return both the subfolder directory object AND file3.txt + options.setRecursive(true); + + // Setting recursive flag to false will return just the meta data of the directory object + //options.setRecursive(false); + + // Read the contents and print them out + InputStream stream = openBIS.downloadFiles(new ArrayList<>(Arrays.asList(fileToDownload)), options); + DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); + DataSetFileDownload file = null; + while ((file = reader.read()) != null) + { + DataSetFile df = file.getDataSetFile(); + String currentPath = df.getPath().replace("original",""); + if(df.isDirectory()) { + File newDeer = new File(targetPath, currentPath); + if (!newDeer.exists()) { + newDeer.mkdirs(); + } + } else { + File toWrite = new File(targetPath, currentPath); + copyInputStreamToFile(file.getInputStream(), toWrite); + } + } + } + public Map queryFullSampleHierarchy(List spaces) { Map hierarchy = new HashMap<>(); if(spaces.isEmpty()) { @@ -97,5 +221,17 @@ public Map queryFullSampleHierarchy(List return hierarchy; } + public List findDataSets(List codes) { + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withCodes().thatIn(codes); + DataSetFetchOptions options = new DataSetFetchOptions(); + return openBIS.searchDataSets(criteria, options).getObjects(); + } + + public boolean experimentExists(String experimentID) { + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withIdentifier().thatEquals(experimentID); + return !openBIS.searchExperiments(criteria, new ExperimentFetchOptions()).getObjects().isEmpty(); + } } From e56f4b8e07f5405fe165dc8018425393c684d026 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 28 Jun 2024 11:35:51 +0200 Subject: [PATCH 08/63] round 1 of review changes --- src/main/java/life/qbic/App.java | 4 +-- .../io/commandline/AuthenticationOptions.java | 12 +++---- .../io/commandline/FindDatasetsCommand.java | 6 ++-- .../io/commandline/UploadDatasetCommand.java | 7 ++-- .../qbic/model/download/OpenbisConnector.java | 34 ++++--------------- 5 files changed, 18 insertions(+), 45 deletions(-) diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index a100f95..6ad90a6 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -41,7 +41,7 @@ private static Boolean isNotNullOrEmpty(String envVariableCommandLineParameter) */ public static OpenBIS loginToOpenBIS( char[] password, String user, String url) { - //setupLog(); + setupLog(); OpenBIS authentication = new OpenBIS(url); @@ -55,7 +55,7 @@ public static OpenBIS loginToOpenBIS( */ public static OpenBIS loginToOpenBIS( char[] password, String user, String url, String dssUrl) { - //setupLog(); + setupLog(); OpenBIS authentication = new OpenBIS(url, dssUrl); diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 795fc33..948ef7e 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -118,10 +118,8 @@ public String toString() { public static class ReadProperties { public static TreeMap getProperties(String infile) { - final int lhs = 0; - final int rhs = 1; - TreeMap map = new TreeMap(); + TreeMap properties = new TreeMap<>(); BufferedReader bfr = null; try { bfr = new BufferedReader(new FileReader(new File(infile))); @@ -132,14 +130,14 @@ public static TreeMap getProperties(String infile) { String line; while (true) { try { - if (!((line = bfr.readLine()) != null)) + if ((line = bfr.readLine()) == null) break; } catch (IOException e) { throw new RuntimeException(e); } if (!line.startsWith("#") && !line.isEmpty()) { - String[] pair = line.trim().split("="); - map.put(pair[lhs].trim(), pair[rhs].trim()); + String[] property = line.trim().split("="); + properties.put(property[0].trim(), property[1].trim()); } } @@ -149,7 +147,7 @@ public static TreeMap getProperties(String infile) { throw new RuntimeException(e); } - return(map); + return(properties); } } } \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index ab08afe..82c2a18 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -45,12 +45,12 @@ public void run() { .sorted(Comparator.comparing( (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( Collectors.toList()); - int i = 0; + int datasetIndex = 0; System.out.println(); System.out.printf("Found %s datasets for experiment %s:%n", datasets.size(), experimentCode); for (DataSet dataSet : datasets) { - i++; - System.out.println("["+i+"]"); + datasetIndex++; + System.out.println("["+datasetIndex+"]"); System.out.printf("ID: %s (%s)%n", dataSet.getCode(), dataSet.getExperiment().getIdentifier()); System.out.println("Type: "+dataSet.getType().getCode()); Person person = dataSet.getRegistrator(); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index a3cd462..9ea557e 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -58,11 +58,8 @@ public void run() { System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } - private boolean datasetsExist(List parents) { - if(parents.isEmpty()) { - return true; - } - return openbis.findDataSets(parents).size() == parents.size(); + private boolean datasetsExist(List datasetCodes) { + return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); } private boolean experimentExists(String experimentID) { diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 844aa9b..1e82adf 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -98,31 +98,6 @@ public DataSetPermId registerDataset(Path uploadPath, String experimentID, List< return null; } - public static void main(String[] args) throws IOException { - String as = ""; - String dss = ""; - String user = ""; - String pass = ""; - - OpenBIS authentication = - new OpenBIS(as, dss); - authentication.login(user, pass); - OpenbisConnector c = new OpenbisConnector(authentication); - - String space = "TEMP_PLAYGROUND"; - String project = "TEMP_PLAYGROUND"; - String experiment = "E123"; - String expID = "/TEMP_PLAYGROUND/TEMP_PLAYGROUND/E123"; - Path toUpload = Path.of("/Users/afriedrich/Downloads/cats"); - //c.registerDataset(toUpload, expID, List.of("20240512182451335-425180", "20240512191248297-425181")); - - String basePath = "/Users/afriedrich/Downloads/downloaded_cats"; - //c.downloadDataset(basePath, "20240503175717814-263844"); - c.listDatasetsOfExperiment(List.of(), experiment); - - } - - private static void copyInputStreamToFile(InputStream inputStream, File file) throws IOException { try (FileOutputStream outputStream = new FileOutputStream(file, false)) { @@ -168,9 +143,9 @@ public void downloadDataset(String targetPath, String datasetID) throws IOExcept DataSetFile df = file.getDataSetFile(); String currentPath = df.getPath().replace("original",""); if(df.isDirectory()) { - File newDeer = new File(targetPath, currentPath); - if (!newDeer.exists()) { - newDeer.mkdirs(); + File newDir = new File(targetPath, currentPath); + if (!newDir.exists()) { + newDir.mkdirs(); } } else { File toWrite = new File(targetPath, currentPath); @@ -222,6 +197,9 @@ public Map queryFullSampleHierarchy(List } public List findDataSets(List codes) { + if(codes.isEmpty()) { + return new ArrayList<>(); + } DataSetSearchCriteria criteria = new DataSetSearchCriteria(); criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); From db4d888cde38b6210b394a261f751253f36c0ab8 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 28 Jun 2024 13:18:58 +0200 Subject: [PATCH 09/63] test and remove todos --- .../qbic/model/download/OpenbisConnector.java | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 1e82adf..f5e1654 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -44,15 +44,16 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); OpenBIS openBIS; + /** * Constructor for a QBiCDataDownloader instance * - * @param AppServerUri The openBIS application server URL (AS) + * @param AppServerUri The openBIS application server URL (AS) * @param sessionToken The session token for the datastore & application servers */ public OpenbisConnector( - String AppServerUri, - String sessionToken) { + String AppServerUri, + String sessionToken) { /* this.sessionToken = sessionToken; @@ -78,7 +79,8 @@ public List getSpaces() { .stream().map(Space::getCode).collect(Collectors.toList()); } - public DataSetPermId registerDataset(Path uploadPath, String experimentID, List parentCodes) { + public DataSetPermId registerDataset(Path uploadPath, String experimentID, + List parentCodes) { final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); final UploadedDataSetCreation creation = new UploadedDataSetCreation(); @@ -88,78 +90,75 @@ public DataSetPermId registerDataset(Path uploadPath, String experimentID, List< Collectors.toList())); creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); - try - { + try { return openBIS.createUploadedDataSet(creation); - } catch (final Exception e) - { - e.printStackTrace(); + } catch (final Exception e) { + LOG.error(e.getMessage()); } return null; } -private static void copyInputStreamToFile(InputStream inputStream, File file) - throws IOException { - try (FileOutputStream outputStream = new FileOutputStream(file, false)) { - int read; - byte[] bytes = new byte[8192]; - while ((read = inputStream.read(bytes)) != -1) { - outputStream.write(bytes, 0, read); + private static void copyInputStreamToFile(InputStream inputStream, File file) + throws IOException { + System.err.println(file.getPath()); + try (FileOutputStream outputStream = new FileOutputStream(file, false)) { + int read; + byte[] bytes = new byte[8192]; + while ((read = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } } } -} -public List listDatasetsOfExperiment(List spaces, String experiment) { - DataSetSearchCriteria criteria = new DataSetSearchCriteria(); - criteria.withExperiment().withCode().thatEquals(experiment); - if(!spaces.isEmpty()) { - criteria.withAndOperator(); - criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + public List listDatasetsOfExperiment(List spaces, String experiment) { + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withExperiment().withCode().thatEquals(experiment); + if (!spaces.isEmpty()) { + criteria.withAndOperator(); + criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + } + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withType(); + options.withRegistrator(); + options.withExperiment().withProject().withSpace(); + return openBIS.searchDataSets(criteria, options).getObjects(); } - DataSetFetchOptions options = new DataSetFetchOptions(); - options.withType(); - options.withRegistrator(); - options.withExperiment().withProject().withSpace(); - return openBIS.searchDataSets(criteria, options).getObjects(); -} public void downloadDataset(String targetPath, String datasetID) throws IOException { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), - "");//TODO test this path + ""); - // Setting recursive flag to true will return both the subfolder directory object AND file3.txt + System.err.println(fileToDownload); + // Setting recursive flag to true will return both subfolders and files options.setRecursive(true); - // Setting recursive flag to false will return just the meta data of the directory object - //options.setRecursive(false); - // Read the contents and print them out - InputStream stream = openBIS.downloadFiles(new ArrayList<>(Arrays.asList(fileToDownload)), options); + InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), + options); DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); DataSetFileDownload file = null; - while ((file = reader.read()) != null) - { + while ((file = reader.read()) != null) { DataSetFile df = file.getDataSetFile(); - String currentPath = df.getPath().replace("original",""); - if(df.isDirectory()) { - File newDir = new File(targetPath, currentPath); - if (!newDir.exists()) { - newDir.mkdirs(); - } + String currentPath = df.getPath().replace("original", ""); + if (df.isDirectory()) { + File newDir = new File(targetPath, currentPath); + if (!newDir.exists()) { + newDir.mkdirs(); + } } else { File toWrite = new File(targetPath, currentPath); copyInputStreamToFile(file.getInputStream(), toWrite); } } - } + } public Map queryFullSampleHierarchy(List spaces) { Map hierarchy = new HashMap<>(); - if(spaces.isEmpty()) { + if (spaces.isEmpty()) { spaces = getSpaces(); } - for(String space : spaces) { + for (String space : spaces) { SampleFetchOptions fetchType = new SampleFetchOptions(); fetchType.withType(); SampleFetchOptions withDescendants = new SampleFetchOptions(); @@ -169,27 +168,27 @@ public Map queryFullSampleHierarchy(List criteria.withSpace().withCode().thatEquals(space.toUpperCase()); SearchResult result = openBIS.searchSamples(criteria, withDescendants); for (Sample s : result.getObjects()) { - SampleType parentType = s.getType(); - List children = s.getChildren(); - if (children.isEmpty()) { - SampleTypeConnection leaf = new SampleTypeConnection(parentType); - if (hierarchy.containsKey(leaf)) { - int count = hierarchy.get(leaf) + 1; - hierarchy.put(leaf, count); - } else { - hierarchy.put(leaf, 1); - } + SampleType parentType = s.getType(); + List children = s.getChildren(); + if (children.isEmpty()) { + SampleTypeConnection leaf = new SampleTypeConnection(parentType); + if (hierarchy.containsKey(leaf)) { + int count = hierarchy.get(leaf) + 1; + hierarchy.put(leaf, count); } else { - for (Sample c : children) { - SampleType childType = c.getType(); - SampleTypeConnection connection = new SampleTypeConnection(parentType, childType); - if (hierarchy.containsKey(connection)) { - int count = hierarchy.get(connection) + 1; - hierarchy.put(connection, count); - } else { - hierarchy.put(connection, 1); - } + hierarchy.put(leaf, 1); + } + } else { + for (Sample c : children) { + SampleType childType = c.getType(); + SampleTypeConnection connection = new SampleTypeConnection(parentType, childType); + if (hierarchy.containsKey(connection)) { + int count = hierarchy.get(connection) + 1; + hierarchy.put(connection, count); + } else { + hierarchy.put(connection, 1); } + } } } } @@ -197,7 +196,7 @@ public Map queryFullSampleHierarchy(List } public List findDataSets(List codes) { - if(codes.isEmpty()) { + if (codes.isEmpty()) { return new ArrayList<>(); } DataSetSearchCriteria criteria = new DataSetSearchCriteria(); @@ -210,6 +209,7 @@ public boolean experimentExists(String experimentID) { ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); criteria.withIdentifier().thatEquals(experimentID); - return !openBIS.searchExperiments(criteria, new ExperimentFetchOptions()).getObjects().isEmpty(); + return !openBIS.searchExperiments(criteria, new ExperimentFetchOptions()).getObjects() + .isEmpty(); } } From 3a99d0d4ae49a63bb968754a228c719cd1de7765 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 28 Jun 2024 13:57:18 +0200 Subject: [PATCH 10/63] remaining fixes --- pom.xml | 7 +++---- .../life/qbic/io/commandline/DownloadDatasetCommand.java | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 0cbf3be..19f2587 100644 --- a/pom.xml +++ b/pom.xml @@ -11,8 +11,7 @@ A client software written in Java to query openBIS jar - 11 - 11 + 11 UTF-8 5.3.31 @@ -142,8 +141,8 @@ org.apache.maven.plugins maven-compiler-plugin - 11 - 11 + ${java.version} + ${java.version} diff --git a/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java index 885a517..102baff 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java @@ -45,10 +45,10 @@ public void run() { .sorted(Comparator.comparing( (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( Collectors.toList()); - int i = 0; + int datasetIndex = 0; for (DataSet dataSet : datasets) { - i++; - System.out.println("["+i+"]"); + datasetIndex++; + System.out.println("["+datasetIndex+"]"); System.out.println(dataSet.getExperiment().getIdentifier()); System.out.println(dataSet.getCode()); System.out.println(dataSet.getType().getCode()); From a3b0572c69f37bc66e409d22b04d9a7d16efb55f Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 15 Jul 2024 17:50:53 +0200 Subject: [PATCH 11/63] petab model --- pom.xml | 5 + src/main/java/life/qbic/io/PetabParser.java | 175 ++++++++++++++++++ .../commandline/DownloadDatasetCommand.java | 68 ------- .../io/commandline/DownloadPetabCommand.java | 57 ++++++ .../io/commandline/UploadDatasetCommand.java | 5 - .../commandline/UploadPetabResultCommand.java | 72 +++++++ .../qbic/model/DatasetWithProperties.java | 60 ++++++ .../qbic/model/download/FileSystemWriter.java | 2 +- ...{ModelReporter.java => SummaryWriter.java} | 4 +- .../java/life/qbic/model/petab/Arguments.java | 16 ++ .../life/qbic/model/petab/CellCountInfo.java | 23 +++ .../qbic/model/petab/ConditionWithUnit.java | 19 ++ .../model/petab/ExperimentalCondition.java | 20 ++ .../life/qbic/model/petab/IdWithPattern.java | 16 ++ .../life/qbic/model/petab/Measurement.java | 19 ++ .../qbic/model/petab/MeasurementData.java | 21 +++ .../java/life/qbic/model/petab/Medium.java | 22 +++ .../qbic/model/petab/MetaInformation.java | 90 +++++++++ .../life/qbic/model/petab/PetabMetadata.java | 16 ++ .../life/qbic/model/petab/Preprocessing.java | 21 +++ .../model/petab/PreprocessingInformation.java | 18 ++ src/main/java/life/qbic/model/petab/Time.java | 15 ++ 22 files changed, 687 insertions(+), 77 deletions(-) create mode 100644 src/main/java/life/qbic/io/PetabParser.java delete mode 100644 src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java create mode 100644 src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java create mode 100644 src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java create mode 100644 src/main/java/life/qbic/model/DatasetWithProperties.java rename src/main/java/life/qbic/model/download/{ModelReporter.java => SummaryWriter.java} (65%) create mode 100644 src/main/java/life/qbic/model/petab/Arguments.java create mode 100644 src/main/java/life/qbic/model/petab/CellCountInfo.java create mode 100644 src/main/java/life/qbic/model/petab/ConditionWithUnit.java create mode 100644 src/main/java/life/qbic/model/petab/ExperimentalCondition.java create mode 100644 src/main/java/life/qbic/model/petab/IdWithPattern.java create mode 100644 src/main/java/life/qbic/model/petab/Measurement.java create mode 100644 src/main/java/life/qbic/model/petab/MeasurementData.java create mode 100644 src/main/java/life/qbic/model/petab/Medium.java create mode 100644 src/main/java/life/qbic/model/petab/MetaInformation.java create mode 100644 src/main/java/life/qbic/model/petab/PetabMetadata.java create mode 100644 src/main/java/life/qbic/model/petab/Preprocessing.java create mode 100644 src/main/java/life/qbic/model/petab/PreprocessingInformation.java create mode 100644 src/main/java/life/qbic/model/petab/Time.java diff --git a/pom.xml b/pom.xml index 19f2587..b9c9238 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,11 @@ picocli 4.6.2 + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.17.2 + jline jline diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java new file mode 100644 index 0000000..b7c7bd1 --- /dev/null +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -0,0 +1,175 @@ +package life.qbic.io; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import life.qbic.model.petab.MetaInformation; +import life.qbic.model.petab.PetabMetadata; + +public class PetabParser { + + private final String META_INFO_YAML = "metaInformation.yaml"; + + public PetabMetadata parse(String dataPath) { + + File directory = new File(dataPath); + List sourcePetabReferences = new ArrayList<>(); + + File yaml = findYaml(directory); + if (yaml != null) { + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(yaml)); + while (true) { + String line = reader.readLine(); + if (line == null) { + break; + } + if (line.startsWith("openbisID")) { + String datasetCode = line.strip().split(": ")[1]; + sourcePetabReferences.add(datasetCode); + } + } + reader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return new PetabMetadata(sourcePetabReferences); + } + + private File findYaml(File directory) { + for (File file : Objects.requireNonNull(directory.listFiles())) { + if (file.isFile() && file.getName().equals(META_INFO_YAML)) { + return file; + } + if (file.isDirectory()) { + return findYaml(file); + } + } + System.out.println(META_INFO_YAML + " not found"); + return null; + } + + /** + * adds key-value pairs to the bottom of a petab.yaml found below the provided path + * + * @param outputPath the path of the PEtab + * @param properties map of properties to add + */ + public void addParameters(String outputPath, Map properties) { + + File directory = new File(outputPath); + + File yaml = findYaml(directory); + if (yaml != null) { + FileOutputStream fos = null; + try { + fos = new FileOutputStream(yaml.getPath(), true); + for (Map.Entry entry : properties.entrySet()) { + String line = entry.getKey() + ": " + entry.getValue() + "\n"; + fos.write(line.getBytes()); + } + fos.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static void main(String[] args) throws IOException { + File testfile = new File("/Users/afriedrich/git/openbis-20-scripts/example_petab/metaInformation.yaml"); + ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + MetaInformation metaInfo = mapper.readValue(testfile, MetaInformation.class); + System.err.println(metaInfo); + metaInfo.getUnits().setOpenbisParentIds(Arrays.asList("1","2")); + metaInfo.getUnits().setOpenbisId("3"); + mapper.writeValue(new File("/Users/afriedrich/git/openbis-20-scripts/Output.yaml"), metaInfo); + + + ObjectMapper mapper2 = new YAMLMapper(); + MetaInformation metaInfo2 = mapper2.readValue(testfile, MetaInformation.class); + System.err.println(metaInfo2); + + /* +MetaInformation{ + units=Units{ + measurement='some technique', + time='min, + t0 = timepoint of first intervention', + stimulus='something', + medium=Medium{type='DMEM', volume=1.5, unit='ml'}, + ncells=CellCountInfo{seeded=0.0, count='null', unit='null'}, + measurement_technique='immunublotting', + openBISId='null', openBISParentIds=null, + dateOfExperiment=[2024-04-30] + }, + preprocessingInformation=null, + measurementData=MeasurementData{ + measurement=Measurement{ + unit='intensity (a.u.)', lloq='null'}, + time=Time{unit='min'}, + replicateId=IdWithPattern{name='null', pattern='date_gel_replicate'} + }, + experimentalCondition=ExperimentalCondition{conditionId=IdWithPattern{name='null', pattern='condition'}, + conditions=null}} + + + ExperimentInformation: +units: + measurement: some technique + time: min, t0 = timepoint of first intervention + treatment: + stimulus: something + medium: + type: DMEM + volume: 1.5 + unit: ml + ncells: + seeded: 0.4 + ncellsCount: ~ + unit: mio + measurement_technique: immunublotting + openBISId: ~ + dateOfExperiment: + - 2024-04-30 +PreprocessingInformation: + normalizationStatus: Raw (data on linear scale) + preprocessing: + method: normalize by blotIt + arguments: + housekeeperObservableIds: + description: +measurementData: + measurement: + unit: intensity (a.u.) + lloq: ~ + time: + unit: min + replicateId: + pattern: date_gel_replicate +experimentalCondition: + conditionId: + pattern: condition + TGFb: + unit: ng/ul + GAS6: + unit: ug/ml + + */ + } + +} diff --git a/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java deleted file mode 100644 index 102baff..0000000 --- a/src/main/java/life/qbic/io/commandline/DownloadDatasetCommand.java +++ /dev/null @@ -1,68 +0,0 @@ -package life.qbic.io.commandline; - -import ch.ethz.sis.openbis.generic.OpenBIS; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import life.qbic.App; -import life.qbic.model.download.OpenbisConnector; -import picocli.CommandLine.Command; -import picocli.CommandLine.Mixin; -import picocli.CommandLine.Option; -import picocli.CommandLine.Parameters; - -@Command(name = "list-data", - description = "lists datasets and their details for a given experiment code") -public class DownloadDatasetCommand implements Runnable { - - @Parameters(arity = "1", paramLabel = "experiment", description = "The code of the experiment data is attached to") - private String experimentCode; - @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter results", names = {"-s", "--space"}) - private String space; - @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); - - @Override - public void run() { - List spaces = new ArrayList<>(); - if (space != null) { - System.out.println("Querying experiment in space: " + space + "..."); - spaces.add(space); - } else { - System.out.println("Querying experiment in all available spaces..."); - } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); - OpenbisConnector openbis = new OpenbisConnector(authentication); - List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() - .sorted(Comparator.comparing( - (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( - Collectors.toList()); - int datasetIndex = 0; - for (DataSet dataSet : datasets) { - datasetIndex++; - System.out.println("["+datasetIndex+"]"); - System.out.println(dataSet.getExperiment().getIdentifier()); - System.out.println(dataSet.getCode()); - System.out.println(dataSet.getType().getCode()); - System.out.println(dataSet.getRegistrationDate()); - System.out.println(new SimpleDateFormat("MM-dd-yyyy").format(dataSet.getRegistrationDate())); - Person person = dataSet.getRegistrator(); - System.out.println(person.getFirstName() + " " + person.getLastName()); - System.out.println(); - } - } - - private String getTimeStamp() { - final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; - DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); - return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); - } -} diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java new file mode 100644 index 0000000..0dc9d95 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -0,0 +1,57 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import life.qbic.App; +import life.qbic.io.PetabParser; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; + +@Command(name = "download-petab", + description = "Downloads PEtab dataset and stores some additional information from openbis in the petab.yaml") +public class DownloadPetabCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset to download. Can be found via list-data.") + private String datasetCode; + @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to store the downloaded data") + private String outputPath; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + + List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); + + if(datasets.isEmpty()) { + System.out.println(datasetCode+" not found"); + return; + } + DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); + Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + result.getExperiment().getIdentifier()); + patientID.ifPresent(s -> result.addProperty("patientID", s)); + result.addProperty("openbisID", datasetCode); + + System.out.println("Found dataset, downloading."); + System.out.println(); + + openbis.downloadDataset(outputPath, datasetCode); + + System.out.println("Adding additional information to petab.yaml"); + System.out.println(); + + PetabParser parser = new PetabParser(); + parser.addParameters(outputPath, result.getProperties()); + System.out.println("Done"); + } + +} diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 9ea557e..74fa98e 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -70,9 +70,4 @@ private boolean pathValid(String dataPath) { return new File(dataPath).exists(); } - private String getTimeStamp() { - final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; - DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); - return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); - } } diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java new file mode 100644 index 0000000..eedc04b --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -0,0 +1,72 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import java.io.File; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import life.qbic.App; +import life.qbic.io.PetabParser; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "upload-petab-result", + description = "uploads a petab based on other PETab downloaded from openbis and attaches it to a provided experiment and any datasets referenced in the PETab metadata.") +public class UploadPetabResultCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") + private String dataPath; + @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. " + + "The identifier must be of the format: /space/project/experiment") + private String experimentID; + @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) + private List parents = new ArrayList<>(); + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + private OpenbisConnector openbis; + private PetabParser petabParser = new PetabParser(); + + @Override + public void run() { + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + openbis = new OpenbisConnector(authentication); + + if(!pathValid(dataPath)) { + System.out.printf("Path %s could not be found%n", dataPath); + return; + } + if(!experimentExists(experimentID)) { + System.out.printf("Experiment %s could not be found%n", experimentID); + return; + } + parents = petabParser.parse(dataPath).getSourcePetabReferences(); + if(!datasetsExist(parents)) { + System.out.printf("One or more datasets %s could not be found%n", parents); + return; + } + System.out.println(); + System.out.println("Parameters verified, uploading dataset..."); + System.out.println();//TODO copy and remove source references + DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); + System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + } + + private boolean datasetsExist(List datasetCodes) { + return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); + } + + private boolean experimentExists(String experimentID) { + return openbis.experimentExists(experimentID); + } + + private boolean pathValid(String dataPath) { + return new File(dataPath).exists(); + } + +} diff --git a/src/main/java/life/qbic/model/DatasetWithProperties.java b/src/main/java/life/qbic/model/DatasetWithProperties.java new file mode 100644 index 0000000..d9a1534 --- /dev/null +++ b/src/main/java/life/qbic/model/DatasetWithProperties.java @@ -0,0 +1,60 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * Wrapper class for openBIS DataSets that collects additional information, e.g. from samples, + * experiments etc. further up in the hierarchy. + */ +public class DatasetWithProperties { + + private final DataSet dataset; + private final Map properties; + + public DatasetWithProperties(DataSet dataset) { + this.dataset = dataset; + this.properties = new HashMap<>(); + } + + public void addProperty(String key, String value) { + this.properties.put(key, value); + } + + public String getProperty(String key) { + return properties.get(key); + } + + public Map getProperties() { + return properties; + } + + public DataSet getDataset() { + return dataset; + } + + public String getCode() { + return dataset.getCode(); + } + + public Experiment getExperiment() { + return dataset.getExperiment(); + } + + public DataSetType getType() { + return dataset.getType(); + } + + public Person getRegistrator() { + return dataset.getRegistrator(); + } + + public Date getRegistrationDate() { + return dataset.getRegistrationDate(); + } +} diff --git a/src/main/java/life/qbic/model/download/FileSystemWriter.java b/src/main/java/life/qbic/model/download/FileSystemWriter.java index 4cf0cb9..59b6419 100644 --- a/src/main/java/life/qbic/model/download/FileSystemWriter.java +++ b/src/main/java/life/qbic/model/download/FileSystemWriter.java @@ -14,7 +14,7 @@ * * @author: Sven Fillinger, Andreas Friedrich */ -public class FileSystemWriter implements ModelReporter { +public class FileSystemWriter implements SummaryWriter { /** * File that stores the summary report content for valid checksums. diff --git a/src/main/java/life/qbic/model/download/ModelReporter.java b/src/main/java/life/qbic/model/download/SummaryWriter.java similarity index 65% rename from src/main/java/life/qbic/model/download/ModelReporter.java rename to src/main/java/life/qbic/model/download/SummaryWriter.java index 875ffb4..dbf2438 100644 --- a/src/main/java/life/qbic/model/download/ModelReporter.java +++ b/src/main/java/life/qbic/model/download/SummaryWriter.java @@ -1,11 +1,9 @@ package life.qbic.model.download; import java.io.IOException; -import java.net.URL; -import java.nio.file.Path; import java.util.List; -public interface ModelReporter { +public interface SummaryWriter { void reportSummary(List summary) throws IOException; } diff --git a/src/main/java/life/qbic/model/petab/Arguments.java b/src/main/java/life/qbic/model/petab/Arguments.java new file mode 100644 index 0000000..05629ed --- /dev/null +++ b/src/main/java/life/qbic/model/petab/Arguments.java @@ -0,0 +1,16 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public class Arguments { + @JsonProperty + List housekeeperObservableIds; + + @Override + public String toString() { + return "Arguments{" + + "housekeeperObservableIds=" + housekeeperObservableIds + + '}'; + } +} diff --git a/src/main/java/life/qbic/model/petab/CellCountInfo.java b/src/main/java/life/qbic/model/petab/CellCountInfo.java new file mode 100644 index 0000000..e65ce59 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/CellCountInfo.java @@ -0,0 +1,23 @@ +package life.qbic.model.petab; + + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class CellCountInfo { + @JsonProperty + double seeded; + @JsonProperty + String ncellsCount; + @JsonProperty + String unit; + + @Override + public String toString() { + return "CellCountInfo{" + + "seeded=" + seeded + + ", ncellsCount='" + ncellsCount + '\'' + + ", unit='" + unit + '\'' + + '}'; + } +} + diff --git a/src/main/java/life/qbic/model/petab/ConditionWithUnit.java b/src/main/java/life/qbic/model/petab/ConditionWithUnit.java new file mode 100644 index 0000000..802f961 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/ConditionWithUnit.java @@ -0,0 +1,19 @@ +package life.qbic.model.petab; + + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ConditionWithUnit { + @JsonProperty + String name; + @JsonProperty + String unit; + + @Override + public String toString() { + return "ConditionWithUnit{" + + "name='" + name + '\'' + + ", unit='" + unit + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/model/petab/ExperimentalCondition.java b/src/main/java/life/qbic/model/petab/ExperimentalCondition.java new file mode 100644 index 0000000..21a62f6 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/ExperimentalCondition.java @@ -0,0 +1,20 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + + +public class ExperimentalCondition { + @JsonProperty + IdWithPattern conditionId; + @JsonProperty + List conditions; + + @Override + public String toString() { + return "ExperimentalCondition{" + + "conditionId=" + conditionId + + ", conditions=" + conditions + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/model/petab/IdWithPattern.java b/src/main/java/life/qbic/model/petab/IdWithPattern.java new file mode 100644 index 0000000..0e94c7b --- /dev/null +++ b/src/main/java/life/qbic/model/petab/IdWithPattern.java @@ -0,0 +1,16 @@ +package life.qbic.model.petab; + + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IdWithPattern { + @JsonProperty + String pattern; + + @Override + public String toString() { + return "IdWithPattern{" + + "pattern='" + pattern + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/model/petab/Measurement.java b/src/main/java/life/qbic/model/petab/Measurement.java new file mode 100644 index 0000000..531e7b5 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/Measurement.java @@ -0,0 +1,19 @@ +package life.qbic.model.petab; + + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Measurement { + @JsonProperty + String unit; + @JsonProperty + String lloq; + + @Override + public String toString() { + return "Measurement{" + + "unit='" + unit + '\'' + + ", lloq='" + lloq + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/model/petab/MeasurementData.java b/src/main/java/life/qbic/model/petab/MeasurementData.java new file mode 100644 index 0000000..62b449a --- /dev/null +++ b/src/main/java/life/qbic/model/petab/MeasurementData.java @@ -0,0 +1,21 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class MeasurementData { + @JsonProperty + private Measurement measurement; + @JsonProperty + private Time time; + @JsonProperty + private IdWithPattern replicateId; + + @Override + public String toString() { + return "MeasurementData{" + + "measurement=" + measurement + + ", time=" + time + + ", replicateId=" + replicateId + + '}'; + } +} diff --git a/src/main/java/life/qbic/model/petab/Medium.java b/src/main/java/life/qbic/model/petab/Medium.java new file mode 100644 index 0000000..d5c4ed3 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/Medium.java @@ -0,0 +1,22 @@ +package life.qbic.model.petab; + + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Medium { + @JsonProperty + String type; + @JsonProperty + double volume; + @JsonProperty + String unit; + + @Override + public String toString() { + return "Medium{" + + "type='" + type + '\'' + + ", volume=" + volume + + ", unit='" + unit + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/model/petab/MetaInformation.java b/src/main/java/life/qbic/model/petab/MetaInformation.java new file mode 100644 index 0000000..ff19ab3 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/MetaInformation.java @@ -0,0 +1,90 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public class MetaInformation { + + @JsonProperty + private ExperimentInformation ExperimentInformation; + + @JsonProperty + private Units units; + @JsonProperty + private PreprocessingInformation PreprocessingInformation; + @JsonProperty + private MeasurementData measurementData; + @JsonProperty + private ExperimentalCondition experimentalCondition; + + + @Override + public String toString() { + return "MetaInformation{" + + "units=" + units + + ", preprocessingInformation=" + PreprocessingInformation + + ", measurementData=" + measurementData + + ", experimentalCondition=" + experimentalCondition + + '}'; + } + + public Units getUnits() { + return units; + } + + public class ExperimentInformation { + + @Override + public String toString() { + return "MetaInformation{}"; + } + } + + public class Units { + @JsonProperty + private String measurement; + @JsonProperty + private String time; + @JsonProperty + private String treatment; + @JsonProperty + private String stimulus; + @JsonProperty + private Medium medium; + @JsonProperty + private CellCountInfo ncells; + @JsonProperty + private String measurement_technique; + @JsonProperty + private String openBISId; + @JsonProperty + private List openBISParentIds; + @JsonProperty + private List dateOfExperiment; + + @Override + public String toString() { + return "Units{" + + "measurement='" + measurement + '\'' + + ", time='" + time + '\'' + + ", treatment='" + treatment + '\'' + + ", stimulus='" + stimulus + '\'' + + ", medium=" + medium + + ", ncells=" + ncells + + ", measurement_technique='" + measurement_technique + '\'' + + ", openBISId='" + openBISId + '\'' + + ", openBISParentIds=" + openBISParentIds + + ", dateOfExperiment=" + dateOfExperiment + + '}'; + } + + public void setOpenbisParentIds(List list) { + this.openBISParentIds = list; + } + + public void setOpenbisId(String id) { + this.openBISId = id; + } + } + +} diff --git a/src/main/java/life/qbic/model/petab/PetabMetadata.java b/src/main/java/life/qbic/model/petab/PetabMetadata.java new file mode 100644 index 0000000..7fb8b01 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/PetabMetadata.java @@ -0,0 +1,16 @@ +package life.qbic.model.petab; + +import java.util.List; + +public class PetabMetadata { + + List sourceDatasetIdentifiers; + + public PetabMetadata(List sourceDatasetIdentifiers) { + this.sourceDatasetIdentifiers = sourceDatasetIdentifiers; + } + + public List getSourcePetabReferences() { + return sourceDatasetIdentifiers; + } +} diff --git a/src/main/java/life/qbic/model/petab/Preprocessing.java b/src/main/java/life/qbic/model/petab/Preprocessing.java new file mode 100644 index 0000000..6bc4dbf --- /dev/null +++ b/src/main/java/life/qbic/model/petab/Preprocessing.java @@ -0,0 +1,21 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Preprocessing { + @JsonProperty + private String method; + @JsonProperty + private String description; + @JsonProperty + private Arguments arguments; + + @Override + public String toString() { + return "Preprocessing{" + + "method='" + method + '\'' + + ", description='" + description + '\'' + + ", arguments=" + arguments + + '}'; + } +} diff --git a/src/main/java/life/qbic/model/petab/PreprocessingInformation.java b/src/main/java/life/qbic/model/petab/PreprocessingInformation.java new file mode 100644 index 0000000..b7a2290 --- /dev/null +++ b/src/main/java/life/qbic/model/petab/PreprocessingInformation.java @@ -0,0 +1,18 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class PreprocessingInformation { + @JsonProperty + private String normalizationStatus; + @JsonProperty + private Preprocessing preprocessing; + + @Override + public String toString() { + return "PreprocessingInformation{" + + "normalizationStatus='" + normalizationStatus + '\'' + + ", preprocessing=" + preprocessing + + '}'; + } +} diff --git a/src/main/java/life/qbic/model/petab/Time.java b/src/main/java/life/qbic/model/petab/Time.java new file mode 100644 index 0000000..ce9ea4f --- /dev/null +++ b/src/main/java/life/qbic/model/petab/Time.java @@ -0,0 +1,15 @@ +package life.qbic.model.petab; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Time { + @JsonProperty + String unit; + + @Override + public String toString() { + return "Time{" + + "unit='" + unit + '\'' + + '}'; + } +} From 7c47eb3fd35ae03bd0ef05f2f85897b405774474 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 15 Jul 2024 17:51:52 +0200 Subject: [PATCH 12/63] statistics feature --- .../io/commandline/CommandLineOptions.java | 3 +- .../io/commandline/FindDatasetsCommand.java | 31 ++- .../commandline/SampleHierarchyCommand.java | 12 +- .../commandline/SpaceStatisticsCommand.java | 135 ++++++++++++ .../qbic/model/download/OpenbisConnector.java | 201 +++++++++++++++--- 5 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index 3acf00a..d050ebf 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,7 +8,8 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, UploadDatasetCommand.class }, + subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class, + UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class }, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 82c2a18..2846fe7 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -4,15 +4,15 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import life.qbic.App; +import life.qbic.model.DatasetWithProperties; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -45,12 +45,28 @@ public void run() { .sorted(Comparator.comparing( (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( Collectors.toList()); + Map properties = new HashMap<>(); + if (!datasets.isEmpty()) { + Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + datasets.get(0).getExperiment().getIdentifier()); + patientID.ifPresent(s -> properties.put("Patient ID", s)); + } + List datasetWithProperties = datasets.stream().map(dataSet -> { + DatasetWithProperties ds = new DatasetWithProperties(dataSet); + for (String key : properties.keySet()) { + ds.addProperty(key, properties.get(key)); + } + return ds; + }).collect(Collectors.toList()); int datasetIndex = 0; System.out.println(); System.out.printf("Found %s datasets for experiment %s:%n", datasets.size(), experimentCode); - for (DataSet dataSet : datasets) { + for (DatasetWithProperties dataSet : datasetWithProperties) { datasetIndex++; System.out.println("["+datasetIndex+"]"); + for(String key : dataSet.getProperties().keySet()) { + System.out.println(key+ ": "+properties.get(key)); + } System.out.printf("ID: %s (%s)%n", dataSet.getCode(), dataSet.getExperiment().getIdentifier()); System.out.println("Type: "+dataSet.getType().getCode()); Person person = dataSet.getRegistrator(); @@ -62,9 +78,4 @@ public void run() { } } - private String getTimeStamp() { - final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; - DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); - return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); - } } diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index e9668d6..47353b6 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -16,7 +16,7 @@ import life.qbic.model.Configuration; import life.qbic.model.SampleTypeConnection; import life.qbic.model.download.FileSystemWriter; -import life.qbic.model.download.ModelReporter; +import life.qbic.model.download.SummaryWriter; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -38,10 +38,10 @@ public void run() { List summary = new ArrayList<>(); List spaces = new ArrayList<>(); if(space!=null) { - summary.add("Querying samples in space: "+space+"...\n"); + summary.add("Querying samples in space: "+space+"..."); spaces.add(space); } else { - summary.add("Querying samples in all available spaces...\n"); + summary.add("Querying samples in all available spaces..."); } OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); @@ -55,13 +55,13 @@ public void run() { System.out.println(s); } Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), - "summary_model_"+getTimeStamp()+".txt"); + "sample_model_summary"+getTimeStamp()+".txt"); if(outpath!=null) { outputPath = Paths.get(outpath); } - ModelReporter modelReporter = new FileSystemWriter(outputPath); + SummaryWriter summaryWriter = new FileSystemWriter(outputPath); try { - modelReporter.reportSummary(summary); + summaryWriter.reportSummary(summary); } catch (IOException e) { throw new RuntimeException("Could not write summary file."); } diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java new file mode 100644 index 0000000..bce2144 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -0,0 +1,135 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import life.qbic.App; +import life.qbic.model.Configuration; +import life.qbic.model.download.FileSystemWriter; +import life.qbic.model.download.SummaryWriter; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +@Command(name = "statistics", + description = "lists the number of collections, sample objects and attached datasets (by type)" + + "for one or all spaces accessible by the user") +public class SpaceStatisticsCommand implements Runnable { + + @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) + private String space; + @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) + private String outpath; + @Option(arity = "0", description = "shows results for openBIS settings and material spaces. Ignored if a specific space is selected.", + names = {"--show-settings"}) + private boolean allSpaces; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + List summary = new ArrayList<>(); + List blackList = new ArrayList<>(Arrays.asList("ELN_SETTINGS", "MATERIAL.GLOBAL")); + List spaces = new ArrayList<>(); + if (space != null) { + summary.add("Querying samples in space: " + space); + spaces.add(space); + } else { + summary.add("Querying samples in all available spaces..."); + } + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + + if (spaces.isEmpty()) { + spaces = openbis.getSpaces(); + if(!allSpaces) { + spaces.removeAll(blackList); + } + } + + Map>> experiments = openbis.getExperimentsByTypeAndSpace(spaces); + Map>> samples = openbis.getSamplesByTypeAndSpace(spaces); + Map>> datasets = openbis.getDatasetsByTypeAndSpace(spaces); + + for(String space : spaces) { + summary.add("-----"); + summary.add("Summary for "+space); + summary.add("-----"); + int numExps = 0; + if (experiments.containsKey(space)) { + numExps = experiments.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Experiments ("+numExps+"):"); + summary.add(""); + if(!experiments.isEmpty()) { + Map> exps = experiments.get(space); + for (String type : exps.keySet()) { + summary.add(type + ": " + exps.get(type).size()); + } + } + summary.add(""); + int numSamples = 0; + if (samples.containsKey(space)) { + numSamples = samples.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Samples ("+numSamples+"):"); + summary.add(""); + if(!samples.isEmpty()) { + Map> samps = samples.get(space); + for (String type : samps.keySet()) { + summary.add(type + ": " + samps.get(type).size()); + } + } + summary.add(""); + int numData = 0; + if (datasets.containsKey(space)) { + numData = datasets.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Attached datasets (" + numData + "):"); + summary.add(""); + if (datasets.get(space) != null) { + Map> dsets = datasets.get(space); + for (String dataType : dsets.keySet()) { + summary.add(dataType + ": " + dsets.get(dataType).size()); + } + } + + summary.add(""); + } + + for(String line : summary) { + System.out.println(line); + } + + Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), + "spaces_summary_"+getTimeStamp()+".txt"); + if(outpath!=null) { + outputPath = Paths.get(outpath); + } + SummaryWriter summaryWriter = new FileSystemWriter(outputPath); + try { + summaryWriter.reportSummary(summary); + } catch (IOException e) { + throw new RuntimeException("Could not write summary file."); + } + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter).toString(); + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index f5e1654..cf822a7 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -8,6 +8,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; @@ -35,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; @@ -45,29 +47,6 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); OpenBIS openBIS; - /** - * Constructor for a QBiCDataDownloader instance - * - * @param AppServerUri The openBIS application server URL (AS) - * @param sessionToken The session token for the datastore & application servers - */ - public OpenbisConnector( - String AppServerUri, - String sessionToken) { - /* - this.sessionToken = sessionToken; - - if (!AppServerUri.isEmpty()) { - applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - applicationServer = null; - } - - */ - } - public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; } @@ -124,12 +103,11 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public void downloadDataset(String targetPath, String datasetID) throws IOException { + public void downloadDataset(String targetPath, String datasetID) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), ""); - System.err.println(fileToDownload); // Setting recursive flag to true will return both subfolders and files options.setRecursive(true); @@ -137,18 +115,24 @@ public void downloadDataset(String targetPath, String datasetID) throws IOExcept InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), options); DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); - DataSetFileDownload file = null; + DataSetFileDownload file; while ((file = reader.read()) != null) { DataSetFile df = file.getDataSetFile(); String currentPath = df.getPath().replace("original", ""); if (df.isDirectory()) { File newDir = new File(targetPath, currentPath); if (!newDir.exists()) { - newDir.mkdirs(); + if(!newDir.mkdirs()) { + throw new RuntimeException("Could not create folders for downloaded dataset."); + } } } else { File toWrite = new File(targetPath, currentPath); - copyInputStreamToFile(file.getInputStream(), toWrite); + try { + copyInputStreamToFile(file.getInputStream(), toWrite); + } catch (IOException e) { + throw new RuntimeException(e); + } } } } @@ -195,6 +179,165 @@ public Map queryFullSampleHierarchy(List return hierarchy; } + private Optional getPropertyFromSampleHierarchy(String propertyName, List samples) { + for(Sample s : samples) { + if(s.getProperties().containsKey(propertyName)) { + return Optional.of(s.getProperties().get(propertyName)); + } + return getPropertyFromSampleHierarchy(propertyName, s.getParents()); + } + return Optional.empty(); + } + + public Optional findPropertyInSampleHierarchy(String propertyName, + ExperimentIdentifier experimentId) { + return getPropertyFromSampleHierarchy(propertyName, + getSamplesWithAncestorsOfExperiment(experimentId)); + } + + public Map> getExperimentsBySpace(List spaces) { + Map> result = new HashMap<>(); + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProject().withSpace(); + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withProject().withSpace().withCodes().thatIn(spaces); + for (Experiment e : openBIS.searchExperiments(criteria, options).getObjects()) { + String space = e.getProject().getSpace().getCode(); + if(result.containsKey(space)) { + result.get(space).add(e); + } else { + result.put(space, new ArrayList<>()); + } + } + return result; + } + + public Map> getSamplesBySpace(List spaces) { + Map> result = new HashMap<>(); + SampleFetchOptions options = new SampleFetchOptions(); + options.withSpace(); + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCodes().thatIn(spaces); + for (Sample s : openBIS.searchSamples(criteria, options).getObjects()) { + String space = s.getSpace().getCode(); + if(!result.containsKey(space)) { + result.put(space, new ArrayList<>()); + } + result.get(space).add(s); + } + return result; + } + + public Map>> getExperimentsByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProject().withSpace(); + options.withType(); + + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withProject().withSpace().withCodes().thatIn(spaces); + for (Experiment exp : openBIS.searchExperiments(criteria, options).getObjects()) { + String space = exp.getProject().getSpace().getCode(); + String type = exp.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(exp))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(exp); + } + } + return result; + } + + public Map>> getSamplesByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + SampleFetchOptions options = new SampleFetchOptions(); + options.withSpace(); + options.withType(); + + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCodes().thatIn(spaces); + for (Sample s : openBIS.searchSamples(criteria, options).getObjects()) { + String space = s.getSpace().getCode(); + String type = s.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(s))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(s); + } + } + return result; + } + + public Map>> getDatasetsByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withSample().withSpace(); + options.withExperiment().withProject().withSpace(); + options.withType(); + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withOrOperator(); + criteria.withSample().withSpace().withCodes().thatIn(spaces); + criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + for (DataSet d : openBIS.searchDataSets(criteria, options).getObjects()) { + String space = getSpaceFromSampleOrExperiment(d); + String type = d.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(d))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(d); + } + } + return result; + } + + private String getSpaceFromSampleOrExperiment(DataSet d) { + try { + if (d.getSample() != null) { + return d.getSample().getSpace().getCode(); + } + if (d.getExperiment() != null) { + return d.getExperiment().getProject().getSpace().getCode(); + } + } catch (NullPointerException e) { + + } + System.out.println("Dataset " + d + "does not seem to be attached to a space"); + return "NO SPACE"; + } + + private List getSamplesWithAncestorsOfExperiment(ExperimentIdentifier experimentId) { + SampleFetchOptions allProps = new SampleFetchOptions(); + allProps.withType(); + allProps.withProperties(); + SampleFetchOptions withAncestors = new SampleFetchOptions(); + withAncestors.withParentsUsing(allProps); + withAncestors.withProperties(); + withAncestors.withType(); + + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withExperiment().withId().thatEquals(experimentId); + + return openBIS.searchSamples(criteria, withAncestors).getObjects(); + } + public List findDataSets(List codes) { if (codes.isEmpty()) { return new ArrayList<>(); @@ -202,6 +345,7 @@ public List findDataSets(List codes) { DataSetSearchCriteria criteria = new DataSetSearchCriteria(); criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); + options.withExperiment(); return openBIS.searchDataSets(criteria, options).getObjects(); } @@ -212,4 +356,5 @@ public boolean experimentExists(String experimentID) { return !openBIS.searchExperiments(criteria, new ExperimentFetchOptions()).getObjects() .isEmpty(); } + } From 4c731eb2b87f46a1badbbf10a862d0d87c509af6 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 15 Jul 2024 17:51:52 +0200 Subject: [PATCH 13/63] statistics feature --- .../io/commandline/CommandLineOptions.java | 3 +- .../io/commandline/FindDatasetsCommand.java | 31 ++- .../commandline/SampleHierarchyCommand.java | 12 +- .../commandline/SpaceStatisticsCommand.java | 135 ++++++++++++ .../qbic/model/download/OpenbisConnector.java | 201 +++++++++++++++--- 5 files changed, 337 insertions(+), 45 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index 3acf00a..d050ebf 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,7 +8,8 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, UploadDatasetCommand.class }, + subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class, + UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class }, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 82c2a18..2846fe7 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -4,15 +4,15 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import life.qbic.App; +import life.qbic.model.DatasetWithProperties; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -45,12 +45,28 @@ public void run() { .sorted(Comparator.comparing( (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( Collectors.toList()); + Map properties = new HashMap<>(); + if (!datasets.isEmpty()) { + Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + datasets.get(0).getExperiment().getIdentifier()); + patientID.ifPresent(s -> properties.put("Patient ID", s)); + } + List datasetWithProperties = datasets.stream().map(dataSet -> { + DatasetWithProperties ds = new DatasetWithProperties(dataSet); + for (String key : properties.keySet()) { + ds.addProperty(key, properties.get(key)); + } + return ds; + }).collect(Collectors.toList()); int datasetIndex = 0; System.out.println(); System.out.printf("Found %s datasets for experiment %s:%n", datasets.size(), experimentCode); - for (DataSet dataSet : datasets) { + for (DatasetWithProperties dataSet : datasetWithProperties) { datasetIndex++; System.out.println("["+datasetIndex+"]"); + for(String key : dataSet.getProperties().keySet()) { + System.out.println(key+ ": "+properties.get(key)); + } System.out.printf("ID: %s (%s)%n", dataSet.getCode(), dataSet.getExperiment().getIdentifier()); System.out.println("Type: "+dataSet.getType().getCode()); Person person = dataSet.getRegistrator(); @@ -62,9 +78,4 @@ public void run() { } } - private String getTimeStamp() { - final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; - DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); - return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); - } } diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index e9668d6..47353b6 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -16,7 +16,7 @@ import life.qbic.model.Configuration; import life.qbic.model.SampleTypeConnection; import life.qbic.model.download.FileSystemWriter; -import life.qbic.model.download.ModelReporter; +import life.qbic.model.download.SummaryWriter; import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -38,10 +38,10 @@ public void run() { List summary = new ArrayList<>(); List spaces = new ArrayList<>(); if(space!=null) { - summary.add("Querying samples in space: "+space+"...\n"); + summary.add("Querying samples in space: "+space+"..."); spaces.add(space); } else { - summary.add("Querying samples in all available spaces...\n"); + summary.add("Querying samples in all available spaces..."); } OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); @@ -55,13 +55,13 @@ public void run() { System.out.println(s); } Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), - "summary_model_"+getTimeStamp()+".txt"); + "sample_model_summary"+getTimeStamp()+".txt"); if(outpath!=null) { outputPath = Paths.get(outpath); } - ModelReporter modelReporter = new FileSystemWriter(outputPath); + SummaryWriter summaryWriter = new FileSystemWriter(outputPath); try { - modelReporter.reportSummary(summary); + summaryWriter.reportSummary(summary); } catch (IOException e) { throw new RuntimeException("Could not write summary file."); } diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java new file mode 100644 index 0000000..bce2144 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -0,0 +1,135 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import life.qbic.App; +import life.qbic.model.Configuration; +import life.qbic.model.download.FileSystemWriter; +import life.qbic.model.download.SummaryWriter; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +@Command(name = "statistics", + description = "lists the number of collections, sample objects and attached datasets (by type)" + + "for one or all spaces accessible by the user") +public class SpaceStatisticsCommand implements Runnable { + + @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) + private String space; + @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) + private String outpath; + @Option(arity = "0", description = "shows results for openBIS settings and material spaces. Ignored if a specific space is selected.", + names = {"--show-settings"}) + private boolean allSpaces; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + List summary = new ArrayList<>(); + List blackList = new ArrayList<>(Arrays.asList("ELN_SETTINGS", "MATERIAL.GLOBAL")); + List spaces = new ArrayList<>(); + if (space != null) { + summary.add("Querying samples in space: " + space); + spaces.add(space); + } else { + summary.add("Querying samples in all available spaces..."); + } + OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + + if (spaces.isEmpty()) { + spaces = openbis.getSpaces(); + if(!allSpaces) { + spaces.removeAll(blackList); + } + } + + Map>> experiments = openbis.getExperimentsByTypeAndSpace(spaces); + Map>> samples = openbis.getSamplesByTypeAndSpace(spaces); + Map>> datasets = openbis.getDatasetsByTypeAndSpace(spaces); + + for(String space : spaces) { + summary.add("-----"); + summary.add("Summary for "+space); + summary.add("-----"); + int numExps = 0; + if (experiments.containsKey(space)) { + numExps = experiments.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Experiments ("+numExps+"):"); + summary.add(""); + if(!experiments.isEmpty()) { + Map> exps = experiments.get(space); + for (String type : exps.keySet()) { + summary.add(type + ": " + exps.get(type).size()); + } + } + summary.add(""); + int numSamples = 0; + if (samples.containsKey(space)) { + numSamples = samples.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Samples ("+numSamples+"):"); + summary.add(""); + if(!samples.isEmpty()) { + Map> samps = samples.get(space); + for (String type : samps.keySet()) { + summary.add(type + ": " + samps.get(type).size()); + } + } + summary.add(""); + int numData = 0; + if (datasets.containsKey(space)) { + numData = datasets.get(space).values().stream().mapToInt(List::size).sum(); + } + summary.add("Attached datasets (" + numData + "):"); + summary.add(""); + if (datasets.get(space) != null) { + Map> dsets = datasets.get(space); + for (String dataType : dsets.keySet()) { + summary.add(dataType + ": " + dsets.get(dataType).size()); + } + } + + summary.add(""); + } + + for(String line : summary) { + System.out.println(line); + } + + Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), + "spaces_summary_"+getTimeStamp()+".txt"); + if(outpath!=null) { + outputPath = Paths.get(outpath); + } + SummaryWriter summaryWriter = new FileSystemWriter(outputPath); + try { + summaryWriter.reportSummary(summary); + } catch (IOException e) { + throw new RuntimeException("Could not write summary file."); + } + } + + private String getTimeStamp() { + final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter).toString(); + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index f5e1654..cf822a7 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -8,6 +8,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; @@ -35,6 +36,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; @@ -45,29 +47,6 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); OpenBIS openBIS; - /** - * Constructor for a QBiCDataDownloader instance - * - * @param AppServerUri The openBIS application server URL (AS) - * @param sessionToken The session token for the datastore & application servers - */ - public OpenbisConnector( - String AppServerUri, - String sessionToken) { - /* - this.sessionToken = sessionToken; - - if (!AppServerUri.isEmpty()) { - applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - applicationServer = null; - } - - */ - } - public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; } @@ -124,12 +103,11 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public void downloadDataset(String targetPath, String datasetID) throws IOException { + public void downloadDataset(String targetPath, String datasetID) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), ""); - System.err.println(fileToDownload); // Setting recursive flag to true will return both subfolders and files options.setRecursive(true); @@ -137,18 +115,24 @@ public void downloadDataset(String targetPath, String datasetID) throws IOExcept InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), options); DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); - DataSetFileDownload file = null; + DataSetFileDownload file; while ((file = reader.read()) != null) { DataSetFile df = file.getDataSetFile(); String currentPath = df.getPath().replace("original", ""); if (df.isDirectory()) { File newDir = new File(targetPath, currentPath); if (!newDir.exists()) { - newDir.mkdirs(); + if(!newDir.mkdirs()) { + throw new RuntimeException("Could not create folders for downloaded dataset."); + } } } else { File toWrite = new File(targetPath, currentPath); - copyInputStreamToFile(file.getInputStream(), toWrite); + try { + copyInputStreamToFile(file.getInputStream(), toWrite); + } catch (IOException e) { + throw new RuntimeException(e); + } } } } @@ -195,6 +179,165 @@ public Map queryFullSampleHierarchy(List return hierarchy; } + private Optional getPropertyFromSampleHierarchy(String propertyName, List samples) { + for(Sample s : samples) { + if(s.getProperties().containsKey(propertyName)) { + return Optional.of(s.getProperties().get(propertyName)); + } + return getPropertyFromSampleHierarchy(propertyName, s.getParents()); + } + return Optional.empty(); + } + + public Optional findPropertyInSampleHierarchy(String propertyName, + ExperimentIdentifier experimentId) { + return getPropertyFromSampleHierarchy(propertyName, + getSamplesWithAncestorsOfExperiment(experimentId)); + } + + public Map> getExperimentsBySpace(List spaces) { + Map> result = new HashMap<>(); + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProject().withSpace(); + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withProject().withSpace().withCodes().thatIn(spaces); + for (Experiment e : openBIS.searchExperiments(criteria, options).getObjects()) { + String space = e.getProject().getSpace().getCode(); + if(result.containsKey(space)) { + result.get(space).add(e); + } else { + result.put(space, new ArrayList<>()); + } + } + return result; + } + + public Map> getSamplesBySpace(List spaces) { + Map> result = new HashMap<>(); + SampleFetchOptions options = new SampleFetchOptions(); + options.withSpace(); + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCodes().thatIn(spaces); + for (Sample s : openBIS.searchSamples(criteria, options).getObjects()) { + String space = s.getSpace().getCode(); + if(!result.containsKey(space)) { + result.put(space, new ArrayList<>()); + } + result.get(space).add(s); + } + return result; + } + + public Map>> getExperimentsByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProject().withSpace(); + options.withType(); + + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withProject().withSpace().withCodes().thatIn(spaces); + for (Experiment exp : openBIS.searchExperiments(criteria, options).getObjects()) { + String space = exp.getProject().getSpace().getCode(); + String type = exp.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(exp))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(exp); + } + } + return result; + } + + public Map>> getSamplesByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + SampleFetchOptions options = new SampleFetchOptions(); + options.withSpace(); + options.withType(); + + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withSpace().withCodes().thatIn(spaces); + for (Sample s : openBIS.searchSamples(criteria, options).getObjects()) { + String space = s.getSpace().getCode(); + String type = s.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(s))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(s); + } + } + return result; + } + + public Map>> getDatasetsByTypeAndSpace(List spaces) { + Map>> result = new HashMap<>(); + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withSample().withSpace(); + options.withExperiment().withProject().withSpace(); + options.withType(); + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withOrOperator(); + criteria.withSample().withSpace().withCodes().thatIn(spaces); + criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + for (DataSet d : openBIS.searchDataSets(criteria, options).getObjects()) { + String space = getSpaceFromSampleOrExperiment(d); + String type = d.getType().getCode(); + if(!result.containsKey(space)) { + Map> typeMap = new HashMap<>(); + typeMap.put(type, new ArrayList<>(Arrays.asList(d))); + result.put(space, typeMap); + } else { + Map> typeMap = result.get(space); + if(!typeMap.containsKey(type)) { + typeMap.put(type, new ArrayList<>()); + } + typeMap.get(type).add(d); + } + } + return result; + } + + private String getSpaceFromSampleOrExperiment(DataSet d) { + try { + if (d.getSample() != null) { + return d.getSample().getSpace().getCode(); + } + if (d.getExperiment() != null) { + return d.getExperiment().getProject().getSpace().getCode(); + } + } catch (NullPointerException e) { + + } + System.out.println("Dataset " + d + "does not seem to be attached to a space"); + return "NO SPACE"; + } + + private List getSamplesWithAncestorsOfExperiment(ExperimentIdentifier experimentId) { + SampleFetchOptions allProps = new SampleFetchOptions(); + allProps.withType(); + allProps.withProperties(); + SampleFetchOptions withAncestors = new SampleFetchOptions(); + withAncestors.withParentsUsing(allProps); + withAncestors.withProperties(); + withAncestors.withType(); + + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withExperiment().withId().thatEquals(experimentId); + + return openBIS.searchSamples(criteria, withAncestors).getObjects(); + } + public List findDataSets(List codes) { if (codes.isEmpty()) { return new ArrayList<>(); @@ -202,6 +345,7 @@ public List findDataSets(List codes) { DataSetSearchCriteria criteria = new DataSetSearchCriteria(); criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); + options.withExperiment(); return openBIS.searchDataSets(criteria, options).getObjects(); } @@ -212,4 +356,5 @@ public boolean experimentExists(String experimentID) { return !openBIS.searchExperiments(criteria, new ExperimentFetchOptions()).getObjects() .isEmpty(); } + } From a9f9e530ed6f393c9c9935882e91cec115ffe0e3 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 15 Jul 2024 18:00:25 +0200 Subject: [PATCH 14/63] fix issues introduced by git cherry pick --- .../java/life/qbic/io/commandline/CommandLineOptions.java | 4 ++-- src/main/java/life/qbic/model/download/FileSystemWriter.java | 2 +- .../model/download/{ModelReporter.java => SummaryWriter.java} | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) rename src/main/java/life/qbic/model/download/{ModelReporter.java => SummaryWriter.java} (65%) diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index d050ebf..f2d1538 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,8 +8,8 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class, - UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class }, + subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, + UploadDatasetCommand.class, SpaceStatisticsCommand.class }, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/model/download/FileSystemWriter.java b/src/main/java/life/qbic/model/download/FileSystemWriter.java index 4cf0cb9..59b6419 100644 --- a/src/main/java/life/qbic/model/download/FileSystemWriter.java +++ b/src/main/java/life/qbic/model/download/FileSystemWriter.java @@ -14,7 +14,7 @@ * * @author: Sven Fillinger, Andreas Friedrich */ -public class FileSystemWriter implements ModelReporter { +public class FileSystemWriter implements SummaryWriter { /** * File that stores the summary report content for valid checksums. diff --git a/src/main/java/life/qbic/model/download/ModelReporter.java b/src/main/java/life/qbic/model/download/SummaryWriter.java similarity index 65% rename from src/main/java/life/qbic/model/download/ModelReporter.java rename to src/main/java/life/qbic/model/download/SummaryWriter.java index 875ffb4..dbf2438 100644 --- a/src/main/java/life/qbic/model/download/ModelReporter.java +++ b/src/main/java/life/qbic/model/download/SummaryWriter.java @@ -1,11 +1,9 @@ package life.qbic.model.download; import java.io.IOException; -import java.net.URL; -import java.nio.file.Path; import java.util.List; -public interface ModelReporter { +public interface SummaryWriter { void reportSummary(List summary) throws IOException; } From 5f773a1dd86e9c018bf70bdaba6c0a981cfda3f8 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 16 Jul 2024 14:04:43 +0200 Subject: [PATCH 15/63] add missing class --- .../qbic/model/DatasetWithProperties.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/main/java/life/qbic/model/DatasetWithProperties.java diff --git a/src/main/java/life/qbic/model/DatasetWithProperties.java b/src/main/java/life/qbic/model/DatasetWithProperties.java new file mode 100644 index 0000000..d9a1534 --- /dev/null +++ b/src/main/java/life/qbic/model/DatasetWithProperties.java @@ -0,0 +1,60 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.person.Person; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +/** + * Wrapper class for openBIS DataSets that collects additional information, e.g. from samples, + * experiments etc. further up in the hierarchy. + */ +public class DatasetWithProperties { + + private final DataSet dataset; + private final Map properties; + + public DatasetWithProperties(DataSet dataset) { + this.dataset = dataset; + this.properties = new HashMap<>(); + } + + public void addProperty(String key, String value) { + this.properties.put(key, value); + } + + public String getProperty(String key) { + return properties.get(key); + } + + public Map getProperties() { + return properties; + } + + public DataSet getDataset() { + return dataset; + } + + public String getCode() { + return dataset.getCode(); + } + + public Experiment getExperiment() { + return dataset.getExperiment(); + } + + public DataSetType getType() { + return dataset.getType(); + } + + public Person getRegistrator() { + return dataset.getRegistrator(); + } + + public Date getRegistrationDate() { + return dataset.getRegistrationDate(); + } +} From a9bca1aebb1c39f26892f0e6be0da35537029e3d Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 23 Jul 2024 18:41:06 +0200 Subject: [PATCH 16/63] add petab model, implement simple use cases --- src/main/java/life/qbic/io/PetabParser.java | 122 ++++++++---------- .../io/commandline/DownloadPetabCommand.java | 14 +- .../commandline/UploadPetabResultCommand.java | 8 +- .../qbic/model/petab/ConditionWithUnit.java | 7 + .../model/petab/ExperimentalCondition.java | 15 +++ .../life/qbic/model/petab/IdWithPattern.java | 6 + 6 files changed, 95 insertions(+), 77 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index b7c7bd1..2ade55e 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -1,20 +1,19 @@ package life.qbic.io; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; -import life.qbic.model.petab.MetaInformation; import life.qbic.model.petab.PetabMetadata; public class PetabParser { @@ -31,14 +30,24 @@ public PetabMetadata parse(String dataPath) { BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(yaml)); + boolean inIDBlock = false; while (true) { String line = reader.readLine(); if (line == null) { break; } - if (line.startsWith("openbisID")) { - String datasetCode = line.strip().split(": ")[1]; - sourcePetabReferences.add(datasetCode); + if(inIDBlock && line.contains(":")) { + inIDBlock = false; + } + if(inIDBlock) { + String[] tokens = line.split("-"); + if(tokens.length == 3) { + String datasetCode = tokens[1].strip()+"-"+tokens[2].strip(); + sourcePetabReferences.add(datasetCode); + } + } + if (line.contains("openBISSourceIds:")) { + inIDBlock = true; } } reader.close(); @@ -50,6 +59,18 @@ public PetabMetadata parse(String dataPath) { return new PetabMetadata(sourcePetabReferences); } + public void addDatasetId(String outputPath, String datasetCode) throws IOException { + + Path path = Paths.get(Objects.requireNonNull(findYaml(new File(outputPath))).getPath()); + Charset charset = StandardCharsets.UTF_8; + + String idInLine = "openBISId:(.*)?(\\r\\n|[\\r\\n])"; + + String content = Files.readString(path, charset); + content = content.replaceAll(idInLine, "openBISId: "+datasetCode+"\n"); + Files.write(path, content.getBytes(charset)); + } + private File findYaml(File directory) { for (File file : Objects.requireNonNull(directory.listFiles())) { if (file.isFile() && file.getName().equals(META_INFO_YAML)) { @@ -90,8 +111,13 @@ public void addParameters(String outputPath, Map properties) { } public static void main(String[] args) throws IOException { + + /* + + File testfile = new File("/Users/afriedrich/git/openbis-20-scripts/example_petab/metaInformation.yaml"); ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); MetaInformation metaInfo = mapper.readValue(testfile, MetaInformation.class); System.err.println(metaInfo); @@ -104,72 +130,26 @@ public static void main(String[] args) throws IOException { MetaInformation metaInfo2 = mapper2.readValue(testfile, MetaInformation.class); System.err.println(metaInfo2); - /* -MetaInformation{ - units=Units{ - measurement='some technique', - time='min, - t0 = timepoint of first intervention', - stimulus='something', - medium=Medium{type='DMEM', volume=1.5, unit='ml'}, - ncells=CellCountInfo{seeded=0.0, count='null', unit='null'}, - measurement_technique='immunublotting', - openBISId='null', openBISParentIds=null, - dateOfExperiment=[2024-04-30] - }, - preprocessingInformation=null, - measurementData=MeasurementData{ - measurement=Measurement{ - unit='intensity (a.u.)', lloq='null'}, - time=Time{unit='min'}, - replicateId=IdWithPattern{name='null', pattern='date_gel_replicate'} - }, - experimentalCondition=ExperimentalCondition{conditionId=IdWithPattern{name='null', pattern='condition'}, - conditions=null}} - - - ExperimentInformation: -units: - measurement: some technique - time: min, t0 = timepoint of first intervention - treatment: - stimulus: something - medium: - type: DMEM - volume: 1.5 - unit: ml - ncells: - seeded: 0.4 - ncellsCount: ~ - unit: mio - measurement_technique: immunublotting - openBISId: ~ - dateOfExperiment: - - 2024-04-30 -PreprocessingInformation: - normalizationStatus: Raw (data on linear scale) - preprocessing: - method: normalize by blotIt - arguments: - housekeeperObservableIds: - description: -measurementData: - measurement: - unit: intensity (a.u.) - lloq: ~ - time: - unit: min - replicateId: - pattern: date_gel_replicate -experimentalCondition: + experimentalCondition: conditionId: - pattern: condition - TGFb: + pattern: treatment1_treatment2 + - treatment1: TGFb unit: ng/ul - GAS6: + - treatment2: GAS6 unit: ug/ml - */ + File test = new File("/Users/afriedrich/git/openbis-20-scripts/Output.yaml"); + ExperimentalCondition metaInfo = mapper.readValue(test, ExperimentalCondition.class); + System.err.println(metaInfo); + + + List list = new ArrayList<>(); + list.add(new ConditionWithUnit("TGFb", "ng/ul")); + list.add(new ConditionWithUnit("GAS6", "ug/ml")); + ExperimentalCondition conditions = new ExperimentalCondition( + new IdWithPattern("treatment1_treatment2"), list); + //mapper.writeValue(new File("/Users/afriedrich/git/openbis-20-scripts/Output.yaml"),conditions); +*/ } } diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index 0dc9d95..d40f08a 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -2,6 +2,7 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -39,18 +40,21 @@ public void run() { Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", result.getExperiment().getIdentifier()); patientID.ifPresent(s -> result.addProperty("patientID", s)); - result.addProperty("openbisID", datasetCode); System.out.println("Found dataset, downloading."); System.out.println(); openbis.downloadDataset(outputPath, datasetCode); - System.out.println("Adding additional information to petab.yaml"); - System.out.println(); - PetabParser parser = new PetabParser(); - parser.addParameters(outputPath, result.getProperties()); + try { + System.out.println("Adding dataset identifier to metaInformation.yaml."); + parser.addDatasetId(outputPath, datasetCode); + } catch (IOException e) { + System.out.println("Could not add dataset identifier."); + throw new RuntimeException(e); + } + //parser.addParameters(outputPath, result.getProperties()); //TODO: might be used in the future System.out.println("Done"); } diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index eedc04b..33917fc 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -45,13 +45,19 @@ public void run() { System.out.printf("Experiment %s could not be found%n", experimentID); return; } + System.out.println("Looking for reference datasets in metaInformation.yaml..."); parents = petabParser.parse(dataPath).getSourcePetabReferences(); + if(parents.isEmpty()) { + System.out.println("No reference datasets found. Did you set the openBISSourceIds property?"); + } else { + System.out.println("Found reference ids: "+String.join(", ",parents)); + } if(!datasetsExist(parents)) { System.out.printf("One or more datasets %s could not be found%n", parents); return; } System.out.println(); - System.out.println("Parameters verified, uploading dataset..."); + System.out.println("Reference datasets found, uploading dataset..."); System.out.println();//TODO copy and remove source references DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); diff --git a/src/main/java/life/qbic/model/petab/ConditionWithUnit.java b/src/main/java/life/qbic/model/petab/ConditionWithUnit.java index 802f961..d46e925 100644 --- a/src/main/java/life/qbic/model/petab/ConditionWithUnit.java +++ b/src/main/java/life/qbic/model/petab/ConditionWithUnit.java @@ -9,6 +9,13 @@ public class ConditionWithUnit { @JsonProperty String unit; + public ConditionWithUnit() {} + + public ConditionWithUnit(String name, String unit) { + this.name = name; + this.unit = unit; + } + @Override public String toString() { return "ConditionWithUnit{" + diff --git a/src/main/java/life/qbic/model/petab/ExperimentalCondition.java b/src/main/java/life/qbic/model/petab/ExperimentalCondition.java index 21a62f6..8672a80 100644 --- a/src/main/java/life/qbic/model/petab/ExperimentalCondition.java +++ b/src/main/java/life/qbic/model/petab/ExperimentalCondition.java @@ -10,6 +10,21 @@ public class ExperimentalCondition { @JsonProperty List conditions; + public ExperimentalCondition() {} + + public ExperimentalCondition(IdWithPattern pattern, List conditions) { + this.conditionId = pattern; + this.conditions = conditions; + } + + public void setConditions(List conditions) { + this.conditions = conditions; + } + + public void setConditionId(IdWithPattern id) { + this.conditionId = id; + } + @Override public String toString() { return "ExperimentalCondition{" + diff --git a/src/main/java/life/qbic/model/petab/IdWithPattern.java b/src/main/java/life/qbic/model/petab/IdWithPattern.java index 0e94c7b..98ab352 100644 --- a/src/main/java/life/qbic/model/petab/IdWithPattern.java +++ b/src/main/java/life/qbic/model/petab/IdWithPattern.java @@ -7,6 +7,12 @@ public class IdWithPattern { @JsonProperty String pattern; + public IdWithPattern() {} + + public IdWithPattern(String pattern) { + this.pattern = pattern; + } + @Override public String toString() { return "IdWithPattern{" + From dbcbfb7de4ba42eb070c9319f0182f42ee360d5a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 23 Jul 2024 18:41:55 +0200 Subject: [PATCH 17/63] remove debugging --- src/main/java/life/qbic/io/PetabParser.java | 41 --------------------- 1 file changed, 41 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index 2ade55e..c5037c0 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -110,46 +110,5 @@ public void addParameters(String outputPath, Map properties) { } } - public static void main(String[] args) throws IOException { - - /* - - - File testfile = new File("/Users/afriedrich/git/openbis-20-scripts/example_petab/metaInformation.yaml"); - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - MetaInformation metaInfo = mapper.readValue(testfile, MetaInformation.class); - System.err.println(metaInfo); - metaInfo.getUnits().setOpenbisParentIds(Arrays.asList("1","2")); - metaInfo.getUnits().setOpenbisId("3"); - mapper.writeValue(new File("/Users/afriedrich/git/openbis-20-scripts/Output.yaml"), metaInfo); - - - ObjectMapper mapper2 = new YAMLMapper(); - MetaInformation metaInfo2 = mapper2.readValue(testfile, MetaInformation.class); - System.err.println(metaInfo2); - - experimentalCondition: - conditionId: - pattern: treatment1_treatment2 - - treatment1: TGFb - unit: ng/ul - - treatment2: GAS6 - unit: ug/ml - - File test = new File("/Users/afriedrich/git/openbis-20-scripts/Output.yaml"); - ExperimentalCondition metaInfo = mapper.readValue(test, ExperimentalCondition.class); - System.err.println(metaInfo); - - - List list = new ArrayList<>(); - list.add(new ConditionWithUnit("TGFb", "ng/ul")); - list.add(new ConditionWithUnit("GAS6", "ug/ml")); - ExperimentalCondition conditions = new ExperimentalCondition( - new IdWithPattern("treatment1_treatment2"), list); - //mapper.writeValue(new File("/Users/afriedrich/git/openbis-20-scripts/Output.yaml"),conditions); -*/ - } } From 62d5ef5ecb07b5c70ac901434afda3d1c8d8c1ff Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Jul 2024 12:55:05 +0200 Subject: [PATCH 18/63] cleanup unused code --- src/main/java/life/qbic/io/PetabParser.java | 29 ------------------- .../io/commandline/DownloadPetabCommand.java | 1 - .../commandline/UploadPetabResultCommand.java | 3 +- 3 files changed, 2 insertions(+), 31 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index c5037c0..f12e4a6 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -2,7 +2,6 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.nio.charset.Charset; @@ -12,7 +11,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import life.qbic.model.petab.PetabMetadata; @@ -84,31 +82,4 @@ private File findYaml(File directory) { return null; } - /** - * adds key-value pairs to the bottom of a petab.yaml found below the provided path - * - * @param outputPath the path of the PEtab - * @param properties map of properties to add - */ - public void addParameters(String outputPath, Map properties) { - - File directory = new File(outputPath); - - File yaml = findYaml(directory); - if (yaml != null) { - FileOutputStream fos = null; - try { - fos = new FileOutputStream(yaml.getPath(), true); - for (Map.Entry entry : properties.entrySet()) { - String line = entry.getKey() + ": " + entry.getValue() + "\n"; - fos.write(line.getBytes()); - } - fos.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - } diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index d40f08a..f09d973 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -54,7 +54,6 @@ public void run() { System.out.println("Could not add dataset identifier."); throw new RuntimeException(e); } - //parser.addParameters(outputPath, result.getProperties()); //TODO: might be used in the future System.out.println("Done"); } diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 33917fc..45c080f 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -58,7 +58,8 @@ public void run() { } System.out.println(); System.out.println("Reference datasets found, uploading dataset..."); - System.out.println();//TODO copy and remove source references + System.out.println(); + //TODO copy and remove source references here DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } From f906e75ff253e9eea7a468a1a53885e6d09bd37d Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Jul 2024 13:01:00 +0200 Subject: [PATCH 19/63] disable parents parameter, it is read from yaml now --- .../life/qbic/io/commandline/UploadPetabResultCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 45c080f..4c3d7de 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -23,8 +23,8 @@ public class UploadPetabResultCommand implements Runnable { @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. " + "The identifier must be of the format: /space/project/experiment") private String experimentID; - @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" - + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) + //@Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + // + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @Mixin AuthenticationOptions auth = new AuthenticationOptions(); From 573eb3c93cbac956267eec0c8db7e10b738f8c7c Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Jul 2024 13:49:42 +0200 Subject: [PATCH 20/63] implement requested changes --- src/main/java/life/qbic/io/PetabParser.java | 29 ++++++++++++++----- .../commandline/UploadPetabResultCommand.java | 21 +++++++------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index f12e4a6..2973bf5 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -34,15 +34,14 @@ public PetabMetadata parse(String dataPath) { if (line == null) { break; } + // the id block ends, when a new key with colon is found if(inIDBlock && line.contains(":")) { inIDBlock = false; } + // if we are in the id block, we collect one dataset code per line if(inIDBlock) { - String[] tokens = line.split("-"); - if(tokens.length == 3) { - String datasetCode = tokens[1].strip()+"-"+tokens[2].strip(); - sourcePetabReferences.add(datasetCode); - } + String datasetCode = parseDatasetCode(line); + sourcePetabReferences.add(datasetCode); } if (line.contains("openBISSourceIds:")) { inIDBlock = true; @@ -57,21 +56,35 @@ public PetabMetadata parse(String dataPath) { return new PetabMetadata(sourcePetabReferences); } + private String parseDatasetCode(String line) { + // expected input: " - 20240702093837370-684137" + String[] tokens = line.split("-"); + if(tokens.length == 3) { + String datasetCode = tokens[1].strip()+"-"+tokens[2].strip(); + return datasetCode; + } else { + System.out.println("Could not extract dataset code from the following line:"); + System.out.println(line); + } + } + public void addDatasetId(String outputPath, String datasetCode) throws IOException { Path path = Paths.get(Objects.requireNonNull(findYaml(new File(outputPath))).getPath()); Charset charset = StandardCharsets.UTF_8; - String idInLine = "openBISId:(.*)?(\\r\\n|[\\r\\n])"; + final String keyWord = "openBISId"; + + String idInLine = keyWord+":(.*)?(\\r\\n|[\\r\\n])"; String content = Files.readString(path, charset); - content = content.replaceAll(idInLine, "openBISId: "+datasetCode+"\n"); + content = content.replaceAll(idInLine, keyWord+": "+datasetCode+"\n"); Files.write(path, content.getBytes(charset)); } private File findYaml(File directory) { for (File file : Objects.requireNonNull(directory.listFiles())) { - if (file.isFile() && file.getName().equals(META_INFO_YAML)) { + if (file.isFile() && file.getName().equalsIgnoreCase(META_INFO_YAML)) { return file; } if (file.isDirectory()) { diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 4c3d7de..9ed344f 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -11,11 +11,10 @@ import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; -import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; -@Command(name = "upload-petab-result", - description = "uploads a petab based on other PETab downloaded from openbis and attaches it to a provided experiment and any datasets referenced in the PETab metadata.") +@Command(name = "upload-petab", + description = "uploads a PETab folder and attaches it to a provided experiment and any datasets referenced in the PETab metadata (e.g. for PETab results).") public class UploadPetabResultCommand implements Runnable { @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") @@ -50,15 +49,15 @@ public void run() { if(parents.isEmpty()) { System.out.println("No reference datasets found. Did you set the openBISSourceIds property?"); } else { - System.out.println("Found reference ids: "+String.join(", ",parents)); + System.out.println("Found reference ids: " + String.join(", ", parents)); + if (!datasetsExist(parents)) { + System.out.printf("One or more datasets %s could not be found%n", parents); + return; + } else { + System.out.println("Referenced datasets found"); + } } - if(!datasetsExist(parents)) { - System.out.printf("One or more datasets %s could not be found%n", parents); - return; - } - System.out.println(); - System.out.println("Reference datasets found, uploading dataset..."); - System.out.println(); + System.out.println("Uploading dataset..."); //TODO copy and remove source references here DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); From c8b0566cc73a1839cf5993667dc5c1d95c82bd74 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Jul 2024 16:13:11 +0200 Subject: [PATCH 21/63] better handling of dataset code parsing --- src/main/java/life/qbic/io/PetabParser.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index 2973bf5..dd2cbf4 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import life.qbic.model.petab.PetabMetadata; public class PetabParser { @@ -35,13 +36,12 @@ public PetabMetadata parse(String dataPath) { break; } // the id block ends, when a new key with colon is found - if(inIDBlock && line.contains(":")) { + if (inIDBlock && line.contains(":")) { inIDBlock = false; } // if we are in the id block, we collect one dataset code per line - if(inIDBlock) { - String datasetCode = parseDatasetCode(line); - sourcePetabReferences.add(datasetCode); + if (inIDBlock) { + parseDatasetCode(line).ifPresent(code -> sourcePetabReferences.add(code)); } if (line.contains("openBISSourceIds:")) { inIDBlock = true; @@ -56,16 +56,16 @@ public PetabMetadata parse(String dataPath) { return new PetabMetadata(sourcePetabReferences); } - private String parseDatasetCode(String line) { + private Optional parseDatasetCode(String line) { // expected input: " - 20240702093837370-684137" String[] tokens = line.split("-"); if(tokens.length == 3) { - String datasetCode = tokens[1].strip()+"-"+tokens[2].strip(); - return datasetCode; + return Optional.of(tokens[1].strip()+"-"+tokens[2].strip()); } else { System.out.println("Could not extract dataset code from the following line:"); System.out.println(line); } + return Optional.empty(); } public void addDatasetId(String outputPath, String datasetCode) throws IOException { From 431edfb1bb7bc77d3be23ccc62b3e796b2910cb0 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Jul 2024 16:15:35 +0200 Subject: [PATCH 22/63] better code --- src/main/java/life/qbic/io/PetabParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index dd2cbf4..dd99656 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -41,7 +41,7 @@ public PetabMetadata parse(String dataPath) { } // if we are in the id block, we collect one dataset code per line if (inIDBlock) { - parseDatasetCode(line).ifPresent(code -> sourcePetabReferences.add(code)); + parseDatasetCode(line).ifPresent(sourcePetabReferences::add); } if (line.contains("openBISSourceIds:")) { inIDBlock = true; From bcd261925a67f3e64187a011536e29d834388a89 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 24 Jul 2024 16:35:37 +0200 Subject: [PATCH 23/63] better logging --- src/main/java/life/qbic/io/PetabParser.java | 3 +-- .../life/qbic/io/commandline/UploadPetabResultCommand.java | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index dd99656..220e41b 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -52,7 +52,6 @@ public PetabMetadata parse(String dataPath) { throw new RuntimeException(e); } } - return new PetabMetadata(sourcePetabReferences); } @@ -95,4 +94,4 @@ private File findYaml(File directory) { return null; } -} +} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 9ed344f..3f29163 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -47,7 +47,8 @@ public void run() { System.out.println("Looking for reference datasets in metaInformation.yaml..."); parents = petabParser.parse(dataPath).getSourcePetabReferences(); if(parents.isEmpty()) { - System.out.println("No reference datasets found. Did you set the openBISSourceIds property?"); + System.out.println("No reference datasets found in openBISSourceIds property. Assuming" + + "this is a new dataset."); } else { System.out.println("Found reference ids: " + String.join(", ", parents)); if (!datasetsExist(parents)) { From 296c981f24a92c95d8d1a86e83f4103877061780 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 15 Aug 2024 18:21:00 +0200 Subject: [PATCH 24/63] allow dataset connection to sample, improve error handling --- .../io/commandline/AuthenticationOptions.java | 30 +++++--- .../io/commandline/DownloadPetabCommand.java | 2 +- .../io/commandline/FindDatasetsCommand.java | 2 +- .../commandline/SampleHierarchyCommand.java | 2 +- .../commandline/SpaceStatisticsCommand.java | 2 +- .../TransferDataToSeekCommand.java | 73 +++++++++++++++++++ .../io/commandline/UploadDatasetCommand.java | 43 ++++++++--- .../commandline/UploadPetabResultCommand.java | 8 +- .../qbic/model/download/Authentication.java | 61 ---------------- .../qbic/model/download/OpenbisConnector.java | 51 ++++++++++--- 10 files changed, 176 insertions(+), 98 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java delete mode 100644 src/main/java/life/qbic/model/download/Authentication.java diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 948ef7e..6809a06 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -21,9 +21,9 @@ public class AuthenticationOptions { @Option( names = {"-u", "--user"}, description = "openBIS user name") - private String user; + private String openbisUser; @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - PasswordOptions passwordOptions; + PasswordOptions openbisPasswordOptions; @Option( names = {"-as", "-as_url"}, @@ -43,11 +43,23 @@ public class AuthenticationOptions { scope = CommandLine.ScopeType.INHERIT) public String configPath; - public String getUser() { - if(user == null & configPath!=null && !configPath.isBlank()) { - user = ReadProperties.getProperties(configPath).get("user"); + @Option( + names = {"-seek-server", "-seek_url"}, + description = "SEEK API URL", + scope = CommandLine.ScopeType.INHERIT) + private String seek_url; + + public String getOpenbisUser() { + if(openbisUser == null & configPath!=null && !configPath.isBlank()) { + openbisUser = ReadProperties.getProperties(configPath).get("user"); } - return user; + return openbisUser; + } + + public String getSeekURL() { + log.error("No URL to the SEEK address provided."); + System.exit(2); + return seek_url; } public String getDSS() { @@ -64,8 +76,8 @@ public String getAS() { return as_url; } - public char[] getPassword() { - return passwordOptions.getPassword(); + public char[] getOpenbisPassword() { + return openbisPasswordOptions.getPassword(); } /** @@ -110,7 +122,7 @@ char[] getPassword() { @Override public String toString() { return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") - .add("user='" + user + "'") + .add("user='" + openbisUser + "'") .toString(); //ATTENTION: do not expose the password here! } diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index f09d973..46efefe 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -27,7 +27,7 @@ public class DownloadPetabCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 2846fe7..2d340bd 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -39,7 +39,7 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() .sorted(Comparator.comparing( diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 47353b6..53b2bf3 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -43,7 +43,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index bce2144..ec2e2d0 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -50,7 +50,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); if (spaces.isEmpty()) { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java new file mode 100644 index 0000000..5704b3d --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -0,0 +1,73 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import life.qbic.App; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.download.OpenbisConnector; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "openbis-to-seek", + description = "Transfers data or metadata from openBIS to SEEK.") +public class TransferDataToSeekCommand implements Runnable { + + @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset (or its metadata) to transfer. Can be found via list-data.") + private String datasetCode; + @Parameters(arity = "1", paramLabel = "seek node", description = "The node in SEEK to which to transfer the dataset.") + private String seekNode; + @Option(names = { "-d", "--data"}, usageHelp = true, description = "Transfers the data itself to SEEK along with the metadata") + private boolean transferData; + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + + @Override + public void run() { + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenbisConnector openbis = new OpenbisConnector(authentication); + + List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); + + if(datasets.isEmpty()) { + System.out.println(datasetCode+" not found"); + return; + } + DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); + Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + result.getExperiment().getIdentifier()); + patientID.ifPresent(s -> result.addProperty("patientID", s)); + + System.out.println("Found dataset, downloading."); + System.out.println(); + + final String tmpPath = "tmp/"; + + File downloadFolder = openbis.downloadDataset(tmpPath, datasetCode); + + + + cleanupTemp(new File(tmpPath)); + + System.out.println("Done"); + } + + private void cleanupTemp(File tmpFolder) { + File[] files = tmpFolder.listFiles(); + if (files != null) { //some JVMs return null for empty dirs + for (File f : files) { + if (f.isDirectory()) { + cleanupTemp(f); + } else { + f.delete(); + } + } + } + } + +} diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 74fa98e..4e91a4a 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -4,10 +4,6 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import java.io.File; import java.nio.file.Path; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import life.qbic.App; @@ -23,9 +19,9 @@ public class UploadDatasetCommand implements Runnable { @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") private String dataPath; - @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. " - + "The identifier must be of the format: /space/project/experiment") - private String experimentID; + @Parameters(arity = "1", paramLabel = "object ID", description = "The full identifier of the experiment or sample the data should be attached to. " + + "The identifier must be of the format: /space/project/experiment for experiments or /space/sample for samples") + private String objectID; @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @@ -36,15 +32,28 @@ public class UploadDatasetCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { System.out.printf("Path %s could not be found%n", dataPath); return; } - if(!experimentExists(experimentID)) { - System.out.printf("Experiment %s could not be found%n", experimentID); + boolean attachToSample = OpenbisConnector.sampleIdPattern.matcher(objectID).find(); + boolean attachToExperiment = false; + if(!attachToSample) { + attachToExperiment = OpenbisConnector.experimentIdPattern.matcher(objectID).find(); + } + if(!attachToExperiment && !attachToSample) { + System.out.printf("%s is neither a valid experiment nor sample identifier%n", objectID); + return; + } + if(attachToExperiment && !experimentExists(objectID)) { + System.out.printf("Experiment with identifier %s could not be found%n", objectID); + return; + } + if(attachToSample && !sampleExists(objectID)) { + System.out.printf("Sample object with identifier %s could not be found%n", objectID); return; } if(!datasetsExist(parents)) { @@ -54,10 +63,19 @@ public void run() { System.out.println(); System.out.println("Parameters verified, uploading dataset..."); System.out.println(); - DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + if(attachToExperiment) { + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, parents); + System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + } else { + DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); + System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + } } + private boolean sampleExists(String objectID) { + return openbis.sampleExists(objectID); + } + private boolean datasetsExist(List datasetCodes) { return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); } @@ -70,4 +88,5 @@ private boolean pathValid(String dataPath) { return new File(dataPath).exists(); } + } diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 3f29163..6ee72a9 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -33,13 +33,17 @@ public class UploadPetabResultCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getPassword(), auth.getUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { System.out.printf("Path %s could not be found%n", dataPath); return; } + if(!new File(dataPath).isDirectory()) { + System.out.printf("%s is not a directory. Please specify the PETab directory root%n", dataPath); + return; + } if(!experimentExists(experimentID)) { System.out.printf("Experiment %s could not be found%n", experimentID); return; @@ -60,7 +64,7 @@ public void run() { } System.out.println("Uploading dataset..."); //TODO copy and remove source references here - DataSetPermId result = openbis.registerDataset(Path.of(dataPath), experimentID, parents); + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } diff --git a/src/main/java/life/qbic/model/download/Authentication.java b/src/main/java/life/qbic/model/download/Authentication.java deleted file mode 100644 index f151fd7..0000000 --- a/src/main/java/life/qbic/model/download/Authentication.java +++ /dev/null @@ -1,61 +0,0 @@ -package life.qbic.model.download; - -import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; -import ch.systemsx.cisd.common.spring.HttpInvokerUtils; - -public class Authentication { - - private static String sessionToken; - private String user; - private String password; - private final IApplicationServerApi applicationServer; - - - public Authentication( - String user, - String password, - String AppServerUri) { - this.user = user; - this.password = password; - if (!AppServerUri.isEmpty()) { - this.applicationServer = - HttpInvokerUtils.createServiceStub( - IApplicationServerApi.class, AppServerUri + IApplicationServerApi.SERVICE_URL, 10000); - } else { - this.applicationServer = null; - } - this.setCredentials(user, password); - } - - /** - * Login method for openBIS authentication - * - * @throws AuthenticationException in case the authentication failed - */ - public void login() throws ConnectionException, AuthenticationException { - try { - sessionToken = applicationServer.login(user, password); - } catch (Exception e) { - throw new ConnectionException("Connection to openBIS server failed.", e); - } - if (sessionToken == null || sessionToken.isEmpty()) { - throw new AuthenticationException("Authentication failed. Are you using the correct " - + "credentials?"); - } - } - - public String getSessionToken() { - return sessionToken; - } - - /** - * Setter for user and password credentials - * - * @param user The openBIS user - * @param password The openBIS user's password - */ - public void setCredentials(String user, String password) { - this.user = user; - this.password = password; - } -} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index cf822a7..c73d796 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -15,6 +15,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Collectors; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; @@ -45,7 +47,10 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); - OpenBIS openBIS; + private final OpenBIS openBIS; + + public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -58,16 +63,10 @@ public List getSpaces() { .stream().map(Space::getCode).collect(Collectors.toList()); } - public DataSetPermId registerDataset(Path uploadPath, String experimentID, + public DataSetPermId registerDatasetForExperiment(Path uploadPath, String experimentID, List parentCodes) { - final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); - - final UploadedDataSetCreation creation = new UploadedDataSetCreation(); - creation.setUploadId(uploadId); + UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, parentCodes); creation.setExperimentId(new ExperimentIdentifier(experimentID)); - creation.setParentIds(parentCodes.stream().map(DataSetPermId::new).collect( - Collectors.toList())); - creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); try { return openBIS.createUploadedDataSet(creation); @@ -77,6 +76,30 @@ public DataSetPermId registerDataset(Path uploadPath, String experimentID, return null; } + public DataSetPermId registerDatasetForSample(Path uploadPath, String sampleID, + List parentCodes) { + UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, parentCodes); + creation.setSampleId(new SampleIdentifier(sampleID)); + + try { + return openBIS.createUploadedDataSet(creation); + } catch (final Exception e) { + LOG.error(e.getMessage()); + } + return null; + } + + private UploadedDataSetCreation prepareDataSetCreation(Path uploadPath, List parentCodes) { + final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); + + final UploadedDataSetCreation creation = new UploadedDataSetCreation(); + creation.setUploadId(uploadId); + creation.setParentIds(parentCodes.stream().map(DataSetPermId::new).collect( + Collectors.toList())); + creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); + return creation; + } + private static void copyInputStreamToFile(InputStream inputStream, File file) throws IOException { System.err.println(file.getPath()); @@ -103,7 +126,7 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public void downloadDataset(String targetPath, String datasetID) { + public File downloadDataset(String targetPath, String datasetID) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), ""); @@ -135,6 +158,7 @@ public void downloadDataset(String targetPath, String datasetID) { } } } + return new File(targetPath); } public Map queryFullSampleHierarchy(List spaces) { @@ -357,4 +381,11 @@ public boolean experimentExists(String experimentID) { .isEmpty(); } + public boolean sampleExists(String objectID) { + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withIdentifier().thatEquals(objectID); + + return !openBIS.searchSamples(criteria, new SampleFetchOptions()).getObjects() + .isEmpty(); + } } From eaf8d7e761d15fd55a9d9c4768b1c92a2667a6bf Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 2 Sep 2024 15:30:52 +0200 Subject: [PATCH 25/63] Update src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java --- .../java/life/qbic/io/commandline/UploadDatasetCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 4e91a4a..bd05795 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -65,7 +65,7 @@ public void run() { System.out.println(); if(attachToExperiment) { DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + System.out.printf("Dataset %s was successfully attached to experiment%n", result.getPermId()); } else { DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); From cb3933dac84f12956e31c8d1a8b0e3cb0388d080 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 2 Sep 2024 15:31:35 +0200 Subject: [PATCH 26/63] Update src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java --- .../java/life/qbic/io/commandline/UploadDatasetCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index bd05795..f165c9a 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -68,7 +68,7 @@ public void run() { System.out.printf("Dataset %s was successfully attached to experiment%n", result.getPermId()); } else { DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + System.out.printf("Dataset %s was successfully attached to sample%n", result.getPermId()); } } From 4d28df98854fd7108942e52f421d26593139739a Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Thu, 26 Sep 2024 17:30:29 +0200 Subject: [PATCH 27/63] wip --- .../io/commandline/AuthenticationOptions.java | 85 +++- .../io/commandline/CommandLineOptions.java | 10 +- .../TransferDataToSeekCommand.java | 244 ++++++++++- .../io/commandline/UploadDatasetCommand.java | 20 +- .../OpenbisExperimentWithDescendants.java | 40 ++ .../qbic/model/OpenbisSeekTranslator.java | 121 ++++++ .../java/life/qbic/model/SeekStructure.java | 34 ++ .../qbic/model/download/OpenbisConnector.java | 71 +++- .../qbic/model/download/SEEKConnector.java | 397 ++++++++++++++++++ .../qbic/model/isa/AbstractISAObject.java | 22 + .../java/life/qbic/model/isa/ISAAssay.java | 257 ++++++++++++ .../java/life/qbic/model/isa/ISADataFile.java | 203 +++++++++ .../java/life/qbic/model/isa/ISASample.java | 168 ++++++++ .../java/life/qbic/model/isa/ISAStudy.java | 158 +++++++ 14 files changed, 1785 insertions(+), 45 deletions(-) create mode 100644 src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java create mode 100644 src/main/java/life/qbic/model/OpenbisSeekTranslator.java create mode 100644 src/main/java/life/qbic/model/SeekStructure.java create mode 100644 src/main/java/life/qbic/model/download/SEEKConnector.java create mode 100644 src/main/java/life/qbic/model/isa/AbstractISAObject.java create mode 100644 src/main/java/life/qbic/model/isa/ISAAssay.java create mode 100644 src/main/java/life/qbic/model/isa/ISADataFile.java create mode 100644 src/main/java/life/qbic/model/isa/ISASample.java create mode 100644 src/main/java/life/qbic/model/isa/ISAStudy.java diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 6809a06..1b9c5d0 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -23,7 +23,9 @@ public class AuthenticationOptions { description = "openBIS user name") private String openbisUser; @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - PasswordOptions openbisPasswordOptions; + OpenbisPasswordOptions openbisPasswordOptions; + @ArgGroup(multiplicity = "1") + SeekPasswordOptions seekPasswordOptions; @Option( names = {"-as", "-as_url"}, @@ -32,17 +34,21 @@ public class AuthenticationOptions { private String as_url; @Option( - names = {"-dss", "-dss_url"}, + names = {"-dss", "--dss_url"}, description = "DatastoreServer URL", scope = CommandLine.ScopeType.INHERIT) private String dss_url; @Option( - names = {"-config", "-config_file"}, - description = "Config file path to provide openbis server information.", + names = {"-config", "--config_file"}, + description = "Config file path to provide server and user information.", scope = CommandLine.ScopeType.INHERIT) public String configPath; - + @Option( + names = {"-su", "--seek-user"}, + description = "Seek user name (email)", + scope = CommandLine.ScopeType.INHERIT) + private String seekUser; @Option( names = {"-seek-server", "-seek_url"}, description = "SEEK API URL", @@ -52,13 +58,30 @@ public class AuthenticationOptions { public String getOpenbisUser() { if(openbisUser == null & configPath!=null && !configPath.isBlank()) { openbisUser = ReadProperties.getProperties(configPath).get("user"); + } else { + log.error("No openBIS user provided."); + System.exit(2); } return openbisUser; } + public String getSeekUser() { + if(seekUser == null & configPath!=null && !configPath.isBlank()) { + seekUser = ReadProperties.getProperties(configPath).get("seek_user"); + } else { + log.error("No SEEK user/email provided."); + System.exit(2); + } + return seekUser; + } + public String getSeekURL() { - log.error("No URL to the SEEK address provided."); - System.exit(2); + if(seek_url == null & configPath!=null && !configPath.isBlank()) { + seek_url = ReadProperties.getProperties(configPath).get("seek_url"); + } else { + log.error("No URL to the SEEK address provided."); + System.exit(2); + } return seek_url; } @@ -76,6 +99,10 @@ public String getAS() { return as_url; } + public char[] getSeekPassword() { + return seekPasswordOptions.getPassword(); + } + public char[] getOpenbisPassword() { return openbisPasswordOptions.getPassword(); } @@ -83,14 +110,14 @@ public char[] getOpenbisPassword() { /** * official picocli documentation example */ - static class PasswordOptions { - @Option(names = "--password:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + static class OpenbisPasswordOptions { + @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") protected String passwordEnvironmentVariable = ""; - @Option(names = "--password:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") protected String passwordProperty = ""; - @Option(names = "--password", arity = "0", description = "please provide your password", interactive = true) + @Option(names = "--openbis-pw", arity = "0", description = "please provide your openBIS password", interactive = true) protected char[] password = null; /** @@ -116,9 +143,43 @@ char[] getPassword() { System.exit(2); return null; // not reachable due to System.exit in previous line } - } + static class SeekPasswordOptions { + @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } @Override public String toString() { return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index d050ebf..a0dcbad 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,11 +8,13 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = { SampleHierarchyCommand.class, FindDatasetsCommand.class, DownloadPetabCommand.class, - UploadPetabResultCommand.class, UploadDatasetCommand.class, SpaceStatisticsCommand.class }, - description = "A client software for querying openBIS.", - mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) + subcommands = {SampleHierarchyCommand.class, FindDatasetsCommand.class, + DownloadPetabCommand.class, UploadPetabResultCommand.class, UploadDatasetCommand.class, + SpaceStatisticsCommand.class, TransferDataToSeekCommand.class}, + description = "A client software for querying openBIS.", + mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { + private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); @Option(names = {"-V", "--version"}, diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 5704b3d..ec9bdaf 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -2,35 +2,164 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; -import java.util.Collections; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import life.qbic.App; -import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SeekStructure; import life.qbic.model.download.OpenbisConnector; +import life.qbic.model.download.SEEKConnector; +import life.qbic.model.download.SEEKConnector.AssetToUpload; +import org.apache.commons.codec.binary.Base64; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @Command(name = "openbis-to-seek", - description = "Transfers data or metadata from openBIS to SEEK.") + description = + "Transfers metadata and (optionally) data from openBIS to SEEK. Experiments, samples and " + + "dataset information are always transferred together (as assays, samples and one of " + + "several data types in SEEK). Dataset info always links back to the openBIS path of " + + "the respective dataset. The data itself will can transferred and stored in SEEK " + + "using the '-d' flag." + + "To completely exclude some dataset information from being transferred, a " + + "file ('--blacklist') containing dataset codes (from openBIS) can be specified." + + "Unless otherwise specified ('--no-update' flag), the command will try to update " + + "existing nodes in SEEK (recognized by openBIS identifiers in their metadata).") public class TransferDataToSeekCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset (or its metadata) to transfer. Can be found via list-data.") - private String datasetCode; - @Parameters(arity = "1", paramLabel = "seek node", description = "The node in SEEK to which to transfer the dataset.") + @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + + "experiment, sample or dataset to transfer.") + private String objectID; + @Option(names = "--blacklist", description = "Path to file specifying by dataset " + + "dataset code which openBIS datasets not to transfer to SEEK. The file must contain one code " + + "per line.") + private String blacklistFile; + @Option(names = "--no-update", description = "Use to specify that existing " + + "information in SEEK for the specified openBIS input should not be updated, but new nodes " + + "created.") + private boolean noUpdate; + /*@Option(names = {"-sn", "--seek-node"}, paramLabel = "seek node", description = + "The target node in SEEK to transfer to. Must correspond to " + + "the type of oopenBIS identifier chosen: experiment - assay; sample - sample; dataset - any of the data types. If no node is specified, " + + "a new data structure will be created in SEEK, starting from the related experiment.") private String seekNode; - @Option(names = { "-d", "--data"}, usageHelp = true, description = "Transfers the data itself to SEEK along with the metadata") + */ + @Option(names = {"-d", "--data"}, description = + "Transfers the data itself to SEEK along with the metadata. " + + "Otherwise only the link(s) to the openBIS object will be created in SEEK.") private boolean transferData; @Mixin AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisConnector openbis; + SEEKConnector seek; + OpenbisSeekTranslator translator = new OpenbisSeekTranslator(); - @Override - public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); - OpenbisConnector openbis = new OpenbisConnector(authentication); + @Override + public void run() { + System.out.println("auth..."); + + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getAS(), auth.getDSS()); + System.out.println("openbis..."); + + openbis = new OpenbisConnector(authentication); + + boolean isSample = false; + boolean isDataSet = false; + System.out.println("search for experiment..."); + + boolean isExperiment = experimentExists(objectID); + if (!isExperiment && sampleExists(objectID)) { + isSample = true; + } + + if (!isExperiment && !isSample && datasetsExist(Arrays.asList(objectID))) { + isDataSet = true; + } + + if (!isSample && !isExperiment && !isDataSet) { + System.out.printf( + "%s could not be found in openBIS. Make sure you either specify an experiment, sample or dataset%n", + objectID); + return; + } + System.out.println("Searching done..."); +/* + if (isExperiment) { + if (seekNode != null && !seekNode.contains("assays")) { + System.out.printf( + "Seek node %s does not correspond to the provided openBIS experiment identifier. " + + "Please select an assay node.%n", objectID); + return; + } + } + if (isSample) { + if (seekNode != null && !seekNode.contains("samples")) { + System.out.printf( + "Seek node %s does not correspond to the provided openBIS sample identifier. " + + "Please select a sample node.%n", objectID); + return; + } + } + + */ + + byte[] httpCredentials = Base64.encodeBase64( + (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + try { + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "Default Project", + "lisym default study"); + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + /* + if (seekNode != null) { + try { + if (!seek.endPointExists(seekNode)) { + System.out.println(seekNode + " could not be found"); + return; + } else { + if (isExperiment) { + Experiment experimentWithSamplesAndDatasets = openbis.getExperimentWithDescendants( + objectID); + //SeekStructure seekStructure = translator.translate(seekNode, experimentWithSamplesAndDatasets); + //seek.fillAssayWithSamplesAndDatasets(seekNode, seekStructure); + } + } + } catch (URISyntaxException | InterruptedException | IOException e) { + throw new RuntimeException(e); + } + } + + */ + try { + OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); + String assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); + List blacklist = parseBlackList(blacklistFile); + if(assayID.isBlank()) { + createNewNodes(experiment, blacklist); + } else { + updateNodes(experiment, assayID, blacklist); + } + } catch (URISyntaxException | InterruptedException | IOException e) { + throw new RuntimeException(e); + } + + /* List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); @@ -38,6 +167,7 @@ public void run() { System.out.println(datasetCode+" not found"); return; } + DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", result.getExperiment().getIdentifier()); @@ -53,9 +183,97 @@ public void run() { cleanupTemp(new File(tmpPath)); +*/ + System.out.println("Done"); + } - System.out.println("Done"); + private List parseBlackList(String blacklistFile) { + List result = new ArrayList<>(); + if(blacklistFile == null) { + return result; + } + // trim whitespace, skip empty lines + try (Stream lines = Files.lines(Paths.get(blacklistFile)) + .map(String::trim) + .filter(s -> !s.isBlank())) { + return lines.collect(Collectors.toList()); + } catch (IOException e) { + throw new RuntimeException(blacklistFile+" could not be found or read."); } + } + + private void updateNodes(OpenbisExperimentWithDescendants experiment, String assayID, List blacklist) { + System.err.println("updating nodes of assay "+assayID); + SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); + System.out.printf("%s was successfully updated.%n", endpoint); + } + + private void createNewNodes(OpenbisExperimentWithDescendants experiment, List blacklist) + throws URISyntaxException, IOException, InterruptedException { + System.err.println("creating new nodes"); + SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + List assetsToUpload = seek.createNode(nodeWithChildren, transferData); + if(transferData) { + for(AssetToUpload asset : assetsToUpload) { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } + } + System.out.printf("%s was successfully created.%n", endpoint); + } + + private boolean sampleExists(String objectID) { + return openbis.sampleExists(objectID); + } + + private boolean datasetsExist(List datasetCodes) { + return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); + } + + private boolean experimentExists(String experimentID) { + return openbis.experimentExists(experimentID); + } + + private String getAssayIDForOpenBISExperiment(Experiment experiment) + throws URISyntaxException, IOException, InterruptedException { + // the perm id is unique and afaik not used by scientists. it is highly unlikely that it would + // "accidentally" be part of another title or description. however, care should be taken here, + // because if a perm id is found in the wrong SEEK node, meta-information in SEEK could be + // overwritten or samples/data added to the wrong assay. + String permID = experiment.getPermId().getPermId(); + List assayIDs = seek.searchAssaysContainingKeyword(permID); + if(assayIDs.isEmpty()) { + return ""; + } + if(assayIDs.size() == 1) { + return assayIDs.get(0); + } + throw new RuntimeException("Search term "+permID+ " was found in more than one assay: "+assayIDs); + } + + private void sendDatasetToSeek(String datasetCode, String assayID) + throws URISyntaxException, IOException, InterruptedException { + assayID = "3"; + System.out.println("Searching dataset in openBIS..."); + List datasets = openbis.findDataSets( + Arrays.asList(datasetCode)); + if(datasets.isEmpty()) { + return; + } + DataSet dataset = datasets.get(0); + List files = openbis.getDatasetFiles(dataset); + List assets = seek.createAssets(files, dataset.getType().getCode(), + Arrays.asList(assayID)); + for(AssetToUpload asset : assets) { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(datasetCode, asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } + } private void cleanupTemp(File tmpFolder) { File[] files = tmpFolder.listFiles(); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index f165c9a..9e2d3a9 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -39,21 +39,13 @@ public void run() { System.out.printf("Path %s could not be found%n", dataPath); return; } - boolean attachToSample = OpenbisConnector.sampleIdPattern.matcher(objectID).find(); - boolean attachToExperiment = false; - if(!attachToSample) { - attachToExperiment = OpenbisConnector.experimentIdPattern.matcher(objectID).find(); + boolean attachToSample = false; + boolean attachToExperiment = experimentExists(objectID); + if(sampleExists(objectID)) { + attachToSample = true; } - if(!attachToExperiment && !attachToSample) { - System.out.printf("%s is neither a valid experiment nor sample identifier%n", objectID); - return; - } - if(attachToExperiment && !experimentExists(objectID)) { - System.out.printf("Experiment with identifier %s could not be found%n", objectID); - return; - } - if(attachToSample && !sampleExists(objectID)) { - System.out.printf("Sample object with identifier %s could not be found%n", objectID); + if(!attachToSample && !attachToExperiment) { + System.out.printf("%s could not be found in openBIS.%n", objectID); return; } if(!datasetsExist(parents)) { diff --git a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java new file mode 100644 index 0000000..86399ec --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java @@ -0,0 +1,40 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; + +public class OpenbisExperimentWithDescendants { + + private Experiment experiment; + private List samples; + private List datasets; + private Map> datasetCodeToFiles; + + public OpenbisExperimentWithDescendants(Experiment experiment, List samples, + List datasets, Map> datasetCodeToFiles) { + this.experiment = experiment; + this.samples = samples; + this.datasets = datasets; + this.datasetCodeToFiles = datasetCodeToFiles; + } + + public Experiment getExperiment() { + return experiment; + } + + public List getSamples() { + return samples; + } + + public List getDatasets() { + return datasets; + } + + public List getFilesForDataset(String permID) { + return datasetCodeToFiles.get(permID); + } +} diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java new file mode 100644 index 0000000..a02317d --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -0,0 +1,121 @@ +package life.qbic.model; + +import static java.util.Map.entry; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; + +public class OpenbisSeekTranslator { + + private final String DEFAULT_PROJECT_ID; + private final String DEFAULT_STUDY_ID; + + public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { + this.DEFAULT_PROJECT_ID = defaultProjectID; + this.DEFAULT_STUDY_ID = defaultStudyID; + } + + Map experimentTypeToAssayClass = Map.ofEntries( + entry("00_MOUSE_DATABASE", "EXP"), + entry("00_PATIENT_DATABASE", "EXP"), + entry("00_STANDARD_OPERATING_PROTOCOLS", "EXP"), + entry("01_BIOLOGICAL_EXPERIMENT", "EXP"), + entry("02_MASSSPECTROMETRY_EXPERIMENT", "EXP"),//where is petab modeling attached? + entry("03_HISTOLOGICAL_ANALYSIS", "EXP"), + entry("04_MICRO_CT", "EXP") + ); + + Map experimentTypeToAssayType = Map.ofEntries( + entry("00_MOUSE_DATABASE", ""), + entry("00_PATIENT_DATABASE", ""),//if this is related to measured data, attach it as sample to the assay + entry("00_STANDARD_OPERATING_PROTOCOLS", ""), + entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), + entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), + entry("03_HISTOLOGICAL_ANALYSIS", ""), + entry("04_MICRO_CT", "") + ); + + Map fileExtensionToDataFormat = Map.ofEntries( + entry("fastq.gz", "http://edamontology.org/format_1930"), + entry("fastq", "http://edamontology.org/format_1930"), + entry("json", "http://edamontology.org/format_3464"), + entry("yaml", "http://edamontology.org/format_3750"), + entry("raw", "http://edamontology.org/format_3712"), + entry("tsv", "http://edamontology.org/format_3475"), + entry("csv", "http://edamontology.org/format_3752") + ); + + Map datasetTypeToAssetType = Map.ofEntries( + entry("ANALYSIS_NOTEBOOK", "Document"), + entry("ANALYZED_DATA", "Data_file"), + entry("ATTACHMENT", "Document"), + entry("ELN_PREVIEW", ""), + entry("EXPERIMENT_PROTOCOL", "SOP"), + entry("EXPERIMENT_RESULT", "Document"), + entry("HISTOLOGICAL_SLIDE", "Data_file"), + entry("IB_DATA", "Data_file"), + entry("LUMINEX_DATA", "Data_file"), + entry("MS_DATA_ANALYZED", "Data_file"), + entry("MS_DATA_RAW", "Data_file"), + entry("OTHER_DATA", "Document"), + entry("PROCESSED_DATA", "Data_file"), + entry("PUBLICATION_DATA", "Publication"), + entry("QPCR_DATA", "Data_file"), + entry("RAW_DATA", "Data_file"), + entry("SOURCE_CODE", "Document"), + entry("TEST_CONT", ""), + entry("TEST_DAT", ""), + entry("UNKNOWN", "") + ); + + public SeekStructure translate(String seekNode, Experiment experimentWithSamplesAndDatasets) { + Experiment exp = experimentWithSamplesAndDatasets; + exp.getType(); + return null; + //new ISAAssay(exp.getCode(), ) + } + + public ISADataFile translate(String seekNode, DataSet dataset) { + return null; //new ISADataFile(); + } + + public String assetForDatasetType(String datasetType) { + return "Data_file";//TODO + } + + public String dataFormatAnnotationForExtension(String fileExtension) { + return fileExtensionToDataFormat.get(fileExtension); + } + + public SeekStructure translate(Experiment experiment, List blacklist) { + System.err.println(experiment.getCode()); + + for(Sample sample : experiment.getSamples()) { + String sampleType = getSampleType(sample.getType()); + Map attributes = new HashMap<>(); + ISASample isaSample = new ISASample(attributes, sampleType, + Collections.singletonList(DEFAULT_PROJECT_ID)); + System.err.println(sample.getCode()); + } + for(DataSet dataset : experiment.getDataSets()) { + System.err.println(dataset.getCode()); + } + System.err.println("blacklisted:"); + for(String dataset : blacklist) { + System.err.println(dataset); + } + return null;//TODO + } + + private String getSampleType(SampleType type) { + } +} diff --git a/src/main/java/life/qbic/model/SeekStructure.java b/src/main/java/life/qbic/model/SeekStructure.java new file mode 100644 index 0000000..5d7cc46 --- /dev/null +++ b/src/main/java/life/qbic/model/SeekStructure.java @@ -0,0 +1,34 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; + +public class SeekStructure { + + private ISAAssay assay; + private List samples; + private List dataFiles; + private Map isaToOpenBISFile; + private Map isaFileToAssetType; + //ISASample isaSample = new ISASample(map, sampleTypeID, Arrays.asList(projectID)); + + public ISAAssay getAssay() { + return assay; + } + + public List getSamples() { + return samples; + } + + public Map getIsaToOpenBISFiles() { + return isaToOpenBISFile; + } + + public String getAssetType(ISADataFile dataFile) { + return isaFileToAssetType.get(dataFile); + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index c73d796..e50dd12 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -25,8 +25,10 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownload; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadOptions; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.download.DataSetFileDownloadReader; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.fetchoptions.DataSetFileFetchOptions; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.DataSetFilePermId; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.search.DataSetFileSearchCriteria; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -40,6 +42,8 @@ import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -49,8 +53,9 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); private final OpenBIS openBIS; - public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + //public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + //public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); + public static Pattern datasetCodePattern = Pattern.compile("[0-9]^{17}-[0-9]+"); public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -161,6 +166,22 @@ public File downloadDataset(String targetPath, String datasetID) { return new File(targetPath); } + public InputStream streamDataset(String datasetCode, String filePath) { + DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); + IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetCode), + filePath); + + // Setting recursive flag to true will return both subfolders and files + options.setRecursive(true); + + // Read the contents and print them out + InputStream stream = openBIS.downloadFiles(new ArrayList<>(List.of(fileToDownload)), + options); + + DataSetFileDownloadReader reader = new DataSetFileDownloadReader(stream); + return reader.read().getInputStream(); + } + public Map queryFullSampleHierarchy(List spaces) { Map hierarchy = new HashMap<>(); if (spaces.isEmpty()) { @@ -370,9 +391,14 @@ public List findDataSets(List codes) { criteria.withCodes().thatIn(codes); DataSetFetchOptions options = new DataSetFetchOptions(); options.withExperiment(); + options.withType(); return openBIS.searchDataSets(criteria, options).getObjects(); } + public boolean datasetExists(String code) { + return !findDataSets(new ArrayList<>(Arrays.asList(code))).isEmpty(); + } + public boolean experimentExists(String experimentID) { ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); criteria.withIdentifier().thatEquals(experimentID); @@ -388,4 +414,45 @@ public boolean sampleExists(String objectID) { return !openBIS.searchSamples(criteria, new SampleFetchOptions()).getObjects() .isEmpty(); } + + public OpenbisExperimentWithDescendants getExperimentWithDescendants(String experimentID) { + ExperimentSearchCriteria criteria = new ExperimentSearchCriteria(); + criteria.withIdentifier().thatEquals(experimentID); + + ExperimentFetchOptions fetchOptions = new ExperimentFetchOptions(); + fetchOptions.withType(); + fetchOptions.withProject(); + fetchOptions.withProperties(); + DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); + dataSetFetchOptions.withType(); + dataSetFetchOptions.withRegistrator(); + SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); + sampleFetchOptions.withProperties(); + sampleFetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withSamplesUsing(sampleFetchOptions); + + Experiment experiment = openBIS.searchExperiments(criteria, fetchOptions).getObjects().get(0); + + Map> datasetCodeToFiles = new HashMap<>(); + for(DataSet dataset : experiment.getDataSets()) { + datasetCodeToFiles.put(dataset.getPermId(), getDatasetFiles(dataset)); + } + + return new OpenbisExperimentWithDescendants(experiment, experiment.getSamples(), + experiment.getDataSets() + .stream().map(DatasetWithProperties::new) + .collect(Collectors.toList()), datasetCodeToFiles); + } + + public List getDatasetFiles(DataSet dataset) { + DataSetFileSearchCriteria criteria = new DataSetFileSearchCriteria(); + + DataSetSearchCriteria dataSetCriteria = criteria.withDataSet().withOrOperator(); + dataSetCriteria.withCode().thatEquals(dataset.getCode()); + + SearchResult result = openBIS.searchFiles(criteria, new DataSetFileFetchOptions()); + + return result.getObjects(); + } } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java new file mode 100644 index 0000000..298364e --- /dev/null +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -0,0 +1,397 @@ +package life.qbic.model.download; + +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SeekStructure; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISADataFile; +import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISAStudy; +import org.apache.http.client.utils.URIBuilder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SEEKConnector { + + private static final Logger LOG = LogManager.getLogger(SEEKConnector.class); + private String apiURL; + private byte[] credentials; + private OpenbisSeekTranslator translator; + + public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjectTitle, + String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { + this.apiURL = apiURL; + this.credentials = httpCredentials; + translator = new OpenbisSeekTranslator(searchNodeWithTitle("projects", defaultProjectTitle), + searchNodeWithTitle("studies", defaultStudyTitle)); + } + + public String addAssay(ISAAssay assay) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + public String createStudy(ISAStudy study) + throws IOException, URISyntaxException, InterruptedException, IOException { + String endpoint = apiURL+"/studies"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, study.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + private HttpRequest buildAuthorizedPOSTRequest(String endpoint, String body) throws URISyntaxException { + return HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .POST(HttpRequest.BodyPublishers.ofString(body)).build(); + } + + public boolean studyExists(String id) throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/studies/"+id; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + return response.statusCode() == 200; + } + + /* + -datatype by extension +-assay equals experiment +-investigation: pre-created in SEEK? +-study: project +-patient id should be linked somehow, maybe gender? +-flexible object type to sample type? + */ + + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/samples"; + HashMap map = new HashMap<>(); + //map.put("title", "really?"); + map.put("test_attribute", "okay then"); + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, isaSample.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + + private AssetToUpload createDataFileAsset(String datasetCode, ISADataFile data) + throws IOException, URISyntaxException, InterruptedException { + String endpoint = apiURL+"/data_files"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, data.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201 && response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data") + .path("attributes") + .path("content_blobs") + .path(0).path("link"); + return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode); + } + + @Deprecated + private void uploadFileContent(String assetType, String assetID, String blobID, String file) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/"+assetType+"/"+assetID+"/content_blobs/"+blobID; + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/octet-stream") + .headers("Accept", "application/octet-stream") + .headers("Authorization", "Basic " + new String(credentials)) + .PUT(BodyPublishers.ofFile(new File(file).toPath())).build(); + + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String uploadStreamContent(String blobEndpoint, + Supplier streamSupplier) + throws URISyntaxException, IOException, InterruptedException { + + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(blobEndpoint)) + .headers("Content-Type", "application/octet-stream") + .headers("Accept", "application/octet-stream") + .headers("Authorization", "Basic " + new String(credentials)) + .PUT(BodyPublishers.ofInputStream(streamSupplier)).build(); + + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } else { + String fileURL = blobEndpoint.split("content_blobs")[0]; + return fileURL; + } + } + + public boolean endPointExists(String endpoint) + throws URISyntaxException, IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + return response.statusCode() == 200; + } + + /** + * Creates an optional asset for a file from an openBIS dataset. Folders are ignored. + * @param file + * @param assetType + * @param assays + * @return + * @throws IOException + * @throws URISyntaxException + * @throws InterruptedException + */ + public Optional createAssetForFile(DataSetFile file, String assetType, + List assays) + throws IOException, URISyntaxException, InterruptedException { + if(!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode+": "+f.getName(); + ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + Arrays.asList("1")); + isaFile.withAssays(assays); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); + String annotation = translator.dataFormatAnnotationForExtension(fileExtension); + if(annotation!=null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + return Optional.of(createDataFileAsset(datasetCode, isaFile)); + } + return Optional.empty(); + } + + public List createAssets(List filesInDataset, String assetType, + List assays) + throws IOException, URISyntaxException, InterruptedException { + List result = new ArrayList<>(); + for(DataSetFile file : filesInDataset) { + if(!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode+": "+f.getName(); + ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + Arrays.asList("1")); + isaFile.withAssays(assays); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); + String annotation = translator.dataFormatAnnotationForExtension(fileExtension); + if(annotation!=null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + result.add(createDataFileAsset(datasetCode, isaFile)); + } + } + return result; + } + + public String listAssays() throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return response.body(); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private String searchNodeWithTitle(String nodeType, String title) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/search/"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("q", title).setParameter("type", "nodeType"); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if(hit.get("title").asText().equals(title)) { + return hit.get("id").asText(); + } + } + throw new RuntimeException("Matching "+nodeType+" title was not found : " + title); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + /** + * Searches for assays containing a search term and returns a list of found assay ids + * @param searchTerm the search term that should be in the assay properties - e.g. an openBIS id + * @return + * @throws URISyntaxException + * @throws IOException + * @throws InterruptedException + */ + public List searchAssaysContainingKeyword(String searchTerm) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/search/"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("q", searchTerm).setParameter("type", "assays"); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + List assayIDs = new ArrayList<>(); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assayIDs.add(hit.get("id").asText()); + } + return assayIDs; + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { + //updateAssay(nodeWithChildren.getAssay()); + return assayID; + } + + public List createNode(SeekStructure nodeWithChildren, boolean transferData) + throws URISyntaxException, IOException, InterruptedException { + String assayID = addAssay(nodeWithChildren.getAssay()); + for(ISASample sample : nodeWithChildren.getSamples()) { + createSample(sample); + } + + List assets = new ArrayList<>(); + + Map isaToFileMap = nodeWithChildren.getIsaToOpenBISFiles(); + for(ISADataFile isaFile : isaToFileMap.keySet()) { + String assetType = nodeWithChildren.getAssetType(isaFile); + createAssetForFile(isaToFileMap.get(isaFile), assetType, Arrays.asList(assayID)) + .ifPresent(assets::add); + } + + return assets; + } + + public static class AssetToUpload { + + private final String blobEndpoint; + private final String filePath; + private final String openBISDataSetCode; + + public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode) { + this.blobEndpoint = blobEndpoint; + this.filePath = filePath; + this.openBISDataSetCode = openBISDataSetCode; + } + + public String getFilePath() { + return filePath; + } + + public String getBlobEndpoint() { + return blobEndpoint; + } + + public String getDataSetCode() { + return openBISDataSetCode; + } + } +} diff --git a/src/main/java/life/qbic/model/isa/AbstractISAObject.java b/src/main/java/life/qbic/model/isa/AbstractISAObject.java new file mode 100644 index 0000000..2d68900 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/AbstractISAObject.java @@ -0,0 +1,22 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.module.SimpleModule; + +public abstract class AbstractISAObject { + + public String toJson(SimpleModule module) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(module); + ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter(); + String json = ow.writeValueAsString(this); + StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{\"data\":"); + jsonBuilder.append(json); + jsonBuilder.append("}"); + return jsonBuilder.toString(); + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java new file mode 100644 index 0000000..188d544 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -0,0 +1,257 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Assays. Contains all mandatory and some optional properties and attributes + * that are needed to create assays in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISAAssay extends AbstractISAObject { + + private final String ISA_TYPE = "assays"; + + private Attributes attributes; + private Relationships relationships; + + public ISAAssay(String title, String studyId, String assayClass, URI assayType) { + this.attributes = new Attributes(title, assayClass, assayType); + this.relationships = new Relationships(studyId); + } + + public ISAAssay withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public ISAAssay withTags(List tags) { + this.attributes.tags = tags; + return this; + } + + public ISAAssay withDescription(String description) { + this.attributes.description = description; + return this; + } + + public ISAAssay withTechnologyType(String technologyType) { + this.attributes.technologyType = technologyType; + return this; + } + + public void setCreatorIDs(List creators) { + this.relationships.setCreatorIDs(creators); + } + public void setOrganismIDs(List organismIDs) { + this.relationships.setOrganismIDs(organismIDs); + } + public void setSampleIDs(List sampleIDs) { + this.relationships.setSampleIDs(sampleIDs); + } + public void setDataFileIDs(List dataFileIDs) { + this.relationships.setDataFileIDs(dataFileIDs); + } + public void setSOPIDs(List sopiDs) { + this.relationships.setSOPIDs(sopiDs); + } + public void setDocumentIDs(List documentIDs) { + this.relationships.setDocumentIDs(documentIDs); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException, URISyntaxException { + ISAAssay assay = new ISAAssay("title", "1", + "EXP", + new URI("http://jermontology.org/ontology/JERMOntology#RNA-Seq")); + assay.setCreatorIDs(Arrays.asList(3,2)); + assay.setOrganismIDs(Arrays.asList(123,3332)); + System.err.println(assay.toJson()); + } + + private class Relationships { + + private String studyId; + private List creators = new ArrayList<>(); + private List samples = new ArrayList<>(); + private List documents = new ArrayList<>(); + private List dataFiles = new ArrayList<>(); + private List sops = new ArrayList<>(); + private List organisms = new ArrayList<>(); + + public Relationships(String studyId) { + this.studyId = studyId; + } + + public String getStudyId() { + return studyId; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + public void setSampleIDs(List samples) { + this.samples = samples; + } + public void setDocumentIDs(List documents) { + this.documents = documents; + } + public void setDataFileIDs(List dataFiles) { + this.dataFiles = dataFiles; + } + public void setSOPIDs(List sops) { + this.sops = sops; + } + public void setOrganismIDs(List organisms) { + this.organisms = organisms; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("study"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.getStudyId()); + jsonGenerator.writeStringField("type", "studies"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + generateListJSON(jsonGenerator, "creators", relationships.getCreators(), "people"); + generateListJSON(jsonGenerator, "samples", relationships.samples, "samples"); + generateListJSON(jsonGenerator, "documents", relationships.documents, "documents"); + generateListJSON(jsonGenerator, "data_files", relationships.dataFiles, "data_files"); + generateListJSON(jsonGenerator, "sops", relationships.sops, "sops"); + generateListJSON(jsonGenerator, "organisms", relationships.organisms, "organisms"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for(int item : items) { + generator.writeStartObject(); + generator.writeStringField("id", Integer.toString(item)); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + public List tags = new ArrayList<>(); + public String description = ""; + public String technologyType = ""; + private String title; + private AssayClass assayClass; + private AssayType assayType; + private String otherCreators = ""; + + public Attributes(String title, String assayClass, URI assayType) { + this.title = title; + this.assayClass = new AssayClass(assayClass); + this.assayType = new AssayType(assayType.toString()); + } + + public List getTags() { + return tags; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getTechnologyType() { + return technologyType; + } + + public AssayClass getAssay_class() { + return assayClass; + } + + public AssayType getAssay_type() { + return assayType; + } + + public String getOther_creators() { + return otherCreators; + } + + private class AssayClass { + + String key; + + public AssayClass(String assayClass) { + this.key = assayClass; + } + + public String getKey() { + return key; + } + } + + private class AssayType { + + String uri; + + public AssayType(String assayType) { + this.uri = assayType; + } + public String getUri() { + return uri; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISADataFile.java b/src/main/java/life/qbic/model/isa/ISADataFile.java new file mode 100644 index 0000000..f2fc346 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISADataFile.java @@ -0,0 +1,203 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Data files. Contains all mandatory and some optional properties and attributes + * that are needed to create a data file in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISADataFile extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "data_files"; + + public ISADataFile(String title, String fileName, List projectIds) { + this.attributes = new Attributes(title, fileName); + this.relationships = new Relationships(projectIds); + } + + public ISADataFile withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public ISADataFile withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public ISADataFile withDataFormatAnnotations(List identifiers) { + this.attributes.withDataFormatAnnotations(identifiers); + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISADataFile sample = new ISADataFile("my file", "myfile.pdf", Arrays.asList("1")); + System.err.println(sample.toJson()); + } + + public String getFileName() { + return attributes.getContent_blobs().get(0).getOriginal_filename(); + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List contentBlobs = new ArrayList<>(); + private String otherCreators = ""; + private List dataFormatAnnotations = new ArrayList<>(); + + + public Attributes(String title, String fileName) { + this.title = title; + this.contentBlobs.add(new ContentBlob(fileName)); + } + + public String getTitle() { + return title; + } + + public List getContent_blobs() { + return contentBlobs; + } + + public String getOther_creators() { + return otherCreators; + } + + public List getData_format_annotations() { + return dataFormatAnnotations; + } + + public void withDataFormatAnnotations(List identifiers) { + List annotations = new ArrayList<>(); + for(String id : identifiers) { + annotations.add(new DataFormatAnnotation(id)); + } + this.dataFormatAnnotations = annotations; + } + + private class DataFormatAnnotation { + + private String label; + private String identifier; + + public DataFormatAnnotation(String identifier) { + this.identifier = identifier; + } + + public String getIdentifier() { + return identifier; + } + + } + + private class ContentBlob { + + private String originalFilename; + private String contentType; + + public ContentBlob(String fileName) { + this.originalFilename = fileName; + String suffix = fileName.substring(fileName.indexOf('.') + 1); + + this.contentType = "application/" + suffix; + } + + public String getContent_type() { + return contentType; + } + + public String getOriginal_filename() { + return originalFilename; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java new file mode 100644 index 0000000..5dce2de --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -0,0 +1,168 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Model class for ISA Samples. Contains all mandatory and some optional properties and attributes + * that are needed to create samples in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISASample extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "samples"; + + public ISASample(Map attributeMap, String sampleType, List projectIds) { + this.attributes = new Attributes(attributeMap); + this.relationships = new Relationships(sampleType, projectIds); + } + + public ISASample withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public void setCreatorIDs(List creatorIDs) { + this.relationships.setCreatorIDs(creatorIDs); + } + + public void setAssayIDs(List assayIDs) { + this.relationships.setAssayIDs(assayIDs); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISASample sample = new ISASample(new HashMap<>(), "1", Arrays.asList("1")); + sample.setCreatorIDs(Arrays.asList("3","2")); + System.err.println(sample.toJson()); + } + + private class Relationships { + + private String sampleType; + private List projects; + private List creators = new ArrayList<>(); + private List assays = new ArrayList<>(); + + public Relationships(String sampleType, List projects) { + this.projects = projects; + this.sampleType = sampleType; + } + + public String getSample_type() { + return sampleType; + } + + public List getAssays() { + return assays; + } + + public void setAssayIDs(List assays) { + this.assays = assays; + } + + public List getProjects() { + return projects; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("sample_type"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.sampleType); + jsonGenerator.writeStringField("type", "sample_types"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + + generateListJSON(jsonGenerator, "creators", relationships.getCreators(), "people"); + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for(String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private Map attributeMap = new HashMap<>(); + private String otherCreators = ""; + + public Attributes(Map attributeMap) { + this.attributeMap = attributeMap; + } + + public Map getAttribute_map() { + return attributeMap; + } + + public String getOther_creators() { + return otherCreators; + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAStudy.java b/src/main/java/life/qbic/model/isa/ISAStudy.java new file mode 100644 index 0000000..1fd19bf --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISAStudy.java @@ -0,0 +1,158 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for ISA Studies. Contains all mandatory and some optional properties and attributes + * that are needed to create studies in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + */ +public class ISAStudy extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "studies"; + + public ISAStudy(String title, String investigationId) { + this.attributes = new Attributes(title); + this.relationships = new Relationships(investigationId); + } + + public ISAStudy withDescription(String description) { + this.attributes.description = description; + return this; + } + + public ISAStudy withExperimentalists(String experimentalists) { + this.attributes.experimentalists = experimentalists; + return this; + } + + public ISAStudy withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public void setCreatorIDs(List creators) { + this.relationships.setCreatorIDs(creators); + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public static void main(String[] args) throws JsonProcessingException { + ISAStudy study = new ISAStudy("title", "1"); + study.setCreatorIDs(Arrays.asList(3,2)); + System.err.println(study.toJson()); + } + + private class Relationships { + + private String investigationId; + private List creators = new ArrayList<>(); + + public Relationships(String investigationId) { + this.investigationId = investigationId; + } + + public String getInvestigationId() { + return investigationId; + } + + public List getCreators() { + return creators; + } + + public void setCreatorIDs(List creators) { + this.creators = creators; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + jsonGenerator.writeObjectFieldStart("investigation"); + jsonGenerator.writeObjectFieldStart("data"); + jsonGenerator.writeStringField("id", relationships.getInvestigationId()); + jsonGenerator.writeStringField("type", "investigations"); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + jsonGenerator.writeObjectFieldStart("creators"); + jsonGenerator.writeArrayFieldStart("data"); + for(int personID : relationships.getCreators()) { + jsonGenerator.writeStartObject(); + jsonGenerator.writeNumberField("id", personID); + jsonGenerator.writeStringField("type", "people"); + jsonGenerator.writeEndObject(); + } + jsonGenerator.writeEndArray(); + jsonGenerator.writeEndObject(); + jsonGenerator.writeEndObject(); + } + } + + private class Attributes { + + private String title; + private String description = ""; + private String experimentalists = ""; + private String otherCreators = ""; + + public Attributes(String title) { + this.title = title; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getExperimentalists() { + return experimentalists; + } + + public String getOther_creators() { + return otherCreators; + } + } + +} From c41c6aa529f36351cda04a2471ee6b41f451c3f7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 7 Oct 2024 11:07:51 +0200 Subject: [PATCH 28/63] structure --- .../qbic/model/{ => isa}/SeekStructure.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) rename src/main/java/life/qbic/model/{ => isa}/SeekStructure.java (51%) diff --git a/src/main/java/life/qbic/model/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java similarity index 51% rename from src/main/java/life/qbic/model/SeekStructure.java rename to src/main/java/life/qbic/model/isa/SeekStructure.java index 5d7cc46..323876c 100644 --- a/src/main/java/life/qbic/model/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -3,18 +3,22 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.util.List; import java.util.Map; +import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; public class SeekStructure { private ISAAssay assay; private List samples; - private List dataFiles; - private Map isaToOpenBISFile; - private Map isaFileToAssetType; - //ISASample isaSample = new ISASample(map, sampleTypeID, Arrays.asList(projectID)); + private Map isaToOpenBISFile; + + public SeekStructure(ISAAssay assay, List samples, + Map isaToOpenBISFile) { + this.assay = assay; + this.samples = samples; + this.isaToOpenBISFile = isaToOpenBISFile; + } public ISAAssay getAssay() { return assay; @@ -24,11 +28,8 @@ public List getSamples() { return samples; } - public Map getIsaToOpenBISFiles() { + public Map getISAFileToDatasetFiles() { return isaToOpenBISFile; } - public String getAssetType(ISADataFile dataFile) { - return isaFileToAssetType.get(dataFile); - } } From 3ec62fd4b7068cd7c58ac3c9cec8a4da6650b11c Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 7 Oct 2024 12:50:54 +0200 Subject: [PATCH 29/63] implement first seek use cases --- .../io/commandline/CommandLineOptions.java | 4 +- .../TransferDataToSeekCommand.java | 103 ++++---- .../TransferSampleTypesToSeekCommand.java | 61 +++++ .../qbic/model/AssayWithQueuedAssets.java | 23 ++ .../OpenbisExperimentWithDescendants.java | 4 +- .../qbic/model/OpenbisSeekTranslator.java | 175 ++++++++++---- .../qbic/model/SampleTypesAndMaterials.java | 23 ++ .../qbic/model/download/OpenbisConnector.java | 40 +++- .../qbic/model/download/SEEKConnector.java | 155 +++++++++--- .../life/qbic/model/isa/GenericSeekAsset.java | 199 ++++++++++++++++ .../java/life/qbic/model/isa/ISAAssay.java | 9 - .../java/life/qbic/model/isa/ISADataFile.java | 5 - .../java/life/qbic/model/isa/ISASample.java | 32 +-- .../life/qbic/model/isa/ISASampleType.java | 220 ++++++++++++++++++ .../java/life/qbic/model/isa/ISAStudy.java | 6 - .../life/qbic/model/isa/SeekStructure.java | 5 +- 16 files changed, 899 insertions(+), 165 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java create mode 100644 src/main/java/life/qbic/model/AssayWithQueuedAssets.java create mode 100644 src/main/java/life/qbic/model/SampleTypesAndMaterials.java create mode 100644 src/main/java/life/qbic/model/isa/GenericSeekAsset.java create mode 100644 src/main/java/life/qbic/model/isa/ISASampleType.java diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index a0dcbad..b583d9c 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -8,9 +8,9 @@ // main command with format specifiers for the usage help message @Command(name = "openbis-scripts", - subcommands = {SampleHierarchyCommand.class, FindDatasetsCommand.class, + subcommands = {SampleHierarchyCommand.class, TransferSampleTypesToSeekCommand.class, DownloadPetabCommand.class, UploadPetabResultCommand.class, UploadDatasetCommand.class, - SpaceStatisticsCommand.class, TransferDataToSeekCommand.class}, + SpaceStatisticsCommand.class, TransferDataToSeekCommand.class, FindDatasetsCommand.class}, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index ec9bdaf..72d8934 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -9,15 +9,19 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import life.qbic.App; +import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.OpenbisSeekTranslator; -import life.qbic.model.SeekStructure; +import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import life.qbic.model.download.SEEKConnector.AssetToUpload; @@ -65,21 +69,21 @@ public class TransferDataToSeekCommand implements Runnable { AuthenticationOptions auth = new AuthenticationOptions(); OpenbisConnector openbis; SEEKConnector seek; - OpenbisSeekTranslator translator = new OpenbisSeekTranslator(); + OpenbisSeekTranslator translator; @Override public void run() { - System.out.println("auth..."); + System.out.println("Connecting to openBIS..."); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); - System.out.println("openbis..."); openbis = new OpenbisConnector(authentication); boolean isSample = false; boolean isDataSet = false; - System.out.println("search for experiment..."); + + System.out.println("Searching for specified object in openBIS..."); boolean isExperiment = experimentExists(objectID); if (!isExperiment && sampleExists(objectID)) { @@ -96,32 +100,15 @@ public void run() { objectID); return; } - System.out.println("Searching done..."); -/* - if (isExperiment) { - if (seekNode != null && !seekNode.contains("assays")) { - System.out.printf( - "Seek node %s does not correspond to the provided openBIS experiment identifier. " - + "Please select an assay node.%n", objectID); - return; - } - } - if (isSample) { - if (seekNode != null && !seekNode.contains("samples")) { - System.out.printf( - "Seek node %s does not correspond to the provided openBIS sample identifier. " - + "Please select a sample node.%n", objectID); - return; - } - } - - */ + System.out.println("Search successful."); + System.out.println("Connecting to SEEK..."); byte[] httpCredentials = Base64.encodeBase64( (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "Default Project", + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", "lisym default study"); + translator = seek.getTranslator(); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); } @@ -147,13 +134,22 @@ public void run() { */ try { + System.out.println("Collecting information from openBIS..."); OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); - String assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); - List blacklist = parseBlackList(blacklistFile); - if(assayID.isBlank()) { - createNewNodes(experiment, blacklist); + System.out.println("Trying to find existing corresponding assay in SEEK..."); + Optional assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); + assayID.ifPresent(x -> System.out.println("Found assay with id "+assayID.get())); + Set blacklist = parseBlackList(blacklistFile); + System.out.println("Translating openBIS property codes to SEEK names..."); + Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); + System.out.println("Creating SEEK structure..."); + SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist); + if(assayID.isEmpty()) { + System.out.println("Creating new nodes..."); + createNewNodes(nodeWithChildren); } else { - updateNodes(experiment, assayID, blacklist); + System.out.println("Updating nodes..."); + updateNodes(nodeWithChildren, assayID.get()); } } catch (URISyntaxException | InterruptedException | IOException e) { throw new RuntimeException(e); @@ -187,42 +183,46 @@ public void run() { System.out.println("Done"); } - private List parseBlackList(String blacklistFile) { - List result = new ArrayList<>(); + private Set parseBlackList(String blacklistFile) { if(blacklistFile == null) { - return result; + return new HashSet<>(); } // trim whitespace, skip empty lines try (Stream lines = Files.lines(Paths.get(blacklistFile)) .map(String::trim) .filter(s -> !s.isBlank())) { - return lines.collect(Collectors.toList()); + + Set codes = lines.collect(Collectors.toSet()); + + for(String code : codes) { + if(!OpenbisConnector.datasetCodePattern.matcher(code).matches()) { + throw new RuntimeException("Invalid dataset code: " + code+". Please make sure to use valid" + + " dataset codes in the blacklist file."); + } + } + return codes; } catch (IOException e) { throw new RuntimeException(blacklistFile+" could not be found or read."); } } - private void updateNodes(OpenbisExperimentWithDescendants experiment, String assayID, List blacklist) { - System.err.println("updating nodes of assay "+assayID); - SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); + private void updateNodes(SeekStructure nodeWithChildren, String assayID) { String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); - System.out.printf("%s was successfully updated.%n", endpoint); + System.out.printf("%s was successfully updated.%n", updatedEndpoint); } - private void createNewNodes(OpenbisExperimentWithDescendants experiment, List blacklist) + private void createNewNodes(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - System.err.println("creating new nodes"); - SeekStructure nodeWithChildren = translator.translate(experiment, blacklist); - List assetsToUpload = seek.createNode(nodeWithChildren, transferData); + AssayWithQueuedAssets assetsOfAssayToUpload = seek.createNode(nodeWithChildren, transferData); if(transferData) { - for(AssetToUpload asset : assetsToUpload) { + for(AssetToUpload asset : assetsOfAssayToUpload.getAssets()) { System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); System.out.printf("File stored here: %s%n", fileURL); } } - System.out.printf("%s was successfully created.%n", endpoint); + System.out.printf("%s was successfully created.%n", assetsOfAssayToUpload.getAssayEndpoint()); } private boolean sampleExists(String objectID) { @@ -237,7 +237,7 @@ private boolean experimentExists(String experimentID) { return openbis.experimentExists(experimentID); } - private String getAssayIDForOpenBISExperiment(Experiment experiment) + private Optional getAssayIDForOpenBISExperiment(Experiment experiment) throws URISyntaxException, IOException, InterruptedException { // the perm id is unique and afaik not used by scientists. it is highly unlikely that it would // "accidentally" be part of another title or description. however, care should be taken here, @@ -246,12 +246,12 @@ private String getAssayIDForOpenBISExperiment(Experiment experiment) String permID = experiment.getPermId().getPermId(); List assayIDs = seek.searchAssaysContainingKeyword(permID); if(assayIDs.isEmpty()) { - return ""; + return Optional.empty(); } if(assayIDs.size() == 1) { - return assayIDs.get(0); + return Optional.of(assayIDs.get(0)); } - throw new RuntimeException("Search term "+permID+ " was found in more than one assay: "+assayIDs); + throw new RuntimeException("Experiment identifier "+permID+ " was found in more than one assay: "+assayIDs); } private void sendDatasetToSeek(String datasetCode, String assayID) @@ -265,8 +265,7 @@ private void sendDatasetToSeek(String datasetCode, String assayID) } DataSet dataset = datasets.get(0); List files = openbis.getDatasetFiles(dataset); - List assets = seek.createAssets(files, dataset.getType().getCode(), - Arrays.asList(assayID)); + List assets = seek.createAssets(files, Arrays.asList(assayID)); for(AssetToUpload asset : assets) { System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java new file mode 100644 index 0000000..cdd6ee4 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -0,0 +1,61 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.io.IOException; +import java.net.URISyntaxException; +import life.qbic.App; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SampleTypesAndMaterials; +import life.qbic.model.download.OpenbisConnector; +import life.qbic.model.download.SEEKConnector; +import org.apache.commons.codec.binary.Base64; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +@Command(name = "sample-type-transfer", + description = + "Transfers sample types from openBIS to SEEK.") +public class TransferSampleTypesToSeekCommand implements Runnable { + @Mixin + AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisConnector openbis; + SEEKConnector seek; + OpenbisSeekTranslator translator; + + @Override + public void run() { + System.out.println("auth..."); + + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getAS(), auth.getDSS()); + System.out.println("openbis..."); + + openbis = new OpenbisConnector(authentication); + + byte[] httpCredentials = Base64.encodeBase64( + (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + try { + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", + "lisym default study"); + translator = seek.getTranslator(); + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + SampleTypesAndMaterials types = openbis.getSampleTypesWithMaterials(); + + try { + for(SampleType type : types.getSampleTypes()) { + System.err.println("creating "+type.getCode()); + String sampleTypeId = seek.createSampleType(translator.translate(type)); + System.err.println("created "+sampleTypeId); + } + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + System.out.println("Done"); + } + +} diff --git a/src/main/java/life/qbic/model/AssayWithQueuedAssets.java b/src/main/java/life/qbic/model/AssayWithQueuedAssets.java new file mode 100644 index 0000000..0186cfc --- /dev/null +++ b/src/main/java/life/qbic/model/AssayWithQueuedAssets.java @@ -0,0 +1,23 @@ +package life.qbic.model; + +import java.util.List; +import life.qbic.model.download.SEEKConnector.AssetToUpload; + +public class AssayWithQueuedAssets { + + private String assayEndpoint; + private List assetsToUpload; + + public AssayWithQueuedAssets(String assayEndpoint, List assetsToUpload) { + this.assayEndpoint = assayEndpoint; + this.assetsToUpload = assetsToUpload; + } + + public String getAssayEndpoint() { + return assayEndpoint; + } + + public List getAssets() { + return assetsToUpload; + } +} diff --git a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java index 86399ec..0782d55 100644 --- a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java +++ b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java @@ -12,10 +12,10 @@ public class OpenbisExperimentWithDescendants { private Experiment experiment; private List samples; private List datasets; - private Map> datasetCodeToFiles; + private Map> datasetCodeToFiles; public OpenbisExperimentWithDescendants(Experiment experiment, List samples, - List datasets, Map> datasetCodeToFiles) { + List datasets, Map> datasetCodeToFiles) { this.experiment = experiment; this.samples = samples; this.datasets = datasets; diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index a02317d..0504636 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -4,20 +4,36 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.DataType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import life.qbic.model.isa.GenericSeekAsset; +import life.qbic.model.isa.ISAAssay; import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISASampleType; +import life.qbic.model.isa.ISASampleType.SampleAttribute; +import life.qbic.model.isa.ISASampleType.SampleAttributeType; +import life.qbic.model.isa.SeekStructure; public class OpenbisSeekTranslator { private final String DEFAULT_PROJECT_ID; private final String DEFAULT_STUDY_ID; + private final String DEFAULT_TRANSFERRED_SAMPLE_TITLE = "openBIS Name"; public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { this.DEFAULT_PROJECT_ID = defaultProjectID; @@ -34,9 +50,27 @@ public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { entry("04_MICRO_CT", "EXP") ); + Map dataTypeToAttributeType = Map.ofEntries( + entry(DataType.INTEGER, new SampleAttributeType("4", "Integer", "Integer")), + entry(DataType.VARCHAR, new SampleAttributeType("8", "String", "String")), + entry(DataType.MULTILINE_VARCHAR, new SampleAttributeType("7", "Text", "Text")), + entry(DataType.REAL, new SampleAttributeType("3", "Real number", "Float")), + entry(DataType.TIMESTAMP, new SampleAttributeType("1", "Date time", "DateTime")), + entry(DataType.BOOLEAN, new SampleAttributeType("16", "Boolean", "Boolean")), + entry(DataType.CONTROLLEDVOCABULARY, //we use String for now + new SampleAttributeType("8", "String", "String")), + entry(DataType.MATERIAL, //not used anymore in this form + new SampleAttributeType("8", "String", "String")), + entry(DataType.HYPERLINK, new SampleAttributeType("8", "String", "String")), + entry(DataType.XML, new SampleAttributeType("7", "Text", "Text")), + entry(DataType.SAMPLE, //should be handled before mapping types + new SampleAttributeType("8", "String", "String")), + entry(DataType.DATE, new SampleAttributeType("2", "Date time", "Date")) + ); + Map experimentTypeToAssayType = Map.ofEntries( entry("00_MOUSE_DATABASE", ""), - entry("00_PATIENT_DATABASE", ""),//if this is related to measured data, attach it as sample to the assay + entry("00_PATIENT_DATABASE", ""), entry("00_STANDARD_OPERATING_PROTOCOLS", ""), entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), @@ -55,67 +89,128 @@ public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { ); Map datasetTypeToAssetType = Map.ofEntries( - entry("ANALYSIS_NOTEBOOK", "Document"), - entry("ANALYZED_DATA", "Data_file"), - entry("ATTACHMENT", "Document"), + entry("ANALYSIS_NOTEBOOK", "documents"), + entry("ANALYZED_DATA", "data_files"), + entry("ATTACHMENT", "documents"), entry("ELN_PREVIEW", ""), - entry("EXPERIMENT_PROTOCOL", "SOP"), - entry("EXPERIMENT_RESULT", "Document"), - entry("HISTOLOGICAL_SLIDE", "Data_file"), - entry("IB_DATA", "Data_file"), - entry("LUMINEX_DATA", "Data_file"), - entry("MS_DATA_ANALYZED", "Data_file"), - entry("MS_DATA_RAW", "Data_file"), - entry("OTHER_DATA", "Document"), - entry("PROCESSED_DATA", "Data_file"), - entry("PUBLICATION_DATA", "Publication"), - entry("QPCR_DATA", "Data_file"), - entry("RAW_DATA", "Data_file"), - entry("SOURCE_CODE", "Document"), + entry("EXPERIMENT_PROTOCOL", "sops"), + entry("EXPERIMENT_RESULT", "documents"), + entry("HISTOLOGICAL_SLIDE", "data_files"), + entry("IB_DATA", "data_files"), + entry("LUMINEX_DATA", "data_files"), + entry("MS_DATA_ANALYZED", "data_files"), + entry("MS_DATA_RAW", "data_files"), + entry("OTHER_DATA", "data_files"), + entry("PROCESSED_DATA", "data_files"), + entry("PUBLICATION_DATA", "publications"), + entry("QPCR_DATA", "data_files"), + entry("RAW_DATA", "data_files"), + entry("SOURCE_CODE", "documents"), entry("TEST_CONT", ""), entry("TEST_DAT", ""), - entry("UNKNOWN", "") + entry("UNKNOWN", "data_files") ); - public SeekStructure translate(String seekNode, Experiment experimentWithSamplesAndDatasets) { - Experiment exp = experimentWithSamplesAndDatasets; - exp.getType(); - return null; - //new ISAAssay(exp.getCode(), ) - } - - public ISADataFile translate(String seekNode, DataSet dataset) { - return null; //new ISADataFile(); + public ISASampleType translate(SampleType sampleType) { + SampleAttribute titleAttribute = new SampleAttribute(DEFAULT_TRANSFERRED_SAMPLE_TITLE, + dataTypeToAttributeType.get(DataType.VARCHAR), true, false); + ISASampleType type = new ISASampleType(sampleType.getCode(), titleAttribute, + DEFAULT_PROJECT_ID); + for (PropertyAssignment a : sampleType.getPropertyAssignments()) { + DataType dataType = a.getPropertyType().getDataType(); + type.addSampleAttribute(a.getPropertyType().getLabel(), dataTypeToAttributeType.get(dataType), + false, null); + } + return type; } public String assetForDatasetType(String datasetType) { - return "Data_file";//TODO + if(datasetTypeToAssetType.get(datasetType) == null || datasetTypeToAssetType.get(datasetType).isBlank()) { + throw new RuntimeException("Dataset type " + datasetType + " could not be mapped to SEEK type."); + } + return datasetTypeToAssetType.get(datasetType); } public String dataFormatAnnotationForExtension(String fileExtension) { return fileExtensionToDataFormat.get(fileExtension); } - public SeekStructure translate(Experiment experiment, List blacklist) { - System.err.println(experiment.getCode()); + public SeekStructure translate(OpenbisExperimentWithDescendants experiment, + Map sampleTypesToIds, Set blacklist) throws URISyntaxException { + + Experiment exp = experiment.getExperiment(); + String expType = exp.getType().getCode(); + String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; + ISAAssay assay = new ISAAssay(title, DEFAULT_STUDY_ID, experimentTypeToAssayClass.get(expType), + new URI(experimentTypeToAssayType.get(expType)));//TODO + List samples = new ArrayList<>(); for(Sample sample : experiment.getSamples()) { - String sampleType = getSampleType(sample.getType()); + SampleType sampleType = sample.getType(); + + //try to put all attributes into sample properties, as they should be a 1:1 mapping + Map typeCodesToNames = new HashMap<>(); + for (PropertyAssignment a : sampleType.getPropertyAssignments()) { + typeCodesToNames.put(a.getPropertyType().getCode(), a.getPropertyType().getLabel()); + } Map attributes = new HashMap<>(); - ISASample isaSample = new ISASample(attributes, sampleType, + for(String code : sample.getProperties().keySet()) { + attributes.put(typeCodesToNames.get(code), sample.getProperties().get(code)); + } + + attributes.put(DEFAULT_TRANSFERRED_SAMPLE_TITLE, sample.getIdentifier().getIdentifier()); + String sampleTypeId = sampleTypesToIds.get(sampleType.getCode()); + ISASample isaSample = new ISASample(sample.getPermId().getPermId(), attributes, sampleTypeId, Collections.singletonList(DEFAULT_PROJECT_ID)); - System.err.println(sample.getCode()); + samples.add(isaSample); } - for(DataSet dataset : experiment.getDataSets()) { - System.err.println(dataset.getCode()); + Map isaToOpenBISFile = new HashMap<>(); + + //create ISA files for assets. If actual data is uploaded is determined later based upon flag + for(DatasetWithProperties dataset : experiment.getDatasets()) { + String permID = dataset.getCode(); + if(!blacklist.contains(permID)) { + for(DataSetFile file : experiment.getFilesForDataset(permID)) { + String datasetType = getDatasetTypeOfFile(file, experiment.getDatasets()); + datasetFileToSeekAsset(file, datasetType) + .ifPresent(seekAsset -> isaToOpenBISFile.put(seekAsset, file)); + } + } } - System.err.println("blacklisted:"); - for(String dataset : blacklist) { - System.err.println(dataset); + return new SeekStructure(assay, samples, isaToOpenBISFile); + } + + private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { + String permId = file.getDataSetPermId().getPermId(); + for(DatasetWithProperties dataset : dataSets) { + if(dataset.getCode().equals(permId)) { + return dataset.getType().getCode(); + } } - return null;//TODO + return ""; } - private String getSampleType(SampleType type) { + /** + * Creates a SEEK asset from an openBIS DataSetFile, if it describes a file (not a folder). + * @param file the openBIS DataSetFile + * @return an optional SEEK asset + */ + private Optional datasetFileToSeekAsset(DataSetFile file, String datasetType) { + if (!file.getPath().isBlank() && !file.isDirectory()) { + File f = new File(file.getPath()); + String datasetCode = file.getDataSetPermId().toString(); + String assetName = datasetCode + ": " + f.getName(); + String assetType = assetForDatasetType(datasetType); + GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), + Arrays.asList(DEFAULT_PROJECT_ID)); + String fileExtension = f.getName().substring(f.getName().lastIndexOf(".") + 1); + String annotation = dataFormatAnnotationForExtension(fileExtension); + if (annotation != null) { + isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); + } + return Optional.of(isaFile); + } + return Optional.empty(); } + } diff --git a/src/main/java/life/qbic/model/SampleTypesAndMaterials.java b/src/main/java/life/qbic/model/SampleTypesAndMaterials.java new file mode 100644 index 0000000..8a82576 --- /dev/null +++ b/src/main/java/life/qbic/model/SampleTypesAndMaterials.java @@ -0,0 +1,23 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import java.util.Set; + +public class SampleTypesAndMaterials { + + Set sampleTypes; + Set sampleTypesAsMaterials; + + public SampleTypesAndMaterials(Set sampleTypes, Set sampleTypesAsMaterials) { + this.sampleTypes = sampleTypes; + this.sampleTypesAsMaterials = sampleTypesAsMaterials; + } + + public Set getSamplesAsMaterials() { + return sampleTypesAsMaterials; + } + + public Set getSampleTypes() { + return sampleTypes; + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index e50dd12..7e5d7ed 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -12,11 +12,15 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleTypeFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleTypeSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; @@ -37,14 +41,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; +import life.qbic.App; import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; +import life.qbic.model.SampleTypesAndMaterials; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -428,15 +436,16 @@ public OpenbisExperimentWithDescendants getExperimentWithDescendants(String expe dataSetFetchOptions.withRegistrator(); SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); sampleFetchOptions.withProperties(); + sampleFetchOptions.withType().withPropertyAssignments().withPropertyType(); sampleFetchOptions.withDataSetsUsing(dataSetFetchOptions); fetchOptions.withDataSetsUsing(dataSetFetchOptions); fetchOptions.withSamplesUsing(sampleFetchOptions); Experiment experiment = openBIS.searchExperiments(criteria, fetchOptions).getObjects().get(0); - Map> datasetCodeToFiles = new HashMap<>(); + Map> datasetCodeToFiles = new HashMap<>(); for(DataSet dataset : experiment.getDataSets()) { - datasetCodeToFiles.put(dataset.getPermId(), getDatasetFiles(dataset)); + datasetCodeToFiles.put(dataset.getPermId().getPermId(), getDatasetFiles(dataset)); } return new OpenbisExperimentWithDescendants(experiment, experiment.getSamples(), @@ -455,4 +464,31 @@ public List getDatasetFiles(DataSet dataset) { return result.getObjects(); } + + public SampleTypesAndMaterials getSampleTypesWithMaterials() { + SampleTypeSearchCriteria criteria = new SampleTypeSearchCriteria(); + SampleTypeFetchOptions typeOptions = new SampleTypeFetchOptions(); + typeOptions.withPropertyAssignments().withPropertyType(); + typeOptions.withPropertyAssignments().withEntityType(); + Set sampleTypes = new HashSet<>(); + Set sampleTypesAsMaterials = new HashSet<>(); + for(SampleType type : openBIS.searchSampleTypes(criteria, typeOptions).getObjects()) { + /* + System.err.println("sample type: "+type.getCode()); + for(PropertyAssignment assignment : type.getPropertyAssignments()) { + if (assignment.getPropertyType().getDataType().name().equals("SAMPLE")) { + System.err.println(assignment.getPropertyType().getLabel()); + System.err.println(assignment.getPropertyType().getDataType().name()); + System.err.println(assignment.getPropertyType().getCode()); + } + } + */ + if(type.getCode().startsWith("MATERIAL.")) { + sampleTypesAsMaterials.add(type); + } else { + sampleTypes.add(type); + } + } + return new SampleTypesAndMaterials(sampleTypes, sampleTypesAsMaterials); + } } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 298364e..580bf0f 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -1,6 +1,7 @@ package life.qbic.model.download; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; @@ -21,12 +22,15 @@ import java.util.Map; import java.util.Optional; import java.util.function.Supplier; +import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.OpenbisSeekTranslator; -import life.qbic.model.SeekStructure; +import life.qbic.model.isa.SeekStructure; +import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISAStudy; +import org.apache.commons.codec.binary.Base64; import org.apache.http.client.utils.URIBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -42,8 +46,8 @@ public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjec String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { this.apiURL = apiURL; this.credentials = httpCredentials; - translator = new OpenbisSeekTranslator(searchNodeWithTitle("projects", defaultProjectTitle), - searchNodeWithTitle("studies", defaultStudyTitle)); + translator = new OpenbisSeekTranslator("1", //searchNodeWithTitle("projects", defaultProjectTitle), + "1");//searchNodeWithTitle("studies", defaultStudyTitle)); } public String addAssay(ISAAssay assay) @@ -55,7 +59,6 @@ public String addAssay(ISAAssay assay) BodyHandlers.ofString()); if(response.statusCode()!=200) { - System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } JsonNode rootNode = new ObjectMapper().readTree(response.body()); @@ -103,6 +106,18 @@ public boolean studyExists(String id) throws URISyntaxException, IOException, In return response.statusCode() == 200; } + public void printAttributeTypes() throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/sample_attribute_types"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + System.err.println(response.body()); + } /* -datatype by extension -assay equals experiment @@ -112,18 +127,52 @@ public boolean studyExists(String id) throws URISyntaxException, IOException, In -flexible object type to sample type? */ + public void deleteSampleType(String id) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/sample_types"; + URIBuilder builder = new URIBuilder(endpoint); + builder.setParameter("id", id); + + HttpResponse response = HttpClient.newBuilder().build() + .send(HttpRequest.newBuilder().uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .DELETE().build(), BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + public String createSampleType(ISASampleType sampleType) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/sample_types"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, sampleType.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=201) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/samples"; - HashMap map = new HashMap<>(); - //map.put("title", "really?"); - map.put("test_attribute", "okay then"); HttpResponse response = HttpClient.newBuilder().build() .send(buildAuthorizedPOSTRequest(endpoint, isaSample.toJson()), BodyHandlers.ofString()); - if(response.statusCode()!=201) { + if(response.statusCode()!=200) { System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } @@ -133,9 +182,9 @@ public String createSample(ISASample isaSample) throws URISyntaxException, IOExc return idNode.asText(); } - private AssetToUpload createDataFileAsset(String datasetCode, ISADataFile data) + private AssetToUpload createDataFileAsset(String datasetCode, GenericSeekAsset data) throws IOException, URISyntaxException, InterruptedException { - String endpoint = apiURL+"/data_files"; + String endpoint = apiURL+"/"+data.getType(); HttpResponse response = HttpClient.newBuilder().build() .send(buildAuthorizedPOSTRequest(endpoint, data.toJson()), @@ -228,8 +277,8 @@ public Optional createAssetForFile(DataSetFile file, String asset File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); String assetName = datasetCode+": "+f.getName(); - ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), - Arrays.asList("1")); + GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), + Arrays.asList("1"));//TODO isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); String annotation = translator.dataFormatAnnotationForExtension(fileExtension); @@ -241,7 +290,7 @@ public Optional createAssetForFile(DataSetFile file, String asset return Optional.empty(); } - public List createAssets(List filesInDataset, String assetType, + public List createAssets(List filesInDataset, List assays) throws IOException, URISyntaxException, InterruptedException { List result = new ArrayList<>(); @@ -249,8 +298,8 @@ public List createAssets(List filesInDataset, String if(!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName(); - ISADataFile isaFile = new ISADataFile(assetName, file.getPath(), + String assetName = datasetCode+": "+f.getName();//TODO? + GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), Arrays.asList("1")); isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); @@ -264,6 +313,27 @@ public List createAssets(List filesInDataset, String return result; } + /** + * Creates + * @param isaToOpenBISFile + * @param assays + * @return + * @throws IOException + * @throws URISyntaxException + * @throws InterruptedException + */ + public List createAssetsForAssays(Map isaToOpenBISFile, + List assays) + throws IOException, URISyntaxException, InterruptedException { + List result = new ArrayList<>(); + for (GenericSeekAsset isaFile : isaToOpenBISFile.keySet()) { + isaFile.withAssays(assays); + result.add(createDataFileAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), + isaFile)); + } + return result; + } + public String listAssays() throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/assays/"; HttpRequest request = HttpRequest.newBuilder() @@ -281,11 +351,42 @@ public String listAssays() throws URISyntaxException, IOException, InterruptedEx } } + public Map getSampleTypeNamesToIDs() + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/sample_types/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return parseSampleTypesJSON(response.body()); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private Map parseSampleTypesJSON(String json) throws JsonProcessingException { + Map typesToIDs = new HashMap<>(); + JsonNode rootNode = new ObjectMapper().readTree(json); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + String id = hit.get("id").asText(); + String title = hit.get("attributes").get("title").asText(); + typesToIDs.put(title, id); + } + return typesToIDs; + } + private String searchNodeWithTitle(String nodeType, String title) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/search/"; URIBuilder builder = new URIBuilder(endpoint); - builder.setParameter("q", title).setParameter("type", "nodeType"); + builder.setParameter("q", title).setParameter("search_type", nodeType); HttpRequest request = HttpRequest.newBuilder() .uri(builder.build()) @@ -295,11 +396,13 @@ private String searchNodeWithTitle(String nodeType, String title) .GET().build(); HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); + System.err.println("searching for: "+title+" ("+nodeType+")"); if(response.statusCode() == 200) { JsonNode rootNode = new ObjectMapper().readTree(response.body()); JsonNode hits = rootNode.path("data"); for (Iterator it = hits.elements(); it.hasNext(); ) { JsonNode hit = it.next(); + System.err.println(hit.asText()); if(hit.get("title").asText().equals(title)) { return hit.get("id").asText(); } @@ -348,26 +451,24 @@ public List searchAssaysContainingKeyword(String searchTerm) public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { //updateAssay(nodeWithChildren.getAssay()); - return assayID; + return apiURL+"/assays/"+assayID; } - public List createNode(SeekStructure nodeWithChildren, boolean transferData) + public AssayWithQueuedAssets createNode(SeekStructure nodeWithChildren, boolean transferData) throws URISyntaxException, IOException, InterruptedException { String assayID = addAssay(nodeWithChildren.getAssay()); for(ISASample sample : nodeWithChildren.getSamples()) { createSample(sample); } - List assets = new ArrayList<>(); + Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); - Map isaToFileMap = nodeWithChildren.getIsaToOpenBISFiles(); - for(ISADataFile isaFile : isaToFileMap.keySet()) { - String assetType = nodeWithChildren.getAssetType(isaFile); - createAssetForFile(isaToFileMap.get(isaFile), assetType, Arrays.asList(assayID)) - .ifPresent(assets::add); - } + return new AssayWithQueuedAssets(apiURL+"/assays/"+assayID, + createAssetsForAssays(isaToFileMap, Arrays.asList(assayID))); + } - return assets; + public OpenbisSeekTranslator getTranslator() { + return translator; } public static class AssetToUpload { diff --git a/src/main/java/life/qbic/model/isa/GenericSeekAsset.java b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java new file mode 100644 index 0000000..34b4fef --- /dev/null +++ b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java @@ -0,0 +1,199 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Model class for Seek assets. Contains all mandatory and some optional properties and attributes + * that are needed to create an asset in SEEK. The model and its getters (names) are structured in a + * way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. Since there are different types of assets, the isaType is a + * parameter here. + */ +public class GenericSeekAsset extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private String assetType; + + public GenericSeekAsset(String assetType, String title, String fileName, List projectIds) { + this.assetType = assetType; + this.attributes = new Attributes(title, fileName); + this.relationships = new Relationships(projectIds); + } + + public GenericSeekAsset withOtherCreators(String otherCreators) { + this.attributes.otherCreators = otherCreators; + return this; + } + + public GenericSeekAsset withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public GenericSeekAsset withDataFormatAnnotations(List identifiers) { + this.attributes.withDataFormatAnnotations(identifiers); + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return assetType; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + public String getFileName() { + return attributes.getContent_blobs().get(0).getOriginal_filename(); + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List contentBlobs = new ArrayList<>(); + private String otherCreators = ""; + private List dataFormatAnnotations = new ArrayList<>(); + + + public Attributes(String title, String fileName) { + this.title = title; + this.contentBlobs.add(new ContentBlob(fileName)); + } + + public String getTitle() { + return title; + } + + public List getContent_blobs() { + return contentBlobs; + } + + public String getOther_creators() { + return otherCreators; + } + + public List getData_format_annotations() { + return dataFormatAnnotations; + } + + public void withDataFormatAnnotations(List identifiers) { + List annotations = new ArrayList<>(); + for(String id : identifiers) { + annotations.add(new DataFormatAnnotation(id)); + } + this.dataFormatAnnotations = annotations; + } + + private class DataFormatAnnotation { + + private String label; + private String identifier; + + public DataFormatAnnotation(String identifier) { + this.identifier = identifier; + } + + public String getIdentifier() { + return identifier; + } + + } + + private class ContentBlob { + + private String originalFilename; + private String contentType; + + public ContentBlob(String fileName) { + this.originalFilename = fileName; + String suffix = fileName.substring(fileName.indexOf('.') + 1); + + this.contentType = "application/" + suffix; + } + + public String getContent_type() { + return contentType; + } + + public String getOriginal_filename() { + return originalFilename; + } + } + } + +} diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java index 188d544..ed2a7c0 100644 --- a/src/main/java/life/qbic/model/isa/ISAAssay.java +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -88,15 +88,6 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException, URISyntaxException { - ISAAssay assay = new ISAAssay("title", "1", - "EXP", - new URI("http://jermontology.org/ontology/JERMOntology#RNA-Seq")); - assay.setCreatorIDs(Arrays.asList(3,2)); - assay.setOrganismIDs(Arrays.asList(123,3332)); - System.err.println(assay.toJson()); - } - private class Relationships { private String studyId; diff --git a/src/main/java/life/qbic/model/isa/ISADataFile.java b/src/main/java/life/qbic/model/isa/ISADataFile.java index f2fc346..7c0ddaf 100644 --- a/src/main/java/life/qbic/model/isa/ISADataFile.java +++ b/src/main/java/life/qbic/model/isa/ISADataFile.java @@ -61,11 +61,6 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException { - ISADataFile sample = new ISADataFile("my file", "myfile.pdf", Arrays.asList("1")); - System.err.println(sample.toJson()); - } - public String getFileName() { return attributes.getContent_blobs().get(0).getOriginal_filename(); } diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java index 5dce2de..1663b99 100644 --- a/src/main/java/life/qbic/model/isa/ISASample.java +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,9 +24,10 @@ public class ISASample extends AbstractISAObject { private Relationships relationships; private final String ISA_TYPE = "samples"; - public ISASample(Map attributeMap, String sampleType, List projectIds) { - this.attributes = new Attributes(attributeMap); - this.relationships = new Relationships(sampleType, projectIds); + public ISASample(String title, Map attributeMap, String sampleTypeId, + List projectIds) { + this.attributes = new Attributes(title, attributeMap); + this.relationships = new Relationships(sampleTypeId, projectIds); } public ISASample withOtherCreators(String otherCreators) { @@ -61,26 +61,20 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException { - ISASample sample = new ISASample(new HashMap<>(), "1", Arrays.asList("1")); - sample.setCreatorIDs(Arrays.asList("3","2")); - System.err.println(sample.toJson()); - } - private class Relationships { - private String sampleType; + private String sampleTypeId; private List projects; private List creators = new ArrayList<>(); private List assays = new ArrayList<>(); - public Relationships(String sampleType, List projects) { + public Relationships(String sampleTypeId, List projects) { this.projects = projects; - this.sampleType = sampleType; + this.sampleTypeId = sampleTypeId; } public String getSample_type() { - return sampleType; + return sampleTypeId; } public List getAssays() { @@ -120,7 +114,7 @@ public void serialize(Relationships relationships, JsonGenerator jsonGenerator, jsonGenerator.writeStartObject(); jsonGenerator.writeObjectFieldStart("sample_type"); jsonGenerator.writeObjectFieldStart("data"); - jsonGenerator.writeStringField("id", relationships.sampleType); + jsonGenerator.writeStringField("id", relationships.sampleTypeId); jsonGenerator.writeStringField("type", "sample_types"); jsonGenerator.writeEndObject(); jsonGenerator.writeEndObject(); @@ -151,9 +145,15 @@ private class Attributes { private Map attributeMap = new HashMap<>(); private String otherCreators = ""; + private String title; - public Attributes(Map attributeMap) { + public Attributes(String title, Map attributeMap) { this.attributeMap = attributeMap; + this.title = title; + } + + public String getTitle() { + return title; } public Map getAttribute_map() { diff --git a/src/main/java/life/qbic/model/isa/ISASampleType.java b/src/main/java/life/qbic/model/isa/ISASampleType.java new file mode 100644 index 0000000..201a263 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/ISASampleType.java @@ -0,0 +1,220 @@ +package life.qbic.model.isa; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Model class for SampleType. Contains all mandatory and some optional properties and attributes + * that are needed to create a sample type in SEEK. The model and its getters (names) are structured + * in a way to enable the easy translation to JSON to use in SEEK queries. + * Mandatory parameters are found in the constructor, optional attributes can be set using + * withAttribute(attribute) notation. + * Can be used to populate a SEEK installation with sample types taken from another system's API. + */ +public class ISASampleType extends AbstractISAObject { + + private Attributes attributes; + private Relationships relationships; + private final String ISA_TYPE = "sample_types"; + + public ISASampleType(String title, SampleAttribute titleAttribute, String projectID) { + this.attributes = new Attributes(title, titleAttribute); + this.relationships = new Relationships(Arrays.asList(projectID)); + } + + public void addSampleAttribute(String title, SampleAttributeType sampleAttributeType, + boolean required, String linkedSampleTypeIdOrNull) { + attributes.addSampleAttribute(title, sampleAttributeType, required, linkedSampleTypeIdOrNull); + } + + public ISASampleType withAssays(List assays) { + this.relationships.assays = assays; + return this; + } + + public String toJson() throws JsonProcessingException { + SimpleModule module = new SimpleModule(); + module.addSerializer(Relationships.class, new RelationshipsSerializer()); + return super.toJson(module); + } + + public String getType() { + return ISA_TYPE; + } + + public Relationships getRelationships() { + return relationships; + } + + public Attributes getAttributes() { + return attributes; + } + + private class Relationships { + + private List projects; + private List assays; + + public Relationships(List projects) { + this.projects = projects; + this.assays = new ArrayList<>(); + } + + public List getProjects() { + return projects; + } + + public List getAssays() { + return assays; + } + } + + public class RelationshipsSerializer extends StdSerializer { + + public RelationshipsSerializer(Class t) { + super(t); + } + + public RelationshipsSerializer() { + this(null); + } + + @Override + public void serialize(Relationships relationships, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeStartObject(); + + generateListJSON(jsonGenerator, "projects", relationships.projects, "projects"); + generateListJSON(jsonGenerator, "assays", relationships.assays, "assays"); + + jsonGenerator.writeEndObject(); + } + } + + private void generateListJSON(JsonGenerator generator, String name, List items, + String type) + throws IOException { + generator.writeObjectFieldStart(name); + generator.writeArrayFieldStart("data"); + for (String item : items) { + generator.writeStartObject(); + generator.writeStringField("id", item); + generator.writeStringField("type", type); + generator.writeEndObject(); + } + generator.writeEndArray(); + generator.writeEndObject(); + } + + private class Attributes { + + private String title; + private List sampleAttributes = new ArrayList<>();; + + public Attributes(String title, SampleAttribute titleAttribute) { + this.title = title; + if(!titleAttribute.isTitle) { + throw new IllegalArgumentException("The first sample attribute must be the title attribute."); + } + this.sampleAttributes.add(titleAttribute); + } + + public void addSampleAttribute(String title, SampleAttributeType sampleAttributeType, + boolean required, String linkedSampleTypeIdOrNull) { + SampleAttribute sampleAttribute = new SampleAttribute(title, sampleAttributeType, false, + required).withLinkedSampleTypeId(linkedSampleTypeIdOrNull); + sampleAttributes.add(sampleAttribute); + } + + public String getTitle() { + return title; + } + + public List getSample_attributes() { + return sampleAttributes; + } + } + + public static class SampleAttribute { + + private String title; + private String description; + private SampleAttributeType sampleAttributeType; + private boolean isTitle; + private boolean required; + private String linkedSampleTypeId; + + public SampleAttribute(String title, SampleAttributeType sampleAttributeType, boolean isTitle, + boolean required) { + this.title = title; + this.isTitle = isTitle; + this.required = required; + this.sampleAttributeType = sampleAttributeType; + } + + public SampleAttribute withDescription(String description) { + this.description = description; + return this; + } + + public SampleAttribute withLinkedSampleTypeId(String linkedSampleTypeId) { + this.linkedSampleTypeId = linkedSampleTypeId; + return this; + } + + public SampleAttributeType getSample_attribute_type() { + return sampleAttributeType; + } + + public String getLinked_sample_type_id() { + return linkedSampleTypeId; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public boolean getRequired() { + return required; + } + + public boolean getIs_title() { + return isTitle; + } + } + + public static class SampleAttributeType { + private String id; + private String title; + private String baseType; + + public SampleAttributeType(String id, String title, String baseType) { + this.id = id; + this.title = title; + this.baseType = baseType; + } + + public String getBase_type() { + return baseType; + } + + public String getId() { + return id; + } + + public String getTitle() { + return title; + } + } +} diff --git a/src/main/java/life/qbic/model/isa/ISAStudy.java b/src/main/java/life/qbic/model/isa/ISAStudy.java index 1fd19bf..7a30d5a 100644 --- a/src/main/java/life/qbic/model/isa/ISAStudy.java +++ b/src/main/java/life/qbic/model/isa/ISAStudy.java @@ -65,12 +65,6 @@ public Attributes getAttributes() { return attributes; } - public static void main(String[] args) throws JsonProcessingException { - ISAStudy study = new ISAStudy("title", "1"); - study.setCreatorIDs(Arrays.asList(3,2)); - System.err.println(study.toJson()); - } - private class Relationships { private String investigationId; diff --git a/src/main/java/life/qbic/model/isa/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java index 323876c..9dd45b4 100644 --- a/src/main/java/life/qbic/model/isa/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -1,11 +1,8 @@ -package life.qbic.model; +package life.qbic.model.isa; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.util.List; import java.util.Map; -import life.qbic.model.isa.GenericSeekAsset; -import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISASample; public class SeekStructure { From a87fed8f0931fb9003cc8b268d6b1a307c7d69c7 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 7 Oct 2024 16:43:13 +0200 Subject: [PATCH 30/63] small fixes and features --- .../io/commandline/AuthenticationOptions.java | 20 ++++- .../io/commandline/DownloadPetabCommand.java | 2 +- .../io/commandline/FindDatasetsCommand.java | 2 +- .../commandline/SampleHierarchyCommand.java | 2 +- .../commandline/SpaceStatisticsCommand.java | 2 +- .../TransferDataToSeekCommand.java | 6 +- .../TransferSampleTypesToSeekCommand.java | 6 +- .../io/commandline/UploadDatasetCommand.java | 2 +- .../commandline/UploadPetabResultCommand.java | 2 +- .../qbic/model/OpenbisSeekTranslator.java | 36 +++++++-- .../qbic/model/download/SEEKConnector.java | 73 +++++++++++++++---- 11 files changed, 116 insertions(+), 37 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java index 1b9c5d0..eba85d1 100644 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java @@ -8,6 +8,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; import java.util.StringJoiner; import java.util.TreeMap; import org.apache.logging.log4j.LogManager; @@ -29,13 +31,13 @@ public class AuthenticationOptions { @Option( names = {"-as", "-as_url"}, - description = "ApplicationServer URL", + description = "OpenBIS ApplicationServer URL", scope = CommandLine.ScopeType.INHERIT) private String as_url; @Option( names = {"-dss", "--dss_url"}, - description = "DatastoreServer URL", + description = "OpenBIS DatastoreServer URL", scope = CommandLine.ScopeType.INHERIT) private String dss_url; @@ -85,14 +87,14 @@ public String getSeekURL() { return seek_url; } - public String getDSS() { + public String getOpenbisDSS() { if(dss_url == null & configPath!=null && !configPath.isBlank()) { dss_url = ReadProperties.getProperties(configPath).get("dss"); } return dss_url; } - public String getAS() { + public String getOpenbisAS() { if(as_url == null & configPath!=null && !configPath.isBlank()) { as_url = ReadProperties.getProperties(configPath).get("as"); } @@ -107,6 +109,16 @@ public char[] getOpenbisPassword() { return openbisPasswordOptions.getPassword(); } + public String getOpenbisBaseURL() throws MalformedURLException { + URL asURL = new URL(as_url); + String res = asURL.getProtocol()+ "://" +asURL.getHost(); + if(asURL.getPort()!=-1) { + res+=":"+asURL.getPort(); + } + System.err.println(res); + return res; + } + /** * official picocli documentation example */ diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index 46efefe..5be6b09 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -27,7 +27,7 @@ public class DownloadPetabCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 2d340bd..9e384f9 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -39,7 +39,7 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() .sorted(Comparator.comparing( diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 53b2bf3..2298693 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -43,7 +43,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index ec2e2d0..6bed496 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -50,7 +50,7 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); if (spaces.isEmpty()) { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 72d8934..dbaeb67 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -76,7 +76,7 @@ public void run() { System.out.println("Connecting to openBIS..."); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getAS(), auth.getDSS()); + auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); @@ -106,8 +106,8 @@ public void run() { byte[] httpCredentials = Base64.encodeBase64( (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", - "lisym default study"); + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), + "seek_test", "lisym default study"); translator = seek.getTranslator(); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java index cdd6ee4..301f09a 100644 --- a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -28,7 +28,7 @@ public void run() { System.out.println("auth..."); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getAS(), auth.getDSS()); + auth.getOpenbisAS(), auth.getOpenbisDSS()); System.out.println("openbis..."); openbis = new OpenbisConnector(authentication); @@ -36,8 +36,8 @@ public void run() { byte[] httpCredentials = Base64.encodeBase64( (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, "seek_test", - "lisym default study"); + seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), + "seek_test", "lisym default study"); translator = seek.getTranslator(); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 9e2d3a9..a24a0c7 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -32,7 +32,7 @@ public class UploadDatasetCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 6ee72a9..8e36b1e 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -33,7 +33,7 @@ public class UploadPetabResultCommand implements Runnable { @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getAS(), auth.getDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 0504636..dfcc025 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -2,7 +2,6 @@ import static java.util.Map.entry; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.DataType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; @@ -16,13 +15,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; -import life.qbic.model.isa.ISADataFile; import life.qbic.model.isa.ISASample; import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISASampleType.SampleAttribute; @@ -34,12 +33,24 @@ public class OpenbisSeekTranslator { private final String DEFAULT_PROJECT_ID; private final String DEFAULT_STUDY_ID; private final String DEFAULT_TRANSFERRED_SAMPLE_TITLE = "openBIS Name"; + private final String openBISBaseURL; - public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { + public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID, String defaultStudyID) { + this.openBISBaseURL = openBISBaseURL; this.DEFAULT_PROJECT_ID = defaultProjectID; this.DEFAULT_STUDY_ID = defaultStudyID; } + private String generateOpenBISLinkFromPermID(String entityType, String permID) { + StringBuilder builder = new StringBuilder(); + builder.append(openBISBaseURL); + builder.append("#entity="); + builder.append(entityType); + builder.append("&permId="); + builder.append(permID); + return builder.toString(); + } + Map experimentTypeToAssayClass = Map.ofEntries( entry("00_MOUSE_DATABASE", "EXP"), entry("00_PATIENT_DATABASE", "EXP"), @@ -63,8 +74,8 @@ public OpenbisSeekTranslator(String defaultProjectID, String defaultStudyID) { new SampleAttributeType("8", "String", "String")), entry(DataType.HYPERLINK, new SampleAttributeType("8", "String", "String")), entry(DataType.XML, new SampleAttributeType("7", "Text", "Text")), - entry(DataType.SAMPLE, //should be handled before mapping types - new SampleAttributeType("8", "String", "String")), + entry(DataType.SAMPLE, //we link the sample as URL to openBIS for now + new SampleAttributeType("5", "Web link", "String")), entry(DataType.DATE, new SampleAttributeType("2", "Date time", "Date")) ); @@ -150,12 +161,23 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, //try to put all attributes into sample properties, as they should be a 1:1 mapping Map typeCodesToNames = new HashMap<>(); + Set propertiesLinkingSamples = new HashSet<>(); for (PropertyAssignment a : sampleType.getPropertyAssignments()) { - typeCodesToNames.put(a.getPropertyType().getCode(), a.getPropertyType().getLabel()); + String code = a.getPropertyType().getCode(); + String label = a.getPropertyType().getLabel(); + DataType type = a.getPropertyType().getDataType(); + typeCodesToNames.put(code, label); + if(type.equals(DataType.SAMPLE)) { + propertiesLinkingSamples.add(code); + } } Map attributes = new HashMap<>(); for(String code : sample.getProperties().keySet()) { - attributes.put(typeCodesToNames.get(code), sample.getProperties().get(code)); + String value = sample.getProperty(code); + if(propertiesLinkingSamples.contains(code)) { + value = generateOpenBISLinkFromPermID("SAMPLE", value); + } + attributes.put(typeCodesToNames.get(code), value); } attributes.put(DEFAULT_TRANSFERRED_SAMPLE_TITLE, sample.getIdentifier().getIdentifier()); diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 580bf0f..16fb0ec 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -30,7 +30,6 @@ import life.qbic.model.isa.ISASample; import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISAStudy; -import org.apache.commons.codec.binary.Base64; import org.apache.http.client.utils.URIBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -41,13 +40,56 @@ public class SEEKConnector { private String apiURL; private byte[] credentials; private OpenbisSeekTranslator translator; + private final String DEFAULT_PROJECT_ID; - public SEEKConnector(String apiURL, byte[] httpCredentials, String defaultProjectTitle, - String defaultStudyTitle) throws URISyntaxException, IOException, InterruptedException { + public SEEKConnector(String apiURL, byte[] httpCredentials, String openBISBaseURL, + String defaultProjectTitle, String defaultStudyTitle) throws URISyntaxException, IOException, + InterruptedException { this.apiURL = apiURL; this.credentials = httpCredentials; - translator = new OpenbisSeekTranslator("1", //searchNodeWithTitle("projects", defaultProjectTitle), - "1");//searchNodeWithTitle("studies", defaultStudyTitle)); + Optional projectID = getProjectWithTitle(defaultProjectTitle); + if(projectID.isEmpty()) { + throw new RuntimeException("Failed to find project with title: " + defaultProjectTitle+". " + + "Please provide an existing default project."); + } + DEFAULT_PROJECT_ID = projectID.get(); + + translator = new OpenbisSeekTranslator(openBISBaseURL, DEFAULT_PROJECT_ID, + searchNodeWithTitle("studies", defaultStudyTitle)); + } + + /** + * Lists projects and returns the optional identifier of the one matching the provided ID. + * Necessary because project search does not seem to work. + * @param projectTitle the title to search for + * @return + */ + private Optional getProjectWithTitle(String projectTitle) + throws IOException, InterruptedException, URISyntaxException { + String endpoint = apiURL+"/projects/"; + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode hits = rootNode.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + String id = hit.get("id").asText(); + String title = hit.get("attributes").get("title").asText(); + if(title.equals(projectTitle)) { + return Optional.of(id); + } + } + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + return Optional.empty(); } public String addAssay(ISAAssay assay) @@ -231,13 +273,16 @@ public String uploadStreamContent(String blobEndpoint, HttpRequest request = HttpRequest.newBuilder() .uri(new URI(blobEndpoint)) .headers("Content-Type", "application/octet-stream") - .headers("Accept", "application/octet-stream") + .headers("Accept", "*/*") .headers("Authorization", "Basic " + new String(credentials)) .PUT(BodyPublishers.ofInputStream(streamSupplier)).build(); HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); + System.err.println("response was: "+response); + System.err.println("response body: "+response.body()); + if(response.statusCode()!=200) { System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); @@ -276,9 +321,9 @@ public Optional createAssetForFile(DataSetFile file, String asset if(!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName(); - GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), - Arrays.asList("1"));//TODO + String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? + GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), + Arrays.asList(DEFAULT_PROJECT_ID)); isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); String annotation = translator.dataFormatAnnotationForExtension(fileExtension); @@ -298,9 +343,9 @@ public List createAssets(List filesInDataset, if(!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName();//TODO? + String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), - Arrays.asList("1")); + Arrays.asList(DEFAULT_PROJECT_ID)); isaFile.withAssays(assays); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); String annotation = translator.dataFormatAnnotationForExtension(fileExtension); @@ -384,7 +429,7 @@ private Map parseSampleTypesJSON(String json) throws JsonProcess private String searchNodeWithTitle(String nodeType, String title) throws URISyntaxException, IOException, InterruptedException { - String endpoint = apiURL+"/search/"; + String endpoint = apiURL+"/search"; URIBuilder builder = new URIBuilder(endpoint); builder.setParameter("q", title).setParameter("search_type", nodeType); @@ -402,8 +447,7 @@ private String searchNodeWithTitle(String nodeType, String title) JsonNode hits = rootNode.path("data"); for (Iterator it = hits.elements(); it.hasNext(); ) { JsonNode hit = it.next(); - System.err.println(hit.asText()); - if(hit.get("title").asText().equals(title)) { + if(hit.get("attributes").get("title").asText().equals(title)) { return hit.get("id").asText(); } } @@ -458,6 +502,7 @@ public AssayWithQueuedAssets createNode(SeekStructure nodeWithChildren, boolean throws URISyntaxException, IOException, InterruptedException { String assayID = addAssay(nodeWithChildren.getAssay()); for(ISASample sample : nodeWithChildren.getSamples()) { + sample.setAssayIDs(Arrays.asList(assayID)); createSample(sample); } From 4ce251a78b90d321270bbd880c969c9412893008 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 11 Oct 2024 16:32:53 +0200 Subject: [PATCH 31/63] several new features and fixes --- src/main/java/life/qbic/App.java | 16 +- .../java/life/qbic/io/PropertyReader.java | 44 ++ .../io/commandline/AuthenticationOptions.java | 238 -------- .../io/commandline/CommandLineOptions.java | 23 +- .../io/commandline/DownloadPetabCommand.java | 14 +- .../io/commandline/FindDatasetsCommand.java | 12 +- .../io/commandline/OpenBISPasswordParser.java | 30 -- .../OpenbisAuthenticationOptions.java | 124 +++++ .../commandline/SampleHierarchyCommand.java | 5 +- .../SeekAuthenticationOptions.java | 97 ++++ .../commandline/SpaceStatisticsCommand.java | 2 +- .../TransferDataToSeekCommand.java | 189 ++++--- .../TransferSampleTypesToSeekCommand.java | 39 +- .../io/commandline/UploadDatasetCommand.java | 2 +- .../commandline/UploadPetabResultCommand.java | 3 +- .../life/qbic/model/AssetInformation.java | 41 ++ .../qbic/model/OpenbisSeekTranslator.java | 192 ++++--- .../life/qbic/model/SampleInformation.java | 28 + .../qbic/model/download/OpenbisConnector.java | 140 ++++- .../qbic/model/download/SEEKConnector.java | 508 ++++++++++++++---- .../qbic/model/isa/AbstractISAObject.java | 3 + .../life/qbic/model/isa/GenericSeekAsset.java | 62 ++- .../java/life/qbic/model/isa/ISAAssay.java | 28 +- .../java/life/qbic/model/isa/ISASample.java | 35 ++ .../life/qbic/model/isa/ISASampleType.java | 10 + .../life/qbic/model/isa/SeekStructure.java | 42 +- 26 files changed, 1294 insertions(+), 633 deletions(-) create mode 100644 src/main/java/life/qbic/io/PropertyReader.java delete mode 100644 src/main/java/life/qbic/io/commandline/AuthenticationOptions.java delete mode 100644 src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java create mode 100644 src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java create mode 100644 src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java create mode 100644 src/main/java/life/qbic/model/AssetInformation.java create mode 100644 src/main/java/life/qbic/model/SampleInformation.java diff --git a/src/main/java/life/qbic/App.java b/src/main/java/life/qbic/App.java index 6ad90a6..18ddaed 100644 --- a/src/main/java/life/qbic/App.java +++ b/src/main/java/life/qbic/App.java @@ -1,6 +1,9 @@ package life.qbic; import ch.ethz.sis.openbis.generic.OpenBIS; +import java.util.HashMap; +import java.util.Map; +import life.qbic.io.PropertyReader; import life.qbic.io.commandline.CommandLineOptions; import life.qbic.model.Configuration; import life.qbic.model.download.AuthenticationException; @@ -18,6 +21,7 @@ public class App { private static final Logger LOG = LogManager.getLogger(App.class); + public static Map configProperties = new HashMap<>(); public static void main(String[] args) { LOG.debug("command line arguments: " + Arrays.deepToString(args)); @@ -26,6 +30,14 @@ public static void main(String[] args) { System.exit(exitCode); } + public static void readConfig() { + System.err.println("reading config"); + String configPath = CommandLineOptions.getConfigPath(); + if(configPath != null && !configPath.isEmpty()) { + configProperties = PropertyReader.getProperties(configPath); + } + } + /** * checks if the commandline parameter for reading out the password from the environment variable * is correctly provided @@ -57,7 +69,9 @@ public static OpenBIS loginToOpenBIS( char[] password, String user, String url, String dssUrl) { setupLog(); - OpenBIS authentication = new OpenBIS(url, dssUrl); + int generousTimeOut = 30*60*1000; //30 mins + + OpenBIS authentication = new OpenBIS(url, dssUrl, generousTimeOut); return tryLogin(authentication, user, password); } diff --git a/src/main/java/life/qbic/io/PropertyReader.java b/src/main/java/life/qbic/io/PropertyReader.java new file mode 100644 index 0000000..61e89a0 --- /dev/null +++ b/src/main/java/life/qbic/io/PropertyReader.java @@ -0,0 +1,44 @@ +package life.qbic.io; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.TreeMap; + +public class PropertyReader { + + public static TreeMap getProperties(String infile) { + + TreeMap properties = new TreeMap<>(); + BufferedReader bfr = null; + try { + bfr = new BufferedReader(new FileReader(new File(infile))); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + + String line; + while (true) { + try { + if ((line = bfr.readLine()) == null) + break; + } catch (IOException e) { + throw new RuntimeException(e); + } + if (!line.startsWith("#") && !line.isEmpty()) { + String[] property = line.trim().split("="); + properties.put(property[0].trim(), property[1].trim()); + } + } + + try { + bfr.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return(properties); + } +} diff --git a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java deleted file mode 100644 index eba85d1..0000000 --- a/src/main/java/life/qbic/io/commandline/AuthenticationOptions.java +++ /dev/null @@ -1,238 +0,0 @@ -package life.qbic.io.commandline; - -import static java.util.Objects.nonNull; -import static picocli.CommandLine.ArgGroup; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.StringJoiner; -import java.util.TreeMap; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import picocli.CommandLine; -import picocli.CommandLine.Option; - -public class AuthenticationOptions { - private static final Logger log = LogManager.getLogger(AuthenticationOptions.class); - - @Option( - names = {"-u", "--user"}, - description = "openBIS user name") - private String openbisUser; - @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. - OpenbisPasswordOptions openbisPasswordOptions; - @ArgGroup(multiplicity = "1") - SeekPasswordOptions seekPasswordOptions; - - @Option( - names = {"-as", "-as_url"}, - description = "OpenBIS ApplicationServer URL", - scope = CommandLine.ScopeType.INHERIT) - private String as_url; - - @Option( - names = {"-dss", "--dss_url"}, - description = "OpenBIS DatastoreServer URL", - scope = CommandLine.ScopeType.INHERIT) - private String dss_url; - - @Option( - names = {"-config", "--config_file"}, - description = "Config file path to provide server and user information.", - scope = CommandLine.ScopeType.INHERIT) - public String configPath; - @Option( - names = {"-su", "--seek-user"}, - description = "Seek user name (email)", - scope = CommandLine.ScopeType.INHERIT) - private String seekUser; - @Option( - names = {"-seek-server", "-seek_url"}, - description = "SEEK API URL", - scope = CommandLine.ScopeType.INHERIT) - private String seek_url; - - public String getOpenbisUser() { - if(openbisUser == null & configPath!=null && !configPath.isBlank()) { - openbisUser = ReadProperties.getProperties(configPath).get("user"); - } else { - log.error("No openBIS user provided."); - System.exit(2); - } - return openbisUser; - } - - public String getSeekUser() { - if(seekUser == null & configPath!=null && !configPath.isBlank()) { - seekUser = ReadProperties.getProperties(configPath).get("seek_user"); - } else { - log.error("No SEEK user/email provided."); - System.exit(2); - } - return seekUser; - } - - public String getSeekURL() { - if(seek_url == null & configPath!=null && !configPath.isBlank()) { - seek_url = ReadProperties.getProperties(configPath).get("seek_url"); - } else { - log.error("No URL to the SEEK address provided."); - System.exit(2); - } - return seek_url; - } - - public String getOpenbisDSS() { - if(dss_url == null & configPath!=null && !configPath.isBlank()) { - dss_url = ReadProperties.getProperties(configPath).get("dss"); - } - return dss_url; - } - - public String getOpenbisAS() { - if(as_url == null & configPath!=null && !configPath.isBlank()) { - as_url = ReadProperties.getProperties(configPath).get("as"); - } - return as_url; - } - - public char[] getSeekPassword() { - return seekPasswordOptions.getPassword(); - } - - public char[] getOpenbisPassword() { - return openbisPasswordOptions.getPassword(); - } - - public String getOpenbisBaseURL() throws MalformedURLException { - URL asURL = new URL(as_url); - String res = asURL.getProtocol()+ "://" +asURL.getHost(); - if(asURL.getPort()!=-1) { - res+=":"+asURL.getPort(); - } - System.err.println(res); - return res; - } - - /** - * official picocli documentation example - */ - static class OpenbisPasswordOptions { - @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") - protected String passwordEnvironmentVariable = ""; - - @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") - protected String passwordProperty = ""; - - @Option(names = "--openbis-pw", arity = "0", description = "please provide your openBIS password", interactive = true) - protected char[] password = null; - - /** - * Gets the password. If no password is provided, the program exits. - * @return the password provided by the user. - */ - char[] getPassword() { - if (nonNull(password)) { - return password; - } - // System.getProperty(String key) does not work for empty or blank keys. - if (!passwordProperty.isBlank()) { - String systemProperty = System.getProperty(passwordProperty); - if (nonNull(systemProperty)) { - return systemProperty.toCharArray(); - } - } - String environmentVariable = System.getenv(passwordEnvironmentVariable); - if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { - return environmentVariable.toCharArray(); - } - log.error("No password provided. Please provide your password."); - System.exit(2); - return null; // not reachable due to System.exit in previous line - } - } - - static class SeekPasswordOptions { - @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") - protected String passwordEnvironmentVariable = ""; - - @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") - protected String passwordProperty = ""; - - @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) - protected char[] password = null; - - /** - * Gets the password. If no password is provided, the program exits. - * @return the password provided by the user. - */ - char[] getPassword() { - if (nonNull(password)) { - return password; - } - // System.getProperty(String key) does not work for empty or blank keys. - if (!passwordProperty.isBlank()) { - String systemProperty = System.getProperty(passwordProperty); - if (nonNull(systemProperty)) { - return systemProperty.toCharArray(); - } - } - String environmentVariable = System.getenv(passwordEnvironmentVariable); - if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { - return environmentVariable.toCharArray(); - } - log.error("No password provided. Please provide your password."); - System.exit(2); - return null; // not reachable due to System.exit in previous line - } - - } - @Override - public String toString() { - return new StringJoiner(", ", AuthenticationOptions.class.getSimpleName() + "[", "]") - .add("user='" + openbisUser + "'") - .toString(); - //ATTENTION: do not expose the password here! - } - - public static class ReadProperties { - - public static TreeMap getProperties(String infile) { - - TreeMap properties = new TreeMap<>(); - BufferedReader bfr = null; - try { - bfr = new BufferedReader(new FileReader(new File(infile))); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } - - String line; - while (true) { - try { - if ((line = bfr.readLine()) == null) - break; - } catch (IOException e) { - throw new RuntimeException(e); - } - if (!line.startsWith("#") && !line.isEmpty()) { - String[] property = line.trim().split("="); - properties.put(property[0].trim(), property[1].trim()); - } - } - - try { - bfr.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - return(properties); - } - } -} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index b583d9c..332bd68 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -17,17 +17,24 @@ public class CommandLineOptions { private static final Logger LOG = LogManager.getLogger(CommandLineOptions.class); + @Option(names = {"-config", "--config_file"}, + description = "Config file path to provide server and user information.", + scope = CommandLine.ScopeType.INHERIT) + static String configPath; + @Option(names = {"-V", "--version"}, - versionHelp = true, - description = "print version information", - scope = CommandLine.ScopeType.INHERIT) + versionHelp = true, + description = "print version information", + scope = CommandLine.ScopeType.INHERIT) boolean versionRequested = false; - @Option( - names = {"-h", "--help"}, - usageHelp = true, - description = "display a help message and exit", - scope = CommandLine.ScopeType.INHERIT) + @Option(names = {"-h", "--help"}, + usageHelp = true, + description = "display a help message and exit", + scope = CommandLine.ScopeType.INHERIT) public boolean helpRequested = false; + public static String getConfigPath() { + return configPath; + } } diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index 5be6b09..a80f861 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import life.qbic.App; import life.qbic.io.PetabParser; import life.qbic.model.DatasetWithProperties; @@ -23,11 +24,12 @@ public class DownloadPetabCommand implements Runnable { @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to store the downloaded data") private String outputPath; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS(), auth.getOpenbisDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); @@ -37,14 +39,16 @@ public void run() { return; } DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); - Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + Set patientIDs = openbis.findPropertiesInSampleHierarchy("PATIENT_DKFZ_ID", result.getExperiment().getIdentifier()); - patientID.ifPresent(s -> result.addProperty("patientID", s)); + if(!patientIDs.isEmpty()) { + result.addProperty("patientIDs", String.join(",", patientIDs)); + } System.out.println("Found dataset, downloading."); System.out.println(); - openbis.downloadDataset(outputPath, datasetCode); + openbis.downloadDataset(outputPath, datasetCode, ""); PetabParser parser = new PetabParser(); try { diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 9e384f9..bb2072f 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import life.qbic.App; import life.qbic.model.DatasetWithProperties; @@ -28,7 +29,7 @@ public class FindDatasetsCommand implements Runnable { @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter samples", names = {"-s", "--space"}) private String space; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { @@ -39,7 +40,8 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() .sorted(Comparator.comparing( @@ -47,9 +49,11 @@ public void run() { Collectors.toList()); Map properties = new HashMap<>(); if (!datasets.isEmpty()) { - Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", + Set patientIDs = openbis.findPropertiesInSampleHierarchy("PATIENT_DKFZ_ID", datasets.get(0).getExperiment().getIdentifier()); - patientID.ifPresent(s -> properties.put("Patient ID", s)); + if(!patientIDs.isEmpty()) { + properties.put("patientIDs", String.join(",", patientIDs)); + } } List datasetWithProperties = datasets.stream().map(dataSet -> { DatasetWithProperties ds = new DatasetWithProperties(dataSet); diff --git a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java b/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java deleted file mode 100644 index 55923f7..0000000 --- a/src/main/java/life/qbic/io/commandline/OpenBISPasswordParser.java +++ /dev/null @@ -1,30 +0,0 @@ -package life.qbic.io.commandline; - -import java.io.Console; -import java.util.Optional; - -public class OpenBISPasswordParser { - - - /** - * Retrieve the password from the system console - * - * @return the password read from the system console input - */ - public static String readPasswordFromConsole() { - Console console = System.console(); - char[] passwordChars = console.readPassword(); - return String.valueOf(passwordChars); - } - - /** - * @param variableName Name of given environment variable - * @return the password read from the environment variable - */ - public static Optional readPasswordFromEnvVariable(String variableName) { - - return Optional.ofNullable(System.getenv(variableName)); - - } - -} diff --git a/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java new file mode 100644 index 0000000..9228d0b --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java @@ -0,0 +1,124 @@ +package life.qbic.io.commandline; + +import static java.util.Objects.nonNull; +import static picocli.CommandLine.ArgGroup; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.StringJoiner; +import life.qbic.App; +import life.qbic.io.PropertyReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +public class OpenbisAuthenticationOptions { + private static final Logger log = LogManager.getLogger(OpenbisAuthenticationOptions.class); + + @Option( + names = {"-u", "--user"}, + description = "openBIS user name") + private String openbisUser; + @ArgGroup(multiplicity = "1") // ensures the password is provided once with at least one of the possible options. + OpenbisPasswordOptions openbisPasswordOptions; + + @Option( + names = {"-as", "-as_url"}, + description = "OpenBIS ApplicationServer URL", + scope = CommandLine.ScopeType.INHERIT) + private String as_url; + + @Option( + names = {"-dss", "--dss_url"}, + description = "OpenBIS DatastoreServer URL", + scope = CommandLine.ScopeType.INHERIT) + private String dss_url; + + public String getOpenbisUser() { + if(openbisUser == null && App.configProperties.containsKey("user")) { + openbisUser = App.configProperties.get("user"); + } else { + log.error("No openBIS user provided."); + System.exit(2); + } + return openbisUser; + } + + public String getOpenbisDSS() { + if(dss_url == null && App.configProperties.containsKey("dss")) { + dss_url = App.configProperties.get("dss"); + } + return dss_url; + } + + public String getOpenbisAS() { + if(as_url == null && App.configProperties.containsKey("as")) { + as_url = App.configProperties.get("as"); + } + return as_url; + } + + public char[] getOpenbisPassword() { + return openbisPasswordOptions.getPassword(); + } + + public String getOpenbisBaseURL() throws MalformedURLException { + URL asURL = new URL(as_url); + String res = asURL.getProtocol()+ "://" +asURL.getHost(); + if(asURL.getPort()!=-1) { + res+=":"+asURL.getPort(); + } + return res; + } + + /** + * official picocli documentation example + */ + static class OpenbisPasswordOptions { + @Option(names = "--openbis-pw:env", arity = "1", paramLabel = "", + description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--openbis-pw:prop", arity = "1", paramLabel = "", + description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--openbis-pw", arity = "0", + description = "please provide your openBIS password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + } + + @Override + public String toString() { + return new StringJoiner(", ", OpenbisAuthenticationOptions.class.getSimpleName() + "[", "]") + .add("user='" + openbisUser + "'") + .toString(); + //ATTENTION: do not expose the password here! + } + +} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 2298693..228d615 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -31,7 +31,7 @@ public class SampleHierarchyCommand implements Runnable { @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) private String outpath; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { @@ -43,7 +43,8 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); diff --git a/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java new file mode 100644 index 0000000..8a58d6e --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java @@ -0,0 +1,97 @@ +package life.qbic.io.commandline; + +import static java.util.Objects.nonNull; +import static picocli.CommandLine.ArgGroup; + +import java.util.StringJoiner; +import life.qbic.App; +import life.qbic.io.PropertyReader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +public class SeekAuthenticationOptions { + private static final Logger log = LogManager.getLogger(SeekAuthenticationOptions.class); + + @ArgGroup(multiplicity = "1") + SeekPasswordOptions seekPasswordOptions; + + @Option( + names = {"-su", "--seek-user"}, + description = "Seek user name (email)", + scope = CommandLine.ScopeType.INHERIT) + private String seekUser; + @Option( + names = {"-seek-server", "-seek_url"}, + description = "SEEK API URL", + scope = CommandLine.ScopeType.INHERIT) + private String seek_url; + + public String getSeekUser() { + if(seekUser == null && App.configProperties.containsKey("seek_user")) { + seekUser = App.configProperties.get("seek_user"); + } else { + log.error("No SEEK user/email provided."); + System.exit(2); + } + return seekUser; + } + + public String getSeekURL() { + if(seek_url == null && App.configProperties.containsKey("seek_url")) { + seek_url = App.configProperties.get("seek_url"); + } else { + log.error("No URL to the SEEK address provided."); + System.exit(2); + } + return seek_url; + } + + public char[] getSeekPassword() { + return seekPasswordOptions.getPassword(); + } + + static class SeekPasswordOptions { + @Option(names = "--seek-pw:env", arity = "1", paramLabel = "", description = "provide the name of an environment variable to read in your password from") + protected String passwordEnvironmentVariable = ""; + + @Option(names = "--seek-pw:prop", arity = "1", paramLabel = "", description = "provide the name of a system property to read in your password from") + protected String passwordProperty = ""; + + @Option(names = "--seek-pw", arity = "0", description = "please provide your SEEK password", interactive = true) + protected char[] password = null; + + /** + * Gets the password. If no password is provided, the program exits. + * @return the password provided by the user. + */ + char[] getPassword() { + if (nonNull(password)) { + return password; + } + // System.getProperty(String key) does not work for empty or blank keys. + if (!passwordProperty.isBlank()) { + String systemProperty = System.getProperty(passwordProperty); + if (nonNull(systemProperty)) { + return systemProperty.toCharArray(); + } + } + String environmentVariable = System.getenv(passwordEnvironmentVariable); + if (nonNull(environmentVariable) && !environmentVariable.isBlank()) { + return environmentVariable.toCharArray(); + } + log.error("No password provided. Please provide your password."); + System.exit(2); + return null; // not reachable due to System.exit in previous line + } + + } + @Override + public String toString() { + return new StringJoiner(", ", SeekAuthenticationOptions.class.getSimpleName() + "[", "]") + .add("user='" + seekUser + "'") + .toString(); + //ATTENTION: do not expose the password here! + } +} \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index 6bed496..4ae0323 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -37,7 +37,7 @@ public class SpaceStatisticsCommand implements Runnable { names = {"--show-settings"}) private boolean allSpaces; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @Override public void run() { diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index dbaeb67..4160d45 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -1,9 +1,7 @@ package life.qbic.io.commandline; import ch.ethz.sis.openbis.generic.OpenBIS; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -17,15 +15,18 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import life.qbic.model.download.SEEKConnector.AssetToUpload; import org.apache.commons.codec.binary.Base64; +import org.xml.sax.SAXException; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; @@ -36,7 +37,7 @@ "Transfers metadata and (optionally) data from openBIS to SEEK. Experiments, samples and " + "dataset information are always transferred together (as assays, samples and one of " + "several data types in SEEK). Dataset info always links back to the openBIS path of " - + "the respective dataset. The data itself will can transferred and stored in SEEK " + + "the respective dataset. The data itself can be transferred and stored in SEEK " + "using the '-d' flag." + "To completely exclude some dataset information from being transferred, a " + "file ('--blacklist') containing dataset codes (from openBIS) can be specified." @@ -47,6 +48,12 @@ public class TransferDataToSeekCommand implements Runnable { @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + "experiment, sample or dataset to transfer.") private String objectID; + @Parameters(arity = "1", paramLabel = "seek study", description = "Title of the study in SEEK" + + "where nodes should be added.") + private String studyTitle; + @Option(names = "--seek-project", description = "Title of the project in SEEK where nodes should" + + "be added. Can alternatively be provided via the config file as 'seek_default_project'.") + private String projectTitle; @Option(names = "--blacklist", description = "Path to file specifying by dataset " + "dataset code which openBIS datasets not to transfer to SEEK. The file must contain one code " + "per line.") @@ -55,28 +62,42 @@ public class TransferDataToSeekCommand implements Runnable { + "information in SEEK for the specified openBIS input should not be updated, but new nodes " + "created.") private boolean noUpdate; - /*@Option(names = {"-sn", "--seek-node"}, paramLabel = "seek node", description = - "The target node in SEEK to transfer to. Must correspond to " - + "the type of oopenBIS identifier chosen: experiment - assay; sample - sample; dataset - any of the data types. If no node is specified, " - + "a new data structure will be created in SEEK, starting from the related experiment.") - private String seekNode; - */ @Option(names = {"-d", "--data"}, description = "Transfers the data itself to SEEK along with the metadata. " + "Otherwise only the link(s) to the openBIS object will be created in SEEK.") private boolean transferData; @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + SeekAuthenticationOptions seekAuth = new SeekAuthenticationOptions(); + @Mixin + OpenbisAuthenticationOptions openbisAuth = new OpenbisAuthenticationOptions(); OpenbisConnector openbis; SEEKConnector seek; OpenbisSeekTranslator translator; + //500 MB - user will be informed that the transfer will take a while, for each file larger than this + private final long FILE_WARNING_SIZE = 500*1024*1024; @Override public void run() { + App.readConfig(); + System.out.printf("Transfer openBIS -> SEEK started.%n"); + System.out.printf("Provided openBIS object: %s%n", objectID); + System.out.printf("Provided SEEK study title: %s%n", studyTitle); + if(projectTitle!=null && !projectTitle.isBlank()) { + System.out.printf("Provided SEEK project title: %s%n", projectTitle); + } else { + System.out.printf("No SEEK project title provided, will search config file.%n"); + } + System.out.printf("Provided SEEK study title: %s%n", studyTitle); + System.out.printf("Transfer datasets to SEEK? %s%n", transferData); + System.out.printf("Update existing assay if found? %s%n", !noUpdate); + if(blacklistFile!=null && !blacklistFile.isBlank()) { + System.out.printf("File with datasets codes that won't be transferred: %s%n", blacklistFile); + } + System.out.println("Connecting to openBIS..."); - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), + openbisAuth.getOpenbisUser(), openbisAuth.getOpenbisAS(), openbisAuth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); @@ -104,35 +125,22 @@ public void run() { System.out.println("Connecting to SEEK..."); byte[] httpCredentials = Base64.encodeBase64( - (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + (seekAuth.getSeekUser() + ":" + new String(seekAuth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), - "seek_test", "lisym default study"); + String project = App.configProperties.get("seek_default_project"); + if(project == null || project.isBlank()) { + throw new RuntimeException("a default project must be provided via config "+ + "('seek_default_project') or parameter."); + } + seek = new SEEKConnector(seekAuth.getSeekURL(), httpCredentials, + openbisAuth.getOpenbisBaseURL(), App.configProperties.get("seek_default_project")); + seek.setDefaultStudy(studyTitle); translator = seek.getTranslator(); - } catch (URISyntaxException | IOException | InterruptedException e) { + } catch (URISyntaxException | IOException | InterruptedException | + ParserConfigurationException | SAXException e) { throw new RuntimeException(e); } - /* - if (seekNode != null) { - try { - if (!seek.endPointExists(seekNode)) { - System.out.println(seekNode + " could not be found"); - return; - } else { - if (isExperiment) { - Experiment experimentWithSamplesAndDatasets = openbis.getExperimentWithDescendants( - objectID); - //SeekStructure seekStructure = translator.translate(seekNode, experimentWithSamplesAndDatasets); - //seek.fillAssayWithSamplesAndDatasets(seekNode, seekStructure); - } - } - } catch (URISyntaxException | InterruptedException | IOException e) { - throw new RuntimeException(e); - } - } - - */ try { System.out.println("Collecting information from openBIS..."); OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); @@ -143,43 +151,24 @@ public void run() { System.out.println("Translating openBIS property codes to SEEK names..."); Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); System.out.println("Creating SEEK structure..."); - SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist); - if(assayID.isEmpty()) { - System.out.println("Creating new nodes..."); - createNewNodes(nodeWithChildren); + SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist, + transferData); + + if(assayID.isEmpty() || noUpdate) { + System.out.println("Creating new node(s)..."); + SeekStructurePostRegistrationInformation postRegInfo = createNewNodes(nodeWithChildren); + System.out.println("Creating links to SEEK objects in openBIS..."); + openbis.createSeekLinks(postRegInfo); } else { System.out.println("Updating nodes..."); - updateNodes(nodeWithChildren, assayID.get()); + SeekStructurePostRegistrationInformation postRegInfo = updateNodes(nodeWithChildren, + assayID.get()); + System.out.println("Updating links to SEEK objects in openBIS..."); + openbis.updateSeekLinks(postRegInfo); } } catch (URISyntaxException | InterruptedException | IOException e) { throw new RuntimeException(e); } - - /* - - List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); - - if(datasets.isEmpty()) { - System.out.println(datasetCode+" not found"); - return; - } - - DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); - Optional patientID = openbis.findPropertyInSampleHierarchy("PATIENT_DKFZ_ID", - result.getExperiment().getIdentifier()); - patientID.ifPresent(s -> result.addProperty("patientID", s)); - - System.out.println("Found dataset, downloading."); - System.out.println(); - - final String tmpPath = "tmp/"; - - File downloadFolder = openbis.downloadDataset(tmpPath, datasetCode); - - - - cleanupTemp(new File(tmpPath)); -*/ System.out.println("Done"); } @@ -206,23 +195,50 @@ private Set parseBlackList(String blacklistFile) { } } - private void updateNodes(SeekStructure nodeWithChildren, String assayID) { - String updatedEndpoint = seek.updateNode(nodeWithChildren, assayID, transferData); - System.out.printf("%s was successfully updated.%n", updatedEndpoint); + private SeekStructurePostRegistrationInformation updateNodes(SeekStructure nodeWithChildren, + String assayID) throws URISyntaxException, IOException, InterruptedException { + SeekStructurePostRegistrationInformation postRegInfo = seek.updateNode(nodeWithChildren, assayID); + System.out.printf("%s was successfully updated.%n", postRegInfo.getExperimentIDWithEndpoint().getRight()); + return postRegInfo; } - private void createNewNodes(SeekStructure nodeWithChildren) + private SeekStructurePostRegistrationInformation createNewNodes(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - AssayWithQueuedAssets assetsOfAssayToUpload = seek.createNode(nodeWithChildren, transferData); + SeekStructurePostRegistrationInformation postRegistrationInformation = + seek.createNode(nodeWithChildren); + AssayWithQueuedAssets assetsOfAssayToUpload = postRegistrationInformation.getAssayWithQueuedAssets(); if(transferData) { + final String tmpFolderPath = "tmp/"; for(AssetToUpload asset : assetsOfAssayToUpload.getAssets()) { - System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); - String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), - () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); - System.out.printf("File stored here: %s%n", fileURL); + String filePath = asset.getFilePath(); + String dsCode = asset.getDataSetCode(); + if(asset.getFileSizeInBytes() > 1000*1024*1024) { + System.out.printf("Skipping %s due to size...%n", + filePath); + } else if (asset.getFileSizeInBytes() > 300 * 1024 * 1024) { + System.out.printf("File is %s MB...streaming might take a while%n", + asset.getFileSizeInBytes() / (1024 * 1024)); + System.out.printf("Downloading file %s from openBIS to tmp folder due to size...%n", + filePath); + File tmpFile = openbis.downloadDataset(tmpFolderPath, dsCode, filePath); + + System.out.printf("Uploading file to SEEK...%n"); + String fileURL = seek.uploadFileContent(asset.getBlobEndpoint(), tmpFile.getAbsolutePath()); + System.out.printf("File stored here: %s%n", fileURL); + } else { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); + } } + System.out.printf("Cleaning up temp folder%n"); + cleanupTemp(new File(tmpFolderPath)); } + System.out.printf("%s was successfully created.%n", assetsOfAssayToUpload.getAssayEndpoint()); + return postRegistrationInformation; } private boolean sampleExists(String objectID) { @@ -246,6 +262,7 @@ private Optional getAssayIDForOpenBISExperiment(Experiment experiment) String permID = experiment.getPermId().getPermId(); List assayIDs = seek.searchAssaysContainingKeyword(permID); if(assayIDs.isEmpty()) { + System.err.println("no assay found containing "+permID); return Optional.empty(); } if(assayIDs.size() == 1) { @@ -254,26 +271,6 @@ private Optional getAssayIDForOpenBISExperiment(Experiment experiment) throw new RuntimeException("Experiment identifier "+permID+ " was found in more than one assay: "+assayIDs); } - private void sendDatasetToSeek(String datasetCode, String assayID) - throws URISyntaxException, IOException, InterruptedException { - assayID = "3"; - System.out.println("Searching dataset in openBIS..."); - List datasets = openbis.findDataSets( - Arrays.asList(datasetCode)); - if(datasets.isEmpty()) { - return; - } - DataSet dataset = datasets.get(0); - List files = openbis.getDatasetFiles(dataset); - List assets = seek.createAssets(files, Arrays.asList(assayID)); - for(AssetToUpload asset : assets) { - System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); - String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), - () -> openbis.streamDataset(datasetCode, asset.getFilePath())); - System.out.printf("File stored here: %s%n", fileURL); - } - } - private void cleanupTemp(File tmpFolder) { File[] files = tmpFolder.listFiles(); if (files != null) { //some JVMs return null for empty dirs diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java index 301f09a..3ba3d55 100644 --- a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -4,21 +4,31 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import java.io.IOException; import java.net.URISyntaxException; +import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; +import life.qbic.io.PropertyReader; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.SampleTypesAndMaterials; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import org.apache.commons.codec.binary.Base64; +import org.xml.sax.SAXException; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; @Command(name = "sample-type-transfer", description = "Transfers sample types from openBIS to SEEK.") public class TransferSampleTypesToSeekCommand implements Runnable { @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + SeekAuthenticationOptions seekAuth = new SeekAuthenticationOptions(); + @Mixin + OpenbisAuthenticationOptions openbisAuth = new OpenbisAuthenticationOptions(); + @Option(names = "--ignore-existing", description = "Use to specify that existing " + + "sample-types of the same name in SEEK should be ignored and the sample-type created a " + + "second time.") + private boolean ignoreExisting; OpenbisConnector openbis; SEEKConnector seek; OpenbisSeekTranslator translator; @@ -27,19 +37,25 @@ public class TransferSampleTypesToSeekCommand implements Runnable { public void run() { System.out.println("auth..."); - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), + openbisAuth.getOpenbisUser(), openbisAuth.getOpenbisAS(), openbisAuth.getOpenbisDSS()); System.out.println("openbis..."); openbis = new OpenbisConnector(authentication); byte[] httpCredentials = Base64.encodeBase64( - (auth.getSeekUser() + ":" + new String(auth.getSeekPassword())).getBytes()); + (seekAuth.getSeekUser() + ":" + new String(seekAuth.getSeekPassword())).getBytes()); try { - seek = new SEEKConnector(auth.getSeekURL(), httpCredentials, auth.getOpenbisBaseURL(), - "seek_test", "lisym default study"); + String project = App.configProperties.get("seek_default_project"); + if(project == null || project.isBlank()) { + throw new RuntimeException("a default project must be provided via config "+ + "('seek_default_project') or parameter."); + } + seek = new SEEKConnector(seekAuth.getSeekURL(), httpCredentials, openbisAuth.getOpenbisBaseURL(), + App.configProperties.get("seek_default_project")); translator = seek.getTranslator(); - } catch (URISyntaxException | IOException | InterruptedException e) { + } catch (URISyntaxException | IOException | InterruptedException | + ParserConfigurationException | SAXException e) { throw new RuntimeException(e); } @@ -48,8 +64,13 @@ public void run() { try { for(SampleType type : types.getSampleTypes()) { System.err.println("creating "+type.getCode()); - String sampleTypeId = seek.createSampleType(translator.translate(type)); - System.err.println("created "+sampleTypeId); + if(!ignoreExisting && seek.sampleTypeExists(type.getCode())) { + System.err.println(type.getCode()+ " is already in SEEK. If you want to create a new " + + "sample type using the same name, you can use the --ignore-existing flag."); + } else { + String sampleTypeId = seek.createSampleType(translator.translate(type)); + System.err.println("created "+sampleTypeId); + } } } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index a24a0c7..98e9f11 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -26,7 +26,7 @@ public class UploadDatasetCommand implements Runnable { + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); private OpenbisConnector openbis; diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 8e36b1e..80d40bc 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -26,7 +26,7 @@ public class UploadPetabResultCommand implements Runnable { // + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); @Mixin - AuthenticationOptions auth = new AuthenticationOptions(); + OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); private OpenbisConnector openbis; private PetabParser petabParser = new PetabParser(); @@ -63,7 +63,6 @@ public void run() { } } System.out.println("Uploading dataset..."); - //TODO copy and remove source references here DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } diff --git a/src/main/java/life/qbic/model/AssetInformation.java b/src/main/java/life/qbic/model/AssetInformation.java new file mode 100644 index 0000000..476d65e --- /dev/null +++ b/src/main/java/life/qbic/model/AssetInformation.java @@ -0,0 +1,41 @@ +package life.qbic.model; + +public class AssetInformation { + + private String seekID; + private String title; + private String description; + private String assetType; + private String openbisPermId; + + public AssetInformation(String assetID, String assetType, String title, String description) { + this.seekID = assetID; + this.title = title; + this.description = description; + this.assetType = assetType; + } + + public String getAssetType() { + return assetType; + } + + public String getTitle() { + return title; + } + + public String getDescription() { + return description; + } + + public String getSeekID() { + return seekID; + } + + public void setOpenbisPermId(String id) { + this.openbisPermId = id; + } + + public String getOpenbisPermId() { + return openbisPermId; + } +} diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index dfcc025..99fb500 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -9,9 +9,9 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -20,6 +20,11 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import life.qbic.App; +import life.qbic.io.PropertyReader; import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; import life.qbic.model.isa.ISASample; @@ -27,18 +32,76 @@ import life.qbic.model.isa.ISASampleType.SampleAttribute; import life.qbic.model.isa.ISASampleType.SampleAttributeType; import life.qbic.model.isa.SeekStructure; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; public class OpenbisSeekTranslator { private final String DEFAULT_PROJECT_ID; - private final String DEFAULT_STUDY_ID; - private final String DEFAULT_TRANSFERRED_SAMPLE_TITLE = "openBIS Name"; + private String INVESTIGATION_ID; + private String STUDY_ID; private final String openBISBaseURL; + private Map experimentTypeToAssayClass; + private Map dataTypeToAttributeType; + private Map datasetTypeToAssetType; + private Map experimentTypeToAssayType; - public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID, String defaultStudyID) { + public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID) + throws IOException, ParserConfigurationException, SAXException { this.openBISBaseURL = openBISBaseURL; this.DEFAULT_PROJECT_ID = defaultProjectID; - this.DEFAULT_STUDY_ID = defaultStudyID; + parseConfigs(); + if(!App.configProperties.containsKey("seek_openbis_sample_title")) { + throw new RuntimeException("Script can not be run, since 'seek_openbis_sample_title' was not " + + "provided."); + } + } + + /** + * Parses mandatory mapping information from mandatory config files. Other files may be added. + */ + private void parseConfigs() throws IOException, ParserConfigurationException, SAXException { + final String dataTypeToAttributeType = "openbis_datatype_to_seek_attributetype.xml"; + final String datasetToAssaytype = "dataset_type_to_assaytype.properties"; + final String experimentTypeToAssayClass = "experiment_type_to_assay_class.properties"; + final String experimentTypeToAssayType = "more.properties"; + this.experimentTypeToAssayType = PropertyReader.getProperties(experimentTypeToAssayType); + this.datasetTypeToAssetType = PropertyReader.getProperties(datasetToAssaytype); + this.experimentTypeToAssayClass = PropertyReader.getProperties(experimentTypeToAssayClass); + this.dataTypeToAttributeType = parseAttributeXML(dataTypeToAttributeType); + } + + private Map parseAttributeXML(String dataTypeToAttributeType) + throws ParserConfigurationException, IOException, SAXException { + Map result = new HashMap<>(); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(dataTypeToAttributeType); + NodeList elements = document.getElementsByTagName("entry"); + for (int i = 0; i < elements.getLength(); i++) { + Node node = elements.item(i); + DataType openbisType = DataType.valueOf(node.getAttributes() + .getNamedItem("type") + .getNodeValue()); + NodeList nodes = node.getChildNodes(); + String seekId = "", seekTitle = "", seekBaseType = ""; + for (int j = 0; j < nodes.getLength(); j++) { + Node n = nodes.item(j); + if (n.getNodeName().equals("seek_id")) { + seekId = n.getTextContent(); + } + if (n.getNodeName().equals("seek_type")) { + seekBaseType = n.getTextContent(); + } + if (n.getNodeName().equals("seek_title")) { + seekTitle = n.getTextContent(); + } + } + result.put(openbisType, new SampleAttributeType(seekId, seekTitle, seekBaseType)); + } + return result; } private String generateOpenBISLinkFromPermID(String entityType, String permID) { @@ -51,44 +114,9 @@ private String generateOpenBISLinkFromPermID(String entityType, String permID) { return builder.toString(); } - Map experimentTypeToAssayClass = Map.ofEntries( - entry("00_MOUSE_DATABASE", "EXP"), - entry("00_PATIENT_DATABASE", "EXP"), - entry("00_STANDARD_OPERATING_PROTOCOLS", "EXP"), - entry("01_BIOLOGICAL_EXPERIMENT", "EXP"), - entry("02_MASSSPECTROMETRY_EXPERIMENT", "EXP"),//where is petab modeling attached? - entry("03_HISTOLOGICAL_ANALYSIS", "EXP"), - entry("04_MICRO_CT", "EXP") - ); - - Map dataTypeToAttributeType = Map.ofEntries( - entry(DataType.INTEGER, new SampleAttributeType("4", "Integer", "Integer")), - entry(DataType.VARCHAR, new SampleAttributeType("8", "String", "String")), - entry(DataType.MULTILINE_VARCHAR, new SampleAttributeType("7", "Text", "Text")), - entry(DataType.REAL, new SampleAttributeType("3", "Real number", "Float")), - entry(DataType.TIMESTAMP, new SampleAttributeType("1", "Date time", "DateTime")), - entry(DataType.BOOLEAN, new SampleAttributeType("16", "Boolean", "Boolean")), - entry(DataType.CONTROLLEDVOCABULARY, //we use String for now - new SampleAttributeType("8", "String", "String")), - entry(DataType.MATERIAL, //not used anymore in this form - new SampleAttributeType("8", "String", "String")), - entry(DataType.HYPERLINK, new SampleAttributeType("8", "String", "String")), - entry(DataType.XML, new SampleAttributeType("7", "Text", "Text")), - entry(DataType.SAMPLE, //we link the sample as URL to openBIS for now - new SampleAttributeType("5", "Web link", "String")), - entry(DataType.DATE, new SampleAttributeType("2", "Date time", "Date")) - ); - - Map experimentTypeToAssayType = Map.ofEntries( - entry("00_MOUSE_DATABASE", ""), - entry("00_PATIENT_DATABASE", ""), - entry("00_STANDARD_OPERATING_PROTOCOLS", ""), - entry("01_BIOLOGICAL_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Cultivation_experiment"), - entry("02_MASSSPECTROMETRY_EXPERIMENT", "http://jermontology.org/ontology/JERMOntology#Proteomics"), - entry("03_HISTOLOGICAL_ANALYSIS", ""), - entry("04_MICRO_CT", "") - ); - + /** + * not mandatory, but nice to have? + */ Map fileExtensionToDataFormat = Map.ofEntries( entry("fastq.gz", "http://edamontology.org/format_1930"), entry("fastq", "http://edamontology.org/format_1930"), @@ -96,34 +124,13 @@ private String generateOpenBISLinkFromPermID(String entityType, String permID) { entry("yaml", "http://edamontology.org/format_3750"), entry("raw", "http://edamontology.org/format_3712"), entry("tsv", "http://edamontology.org/format_3475"), - entry("csv", "http://edamontology.org/format_3752") + entry("csv", "http://edamontology.org/format_3752"), + entry("txt", "http://edamontology.org/format_2330") ); - Map datasetTypeToAssetType = Map.ofEntries( - entry("ANALYSIS_NOTEBOOK", "documents"), - entry("ANALYZED_DATA", "data_files"), - entry("ATTACHMENT", "documents"), - entry("ELN_PREVIEW", ""), - entry("EXPERIMENT_PROTOCOL", "sops"), - entry("EXPERIMENT_RESULT", "documents"), - entry("HISTOLOGICAL_SLIDE", "data_files"), - entry("IB_DATA", "data_files"), - entry("LUMINEX_DATA", "data_files"), - entry("MS_DATA_ANALYZED", "data_files"), - entry("MS_DATA_RAW", "data_files"), - entry("OTHER_DATA", "data_files"), - entry("PROCESSED_DATA", "data_files"), - entry("PUBLICATION_DATA", "publications"), - entry("QPCR_DATA", "data_files"), - entry("RAW_DATA", "data_files"), - entry("SOURCE_CODE", "documents"), - entry("TEST_CONT", ""), - entry("TEST_DAT", ""), - entry("UNKNOWN", "data_files") - ); - public ISASampleType translate(SampleType sampleType) { - SampleAttribute titleAttribute = new SampleAttribute(DEFAULT_TRANSFERRED_SAMPLE_TITLE, + SampleAttribute titleAttribute = new SampleAttribute( + App.configProperties.get("seek_openbis_sample_title"), dataTypeToAttributeType.get(DataType.VARCHAR), true, false); ISASampleType type = new ISASampleType(sampleType.getCode(), titleAttribute, DEFAULT_PROJECT_ID); @@ -147,15 +154,17 @@ public String dataFormatAnnotationForExtension(String fileExtension) { } public SeekStructure translate(OpenbisExperimentWithDescendants experiment, - Map sampleTypesToIds, Set blacklist) throws URISyntaxException { + Map sampleTypesToIds, Set blacklist, boolean transferData) + throws URISyntaxException { Experiment exp = experiment.getExperiment(); String expType = exp.getType().getCode(); String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; - ISAAssay assay = new ISAAssay(title, DEFAULT_STUDY_ID, experimentTypeToAssayClass.get(expType), - new URI(experimentTypeToAssayType.get(expType)));//TODO + ISAAssay assay = new ISAAssay(title, STUDY_ID, experimentTypeToAssayClass.get(expType), + new URI(experimentTypeToAssayType.get(expType))); + + SeekStructure result = new SeekStructure(assay, exp.getIdentifier().getIdentifier()); - List samples = new ArrayList<>(); for(Sample sample : experiment.getSamples()) { SampleType sampleType = sample.getType(); @@ -180,26 +189,26 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, attributes.put(typeCodesToNames.get(code), value); } - attributes.put(DEFAULT_TRANSFERRED_SAMPLE_TITLE, sample.getIdentifier().getIdentifier()); + String sampleID = sample.getIdentifier().getIdentifier(); + attributes.put(App.configProperties.get("seek_openbis_sample_title"), sampleID); String sampleTypeId = sampleTypesToIds.get(sampleType.getCode()); - ISASample isaSample = new ISASample(sample.getPermId().getPermId(), attributes, sampleTypeId, - Collections.singletonList(DEFAULT_PROJECT_ID)); - samples.add(isaSample); + ISASample isaSample = new ISASample(sample.getIdentifier().getIdentifier(), attributes, + sampleTypeId, Collections.singletonList(DEFAULT_PROJECT_ID)); + result.addSample(isaSample, sampleID); } - Map isaToOpenBISFile = new HashMap<>(); - //create ISA files for assets. If actual data is uploaded is determined later based upon flag + //create ISA files for assets. If actual data is to be uploaded is determined later based on flag for(DatasetWithProperties dataset : experiment.getDatasets()) { String permID = dataset.getCode(); if(!blacklist.contains(permID)) { for(DataSetFile file : experiment.getFilesForDataset(permID)) { String datasetType = getDatasetTypeOfFile(file, experiment.getDatasets()); - datasetFileToSeekAsset(file, datasetType) - .ifPresent(seekAsset -> isaToOpenBISFile.put(seekAsset, file)); + datasetFileToSeekAsset(file, datasetType, transferData) + .ifPresent(seekAsset -> result.addAsset(seekAsset, file)); } } } - return new SeekStructure(assay, samples, isaToOpenBISFile); + return result; } private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { @@ -217,14 +226,20 @@ private String getDatasetTypeOfFile(DataSetFile file, List datasetFileToSeekAsset(DataSetFile file, String datasetType) { + private Optional datasetFileToSeekAsset(DataSetFile file, String datasetType, + boolean transferData) { if (!file.getPath().isBlank() && !file.isDirectory()) { File f = new File(file.getPath()); - String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode + ": " + f.getName(); + String datasetPermID = file.getDataSetPermId().toString(); + String assetName = datasetPermID + ": " + f.getName(); String assetType = assetForDatasetType(datasetType); GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), - Arrays.asList(DEFAULT_PROJECT_ID)); + Arrays.asList(DEFAULT_PROJECT_ID), file.getFileLength()); + //reference the openbis dataset in the description - if transferData flag is false, this will + //also add a second link instead of the (non-functional) download link to a non-existent blob. + //it seems that directly linking to files needs an open session, so we only set the dataset for now + String datasetLink = generateOpenBISLinkFromPermID("DATA_SET", datasetPermID); + isaFile.setDatasetLink(datasetLink, transferData); String fileExtension = f.getName().substring(f.getName().lastIndexOf(".") + 1); String annotation = dataFormatAnnotationForExtension(fileExtension); if (annotation != null) { @@ -235,4 +250,11 @@ private Optional datasetFileToSeekAsset(DataSetFile file, Stri return Optional.empty(); } + public void setDefaultStudy(String studyID) { + this.STUDY_ID = studyID; + } + + public void setDefaultInvestigation(String investigationID) { + this.INVESTIGATION_ID = investigationID; + } } diff --git a/src/main/java/life/qbic/model/SampleInformation.java b/src/main/java/life/qbic/model/SampleInformation.java new file mode 100644 index 0000000..73a264b --- /dev/null +++ b/src/main/java/life/qbic/model/SampleInformation.java @@ -0,0 +1,28 @@ +package life.qbic.model; + +import java.util.Map; + +public class SampleInformation { + + private String seekID; + private String openBisIdentifier; + private Map attributes; + + public SampleInformation(String sampleID, String title, Map attributesMap) { + this.seekID = sampleID; + this.openBisIdentifier = title; + this.attributes = attributesMap; + } + + public String getSeekID() { + return seekID; + } + + public Map getAttributes() { + return attributes; + } + + public String getOpenBisIdentifier() { + return openBisIdentifier; + } +} diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 7e5d7ed..386e494 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -6,21 +6,22 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.update.DataSetUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSearchCriteria; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.property.PropertyAssignment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.update.ExperimentUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleTypeFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleTypeSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.update.SampleUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria; @@ -46,13 +47,15 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import life.qbic.App; import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.SampleTypeConnection; import life.qbic.model.SampleTypesAndMaterials; +import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -61,9 +64,10 @@ public class OpenbisConnector { private static final Logger LOG = LogManager.getLogger(OpenbisConnector.class); private final OpenBIS openBIS; - //public final static Pattern experimentIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - //public final static Pattern sampleIdPattern = Pattern.compile("\\/[A-Za-z0-9]+\\/[A-Za-z0-9]+"); - public static Pattern datasetCodePattern = Pattern.compile("[0-9]^{17}-[0-9]+"); + public static Pattern datasetCodePattern = Pattern.compile("[0-9]{17}-[0-9]+"); + public final String EXPERIMENT_LINK_PROPERTY = "EXPERIMENT_NAME"; + public final String SAMPLE_LINK_PROPERTY = "experimentLink"; + public final String DATASET_LINK_PROPERTY = "experimentLink"; public OpenbisConnector(OpenBIS authentication) { this.openBIS = authentication; @@ -139,10 +143,10 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } - public File downloadDataset(String targetPath, String datasetID) { + public File downloadDataset(String targetPath, String datasetID, String filePath) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), - ""); + filePath); // Setting recursive flag to true will return both subfolders and files options.setRecursive(true); @@ -171,7 +175,7 @@ public File downloadDataset(String targetPath, String datasetID) { } } } - return new File(targetPath); + return new File(targetPath, filePath.replace("original/","")); } public InputStream streamDataset(String datasetCode, String filePath) { @@ -232,20 +236,22 @@ public Map queryFullSampleHierarchy(List return hierarchy; } - private Optional getPropertyFromSampleHierarchy(String propertyName, List samples) { + private Set getPropertiesFromSampleHierarchy(String propertyName, List samples, + Set foundProperties) { for(Sample s : samples) { if(s.getProperties().containsKey(propertyName)) { - return Optional.of(s.getProperties().get(propertyName)); + foundProperties.add(s.getProperties().get(propertyName)); + return foundProperties; } - return getPropertyFromSampleHierarchy(propertyName, s.getParents()); + return getPropertiesFromSampleHierarchy(propertyName, s.getParents(), foundProperties); } - return Optional.empty(); + return foundProperties; } - public Optional findPropertyInSampleHierarchy(String propertyName, + public Set findPropertiesInSampleHierarchy(String propertyName, ExperimentIdentifier experimentId) { - return getPropertyFromSampleHierarchy(propertyName, - getSamplesWithAncestorsOfExperiment(experimentId)); + return getPropertiesFromSampleHierarchy(propertyName, + getSamplesWithAncestorsOfExperiment(experimentId), new HashSet<>()); } public Map> getExperimentsBySpace(List spaces) { @@ -377,18 +383,22 @@ private String getSpaceFromSampleOrExperiment(DataSet d) { } private List getSamplesWithAncestorsOfExperiment(ExperimentIdentifier experimentId) { - SampleFetchOptions allProps = new SampleFetchOptions(); - allProps.withType(); - allProps.withProperties(); - SampleFetchOptions withAncestors = new SampleFetchOptions(); - withAncestors.withParentsUsing(allProps); - withAncestors.withProperties(); - withAncestors.withType(); + int numberOfFetchedLevels = 10; + SampleFetchOptions previousLevel = null; + for(int i = 0; i < numberOfFetchedLevels; i++) { + SampleFetchOptions withAncestors = new SampleFetchOptions(); + withAncestors.withProperties(); + withAncestors.withType(); + if (previousLevel != null) { + withAncestors.withParentsUsing(previousLevel); + } + previousLevel = withAncestors; + } SampleSearchCriteria criteria = new SampleSearchCriteria(); criteria.withExperiment().withId().thatEquals(experimentId); - return openBIS.searchSamples(criteria, withAncestors).getObjects(); + return openBIS.searchSamples(criteria, previousLevel).getObjects(); } public List findDataSets(List codes) { @@ -491,4 +501,86 @@ public SampleTypesAndMaterials getSampleTypesWithMaterials() { } return new SampleTypesAndMaterials(sampleTypes, sampleTypesAsMaterials); } + + public void createSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { + Pair experimentInfo = postRegistrationInformation.getExperimentIDWithEndpoint(); + ExperimentIdentifier id = new ExperimentIdentifier(experimentInfo.getLeft()); + String endpoint = experimentInfo.getRight(); + Map props = new HashMap<>(); + props.put(EXPERIMENT_LINK_PROPERTY, endpoint); + updateExperimentProperties(id, props, false); + } + + public void updateSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { + } + + private void updateExperimentProperties(ExperimentIdentifier id, Map properties, + boolean overwrite) { + ExperimentUpdate update = new ExperimentUpdate(); + update.setExperimentId(id); + if(overwrite) { + update.setProperties(properties); + } else { + ExperimentFetchOptions options = new ExperimentFetchOptions(); + options.withProperties(); + Experiment oldExp = openBIS.getExperiments(Arrays.asList(id), options).get(id); + for(String property : properties.keySet()) { + String newValue = properties.get(property); + String oldValue = oldExp.getProperty(property); + if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + update.setProperty(property, newValue); + } else if(!newValue.isBlank()) { + update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + } + } + } + openBIS.updateExperiments(Arrays.asList(update)); + } + + private void updateSampleProperties(SampleIdentifier id, Map properties, + boolean overwrite) { + SampleUpdate update = new SampleUpdate(); + update.setSampleId(id); + if(overwrite) { + update.setProperties(properties); + } else { + SampleFetchOptions options = new SampleFetchOptions(); + options.withProperties(); + Sample oldSample = openBIS.getSamples(Arrays.asList(id), options).get(id); + for(String property : properties.keySet()) { + String newValue = properties.get(property); + String oldValue = oldSample.getProperty(property); + if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + update.setProperty(property, newValue); + } else { + update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + } + } + } + openBIS.updateSamples(Arrays.asList(update)); + } + + private void updateDatasetProperties(DataSetPermId id, Map properties, + boolean overwrite) { + DataSetUpdate update = new DataSetUpdate(); + update.setDataSetId(id); + if(overwrite) { + update.setProperties(properties); + } else { + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withProperties(); + DataSet oldDataset = openBIS.getDataSets(Arrays.asList(id), options).get(id); + for(String property : properties.keySet()) { + String newValue = properties.get(property); + String oldValue = oldDataset.getProperty(property); + if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + update.setProperty(property, newValue); + } else { + update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + } + } + } + openBIS.updateDataSets(Arrays.asList(update)); + } + } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 16fb0ec..6214c7e 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -16,23 +16,33 @@ import java.net.http.HttpResponse.BodyHandlers; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Supplier; +import java.util.regex.Matcher; +import javax.xml.parsers.ParserConfigurationException; import life.qbic.model.AssayWithQueuedAssets; +import life.qbic.model.AssetInformation; import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.SampleInformation; import life.qbic.model.isa.SeekStructure; import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.ISAAssay; import life.qbic.model.isa.ISASample; import life.qbic.model.isa.ISASampleType; import life.qbic.model.isa.ISAStudy; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.client.utils.URIBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.xml.sax.SAXException; public class SEEKConnector { @@ -42,20 +52,29 @@ public class SEEKConnector { private OpenbisSeekTranslator translator; private final String DEFAULT_PROJECT_ID; - public SEEKConnector(String apiURL, byte[] httpCredentials, String openBISBaseURL, - String defaultProjectTitle, String defaultStudyTitle) throws URISyntaxException, IOException, - InterruptedException { - this.apiURL = apiURL; + public SEEKConnector(String seekURL, byte[] httpCredentials, String openBISBaseURL, + String defaultProjectTitle) throws URISyntaxException, IOException, + InterruptedException, ParserConfigurationException, SAXException { + this.apiURL = seekURL; this.credentials = httpCredentials; Optional projectID = getProjectWithTitle(defaultProjectTitle); - if(projectID.isEmpty()) { - throw new RuntimeException("Failed to find project with title: " + defaultProjectTitle+". " + if (projectID.isEmpty()) { + throw new RuntimeException("Failed to find project with title: " + defaultProjectTitle + ". " + "Please provide an existing default project."); } DEFAULT_PROJECT_ID = projectID.get(); + translator = new OpenbisSeekTranslator(openBISBaseURL, DEFAULT_PROJECT_ID); + } + + public void setDefaultInvestigation(String investigationTitle) + throws URISyntaxException, IOException, InterruptedException { + translator.setDefaultInvestigation(searchNodeWithTitle("investigations", + investigationTitle)); + } - translator = new OpenbisSeekTranslator(openBISBaseURL, DEFAULT_PROJECT_ID, - searchNodeWithTitle("studies", defaultStudyTitle)); + public void setDefaultStudy(String studyTitle) + throws URISyntaxException, IOException, InterruptedException { + translator.setDefaultStudy(searchNodeWithTitle("studies", studyTitle)); } /** @@ -92,6 +111,23 @@ private Optional getProjectWithTitle(String projectTitle) return Optional.empty(); } + public String addStudy(ISAStudy assay) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/studies"; + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return idNode.asText(); + } + public String addAssay(ISAAssay assay) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/assays"; @@ -126,6 +162,15 @@ public String createStudy(ISAStudy study) return idNode.asText(); } + private HttpRequest buildAuthorizedPATCHRequest(String endpoint, String body) throws URISyntaxException { + return HttpRequest.newBuilder() + .uri(new URI(endpoint)) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .method("PATCH", HttpRequest.BodyPublishers.ofString(body)).build(); + } + private HttpRequest buildAuthorizedPOSTRequest(String endpoint, String body) throws URISyntaxException { return HttpRequest.newBuilder() .uri(new URI(endpoint)) @@ -160,13 +205,9 @@ public void printAttributeTypes() throws URISyntaxException, IOException, Interr .send(request, BodyHandlers.ofString()); System.err.println(response.body()); } + /* - -datatype by extension --assay equals experiment --investigation: pre-created in SEEK? --study: project -patient id should be linked somehow, maybe gender? --flexible object type to sample type? */ public void deleteSampleType(String id) throws URISyntaxException, IOException, @@ -188,8 +229,8 @@ public void deleteSampleType(String id) throws URISyntaxException, IOException, } } - public String createSampleType(ISASampleType sampleType) throws URISyntaxException, IOException, - InterruptedException { + public String createSampleType(ISASampleType sampleType) + throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/sample_types"; HttpResponse response = HttpClient.newBuilder().build() @@ -206,6 +247,25 @@ public String createSampleType(ISASampleType sampleType) throws URISyntaxExcepti return idNode.asText(); } + public String updateSample(ISASample isaSample, String sampleID) throws URISyntaxException, IOException, + InterruptedException { + String endpoint = apiURL+"/samples/"+sampleID; + isaSample.setSampleID(sampleID); + + HttpResponse response = HttpClient.newBuilder().build() + .send(buildAuthorizedPATCHRequest(endpoint, isaSample.toJson()), + BodyHandlers.ofString()); + + if(response.statusCode()!=200) { + System.err.println(response.body()); + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + JsonNode rootNode = new ObjectMapper().readTree(response.body()); + JsonNode idNode = rootNode.path("data").path("id"); + + return endpoint+"/"+idNode.asText(); + } + public String createSample(ISASample isaSample) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/samples"; @@ -221,10 +281,10 @@ public String createSample(ISASample isaSample) throws URISyntaxException, IOExc JsonNode rootNode = new ObjectMapper().readTree(response.body()); JsonNode idNode = rootNode.path("data").path("id"); - return idNode.asText(); + return endpoint+"/"+idNode.asText(); } - private AssetToUpload createDataFileAsset(String datasetCode, GenericSeekAsset data) + private AssetToUpload createAsset(String datasetCode, GenericSeekAsset data) throws IOException, URISyntaxException, InterruptedException { String endpoint = apiURL+"/"+data.getType(); @@ -242,16 +302,14 @@ private AssetToUpload createDataFileAsset(String datasetCode, GenericSeekAsset d .path("attributes") .path("content_blobs") .path(0).path("link"); - return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode); + return new AssetToUpload(idNode.asText(), data.getFileName(), datasetCode, data.fileSizeInBytes()); } - @Deprecated - private void uploadFileContent(String assetType, String assetID, String blobID, String file) + public String uploadFileContent(String blobEndpoint, String file) throws URISyntaxException, IOException, InterruptedException { - String endpoint = apiURL+"/"+assetType+"/"+assetID+"/content_blobs/"+blobID; HttpRequest request = HttpRequest.newBuilder() - .uri(new URI(endpoint)) + .uri(new URI(blobEndpoint)) .headers("Content-Type", "application/octet-stream") .headers("Accept", "application/octet-stream") .headers("Authorization", "Basic " + new String(credentials)) @@ -264,6 +322,7 @@ private void uploadFileContent(String assetType, String assetID, String blobID, System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } + return blobEndpointToAssetURL(blobEndpoint); } public String uploadStreamContent(String blobEndpoint, @@ -287,11 +346,14 @@ public String uploadStreamContent(String blobEndpoint, System.err.println(response.body()); throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } else { - String fileURL = blobEndpoint.split("content_blobs")[0]; - return fileURL; + return blobEndpointToAssetURL(blobEndpoint); } } + private String blobEndpointToAssetURL(String blobEndpoint) { + return blobEndpoint.split("content_blobs")[0]; + } + public boolean endPointExists(String endpoint) throws URISyntaxException, IOException, InterruptedException { HttpRequest request = HttpRequest.newBuilder() @@ -305,59 +367,6 @@ public boolean endPointExists(String endpoint) return response.statusCode() == 200; } - /** - * Creates an optional asset for a file from an openBIS dataset. Folders are ignored. - * @param file - * @param assetType - * @param assays - * @return - * @throws IOException - * @throws URISyntaxException - * @throws InterruptedException - */ - public Optional createAssetForFile(DataSetFile file, String assetType, - List assays) - throws IOException, URISyntaxException, InterruptedException { - if(!file.getPath().isBlank() && !file.isDirectory()) { - File f = new File(file.getPath()); - String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? - GenericSeekAsset isaFile = new GenericSeekAsset(assetType, assetName, file.getPath(), - Arrays.asList(DEFAULT_PROJECT_ID)); - isaFile.withAssays(assays); - String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); - String annotation = translator.dataFormatAnnotationForExtension(fileExtension); - if(annotation!=null) { - isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); - } - return Optional.of(createDataFileAsset(datasetCode, isaFile)); - } - return Optional.empty(); - } - - public List createAssets(List filesInDataset, - List assays) - throws IOException, URISyntaxException, InterruptedException { - List result = new ArrayList<>(); - for(DataSetFile file : filesInDataset) { - if(!file.getPath().isBlank() && !file.isDirectory()) { - File f = new File(file.getPath()); - String datasetCode = file.getDataSetPermId().toString(); - String assetName = datasetCode+": "+f.getName();//TODO what do we want to call the asset? - GenericSeekAsset isaFile = new GenericSeekAsset("data_files", assetName, file.getPath(), - Arrays.asList(DEFAULT_PROJECT_ID)); - isaFile.withAssays(assays); - String fileExtension = f.getName().substring(f.getName().lastIndexOf(".")+1); - String annotation = translator.dataFormatAnnotationForExtension(fileExtension); - if(annotation!=null) { - isaFile.withDataFormatAnnotations(Arrays.asList(annotation)); - } - result.add(createDataFileAsset(datasetCode, isaFile)); - } - } - return result; - } - /** * Creates * @param isaToOpenBISFile @@ -373,7 +382,7 @@ public List createAssetsForAssays(Map result = new ArrayList<>(); for (GenericSeekAsset isaFile : isaToOpenBISFile.keySet()) { isaFile.withAssays(assays); - result.add(createDataFileAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), + result.add(createAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), isaFile)); } return result; @@ -427,11 +436,30 @@ private Map parseSampleTypesJSON(String json) throws JsonProcess return typesToIDs; } - private String searchNodeWithTitle(String nodeType, String title) + public boolean sampleTypeExists(String typeCode) + throws URISyntaxException, IOException, InterruptedException { + JsonNode result = genericSearch("sample_types", typeCode); + JsonNode hits = result.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if (hit.get("attributes").get("title").asText().equals(typeCode)) { + return true; + } + } + return false; + } + + /** + * Performs a generic search and returns the response in JSON format + * @param nodeType the type of SEEK node to search for + * @param searchTerm the term to search for + * @return JsonNode of the server's response + */ + private JsonNode genericSearch(String nodeType, String searchTerm) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/search"; URIBuilder builder = new URIBuilder(endpoint); - builder.setParameter("q", title).setParameter("search_type", nodeType); + builder.setParameter("q", searchTerm).setParameter("search_type", nodeType); HttpRequest request = HttpRequest.newBuilder() .uri(builder.build()) @@ -441,22 +469,26 @@ private String searchNodeWithTitle(String nodeType, String title) .GET().build(); HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); - System.err.println("searching for: "+title+" ("+nodeType+")"); if(response.statusCode() == 200) { - JsonNode rootNode = new ObjectMapper().readTree(response.body()); - JsonNode hits = rootNode.path("data"); - for (Iterator it = hits.elements(); it.hasNext(); ) { - JsonNode hit = it.next(); - if(hit.get("attributes").get("title").asText().equals(title)) { - return hit.get("id").asText(); - } - } - throw new RuntimeException("Matching "+nodeType+" title was not found : " + title); + return new ObjectMapper().readTree(response.body()); } else { throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } } + private String searchNodeWithTitle(String nodeType, String title) + throws URISyntaxException, IOException, InterruptedException { + JsonNode result = genericSearch(nodeType, title); + JsonNode hits = result.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + if (hit.get("attributes").get("title").asText().equals(title)) { + return hit.get("id").asText(); + } + } + throw new RuntimeException("Matching " + nodeType + " title was not found : " + title); + } + /** * Searches for assays containing a search term and returns a list of found assay ids * @param searchTerm the search term that should be in the assay properties - e.g. an openBIS id @@ -467,9 +499,147 @@ private String searchNodeWithTitle(String nodeType, String title) */ public List searchAssaysContainingKeyword(String searchTerm) throws URISyntaxException, IOException, InterruptedException { - String endpoint = apiURL+"/search/"; + + JsonNode result = genericSearch("assays", "*"+searchTerm+"*"); + + JsonNode hits = result.path("data"); + List assayIDs = new ArrayList<>(); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assayIDs.add(hit.get("id").asText()); + } + return assayIDs; + } + + /** + * Updates information of an existing assay, its samples and attached assets. Missing samples and + * assets are created, but nothing missing from the new structure is deleted from SEEK. + * @param nodeWithChildren the translated Seek structure as it should be once the update is done + * @param assayID the assay id of the existing assay, that should be compared to the new structure + * @return information necessary to make post registration updates in openBIS and upload missing + * data to newly created assets. In the case of the update use case, only newly created objects + * will be contained in the return object. + */ + public SeekStructurePostRegistrationInformation updateNode(SeekStructure nodeWithChildren, + String assayID) throws URISyntaxException, IOException, InterruptedException { + JsonNode assayData = fetchAssayData(assayID).get("data"); + Map sampleInfos = collectSampleInformation(assayData); + + // compare samples + Map newSamplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + + List samplesToCreate = new ArrayList<>(); + for (ISASample newSample : newSamplesWithReferences.keySet()) { + String openBisID = newSamplesWithReferences.get(newSample); + SampleInformation existingSample = sampleInfos.get(openBisID); + if (existingSample == null) { + samplesToCreate.add(newSample); + System.out.printf("%s not found in SEEK. It will be created.%n", openBisID); + } else { + Map newAttributes = newSample.fetchCopyOfAttributeMap(); + for (String key : newAttributes.keySet()) { + Object newValue = newAttributes.get(key); + Object oldValue = existingSample.getAttributes().get(key); + + boolean oldEmpty = oldValue == null || oldValue.toString().isEmpty(); + boolean newEmpty = newValue == null || newValue.toString().isEmpty(); + if ((!oldEmpty && !newEmpty) && !newValue.equals(oldValue)) { + System.out.printf("Mismatch found in attributes of %s. Sample will be updated.%n", + openBisID); + newSample.setAssayIDs(List.of(assayID)); + updateSample(newSample, existingSample.getSeekID()); + } + } + } + } + + // compare assets + Map assetInfos = collectAssetInformation(assayData); + Map newAssetsToFiles = nodeWithChildren.getISAFileToDatasetFiles(); + + List assetsToCreate = new ArrayList<>(); + for (GenericSeekAsset newAsset : newAssetsToFiles.keySet()) { + DataSetFile file = newAssetsToFiles.get(newAsset); + String newPermId = file.getDataSetPermId().getPermId(); + if (!assetInfos.containsKey(newPermId)) { + assetsToCreate.add(newAsset); + System.out.printf("Assets with Dataset PermId %s not found in SEEK. File %s from this " + + "Dataset will be created.%n", newPermId, newAsset.getFileName()); + } + } + Map sampleIDsWithEndpoints = new HashMap<>(); + for (ISASample sample : samplesToCreate) { + sample.setAssayIDs(Collections.singletonList(assayID)); + String sampleEndpoint = createSample(sample); + sampleIDsWithEndpoints.put(newSamplesWithReferences.get(sample), sampleEndpoint); + } + List assetsToUpload = new ArrayList<>(); + for (GenericSeekAsset asset : assetsToCreate) { + asset.withAssays(Collections.singletonList(assayID)); + assetsToUpload.add(createAsset(newAssetsToFiles.get(asset).getDataSetPermId().getPermId(), + asset)); + } + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for (AssetToUpload asset : assetsToUpload) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if (datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + + String assayEndpoint = apiURL + "/assays/" + assayID; + AssayWithQueuedAssets assayWithQueuedAssets = + new AssayWithQueuedAssets(assayEndpoint, assetsToUpload); + String expID = nodeWithChildren.getAssayWithOpenBISReference().getRight(); + Pair experimentIDWithEndpoint = new ImmutablePair<>(expID, assayEndpoint); + + return new SeekStructurePostRegistrationInformation(assayWithQueuedAssets, + experimentIDWithEndpoint, sampleIDsWithEndpoints, datasetIDsWithEndpoints); + } + + private Map collectAssetInformation(JsonNode assayData) + throws URISyntaxException, IOException, InterruptedException { + List assetTypes = new ArrayList<>(Arrays.asList("data_files", "models", "sops", + "documents", "publications")); + Map assets = new HashMap<>(); + JsonNode relationships = assayData.get("relationships"); + for(String type : assetTypes) { + for (Iterator it = relationships.get(type).get("data").elements(); it.hasNext(); ) { + String assetID = it.next().get("id").asText(); + AssetInformation assetInfo = fetchAssetInformation(assetID, type); + if(assetInfo.getOpenbisPermId()!=null) { + assets.put(assetInfo.getOpenbisPermId(), assetInfo); + } else { + System.out.printf("No Dataset permID found for existing %s %s (id: %s)%n" + + "This asset will be treated as if it would not exist in the update.%n", + type, assetInfo.getTitle(), assetID); + } + } + } + return assets; + } + + private Map collectSampleInformation(JsonNode assayData) + throws URISyntaxException, IOException, InterruptedException { + Map samples = new HashMap<>(); + JsonNode relationships = assayData.get("relationships"); + for (Iterator it = relationships.get("samples").get("data").elements(); it.hasNext(); ) { + String sampleID = it.next().get("id").asText(); + SampleInformation info = fetchSampleInformation(sampleID); + samples.put(info.getOpenBisIdentifier(), info); + } + return samples; + } + + private AssetInformation fetchAssetInformation(String assetID, String assetType) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/"+assetType+"/"+assetID; URIBuilder builder = new URIBuilder(endpoint); - builder.setParameter("q", searchTerm).setParameter("type", "assays"); HttpRequest request = HttpRequest.newBuilder() .uri(builder.build()) @@ -480,36 +650,113 @@ public List searchAssaysContainingKeyword(String searchTerm) HttpResponse response = HttpClient.newBuilder().build() .send(request, BodyHandlers.ofString()); if(response.statusCode() == 200) { - JsonNode rootNode = new ObjectMapper().readTree(response.body()); - JsonNode hits = rootNode.path("data"); - List assayIDs = new ArrayList<>(); - for (Iterator it = hits.elements(); it.hasNext(); ) { - JsonNode hit = it.next(); - assayIDs.add(hit.get("id").asText()); + JsonNode attributes = new ObjectMapper().readTree(response.body()).get("data").get("attributes"); + String title = attributes.get("title").asText(); + String description = attributes.get("description").asText(); + AssetInformation result = new AssetInformation(assetID, assetType, title, description); + Optional permID = tryParseDatasetPermID(title); + if(permID.isPresent()) { + result.setOpenbisPermId(permID.get()); + } else { + tryParseDatasetPermID(description).ifPresent(result::setOpenbisPermId); } - return assayIDs; + return result; + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } + } + + private Optional tryParseDatasetPermID(String input) { + Matcher titleMatcher = OpenbisConnector.datasetCodePattern.matcher(input); + if(titleMatcher.find()) { + return Optional.of(titleMatcher.group()); + } + return Optional.empty(); + } + + private SampleInformation fetchSampleInformation(String sampleID) throws URISyntaxException, + IOException, InterruptedException { + String endpoint = apiURL+"/samples/"+sampleID; + URIBuilder builder = new URIBuilder(endpoint); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + JsonNode attributeNode = new ObjectMapper().readTree(response.body()).get("data").get("attributes"); + //title is openbis identifier - this is also added to attribute_map under the name: + //App.configProperties.get("seek_openbis_sample_title"); + String openBisId = attributeNode.get("title").asText(); + Map attributesMap = new ObjectMapper() + .convertValue(attributeNode.get("attribute_map"), Map.class); + return new SampleInformation(sampleID, openBisId, attributesMap); } else { throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } } - public String updateNode(SeekStructure nodeWithChildren, String assayID, boolean transferData) { - //updateAssay(nodeWithChildren.getAssay()); - return apiURL+"/assays/"+assayID; + private JsonNode fetchAssayData(String assayID) + throws URISyntaxException, IOException, InterruptedException { + String endpoint = apiURL+"/assays/"+assayID; + URIBuilder builder = new URIBuilder(endpoint); + + HttpRequest request = HttpRequest.newBuilder() + .uri(builder.build()) + .headers("Content-Type", "application/json") + .headers("Accept", "application/json") + .headers("Authorization", "Basic " + new String(credentials)) + .GET().build(); + HttpResponse response = HttpClient.newBuilder().build() + .send(request, BodyHandlers.ofString()); + if(response.statusCode() == 200) { + return new ObjectMapper().readTree(response.body()); + } else { + throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); + } } - public AssayWithQueuedAssets createNode(SeekStructure nodeWithChildren, boolean transferData) + public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - String assayID = addAssay(nodeWithChildren.getAssay()); - for(ISASample sample : nodeWithChildren.getSamples()) { - sample.setAssayIDs(Arrays.asList(assayID)); - createSample(sample); + + Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference(); + + String assayID = addAssay(assayIDPair.getKey()); + String assayEndpoint = apiURL+"/assays/"+assayID; + Pair experimentIDWithEndpoint = + new ImmutablePair<>(assayIDPair.getValue(), assayEndpoint); + + Map sampleIDsWithEndpoints = new HashMap<>(); + Map samplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + for(ISASample sample : samplesWithReferences.keySet()) { + sample.setAssayIDs(Collections.singletonList(assayID)); + String sampleEndpoint = createSample(sample); + sampleIDsWithEndpoints.put(samplesWithReferences.get(sample), sampleEndpoint); } Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); - return new AssayWithQueuedAssets(apiURL+"/assays/"+assayID, - createAssetsForAssays(isaToFileMap, Arrays.asList(assayID))); + AssayWithQueuedAssets assayWithQueuedAssets = new AssayWithQueuedAssets(assayEndpoint, + createAssetsForAssays(isaToFileMap, Collections.singletonList(assayID))); + + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for(AssetToUpload asset : assayWithQueuedAssets.getAssets()) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if(datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + return new SeekStructurePostRegistrationInformation(assayWithQueuedAssets, + experimentIDWithEndpoint, sampleIDsWithEndpoints, datasetIDsWithEndpoints); } public OpenbisSeekTranslator getTranslator() { @@ -521,11 +768,18 @@ public static class AssetToUpload { private final String blobEndpoint; private final String filePath; private final String openBISDataSetCode; + private final long fileSizeInBytes; - public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode) { + public AssetToUpload(String blobEndpoint, String filePath, String openBISDataSetCode, + long fileSizeInBytes) { this.blobEndpoint = blobEndpoint; this.filePath = filePath; this.openBISDataSetCode = openBISDataSetCode; + this.fileSizeInBytes = fileSizeInBytes; + } + + public long getFileSizeInBytes() { + return fileSizeInBytes; } public String getFilePath() { @@ -540,4 +794,38 @@ public String getDataSetCode() { return openBISDataSetCode; } } + + public class SeekStructurePostRegistrationInformation { + + private final AssayWithQueuedAssets assayWithQueuedAssets; + private final Pair experimentIDWithEndpoint; + private final Map sampleIDsWithEndpoints; + private final Map> datasetIDsWithEndpoints; + + public SeekStructurePostRegistrationInformation(AssayWithQueuedAssets assayWithQueuedAssets, + Pair experimentIDWithEndpoint, Map sampleIDsWithEndpoints, + Map> datasetIDsWithEndpoints) { + this.assayWithQueuedAssets = assayWithQueuedAssets; + this.experimentIDWithEndpoint = experimentIDWithEndpoint; + this.sampleIDsWithEndpoints = sampleIDsWithEndpoints; + this.datasetIDsWithEndpoints = datasetIDsWithEndpoints; + } + + public AssayWithQueuedAssets getAssayWithQueuedAssets() { + return assayWithQueuedAssets; + } + + public Pair getExperimentIDWithEndpoint() { + return experimentIDWithEndpoint; + } + + public Map getSampleIDsWithEndpoints() { + return sampleIDsWithEndpoints; + } + + public Map> getDatasetIDsWithEndpoints() { + return datasetIDsWithEndpoints; + } + + } } diff --git a/src/main/java/life/qbic/model/isa/AbstractISAObject.java b/src/main/java/life/qbic/model/isa/AbstractISAObject.java index 2d68900..c8f952c 100644 --- a/src/main/java/life/qbic/model/isa/AbstractISAObject.java +++ b/src/main/java/life/qbic/model/isa/AbstractISAObject.java @@ -5,6 +5,9 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.module.SimpleModule; +/** + * Used to create the outer "data" node of all SEEK json objects. + */ public abstract class AbstractISAObject { public String toJson(SimpleModule module) throws JsonProcessingException { diff --git a/src/main/java/life/qbic/model/isa/GenericSeekAsset.java b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java index 34b4fef..28bff90 100644 --- a/src/main/java/life/qbic/model/isa/GenericSeekAsset.java +++ b/src/main/java/life/qbic/model/isa/GenericSeekAsset.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Model class for Seek assets. Contains all mandatory and some optional properties and attributes @@ -22,11 +23,18 @@ public class GenericSeekAsset extends AbstractISAObject { private Attributes attributes; private Relationships relationships; private String assetType; + private long fileSizeInBytes; - public GenericSeekAsset(String assetType, String title, String fileName, List projectIds) { + public GenericSeekAsset(String assetType, String title, String fileName, List projectIds, + long fileSizeInBytes) { this.assetType = assetType; this.attributes = new Attributes(title, fileName); this.relationships = new Relationships(projectIds); + this.fileSizeInBytes = fileSizeInBytes; + } + + public long fileSizeInBytes() { + return fileSizeInBytes; } public GenericSeekAsset withOtherCreators(String otherCreators) { @@ -66,6 +74,13 @@ public String getFileName() { return attributes.getContent_blobs().get(0).getOriginal_filename(); } + public void setDatasetLink(String dataSetLink, boolean transferDate) { + attributes.description = "This asset was imported from openBIS: "+dataSetLink; + if(!transferDate) { + attributes.setExternalLinkToData(dataSetLink); + } + } + private class Relationships { private List projects; @@ -125,14 +140,25 @@ private void generateListJSON(JsonGenerator generator, String name, List private class Attributes { private String title; + private String description; private List contentBlobs = new ArrayList<>(); private String otherCreators = ""; private List dataFormatAnnotations = new ArrayList<>(); - public Attributes(String title, String fileName) { this.title = title; - this.contentBlobs.add(new ContentBlob(fileName)); + ContentBlob blob = new ContentBlob(fileName); + this.contentBlobs.add(blob); + } + + public void setExternalLinkToData(String dataSetLink) { + for(ContentBlob blob : contentBlobs) { + blob.setURL(dataSetLink); + } + } + + public String getDescription() { + return description; } public String getTitle() { @@ -178,6 +204,7 @@ private class ContentBlob { private String originalFilename; private String contentType; + private String url; public ContentBlob(String fileName) { this.originalFilename = fileName; @@ -193,7 +220,36 @@ public String getContent_type() { public String getOriginal_filename() { return originalFilename; } + + public void setURL(String dataSetLink) { + this.url = dataSetLink; + } + + public String getUrl() { + return url; + } + } + } + + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof GenericSeekAsset)) { + return false; } + + GenericSeekAsset that = (GenericSeekAsset) o; + return Objects.equals(attributes, that.attributes) && Objects.equals( + relationships, that.relationships) && Objects.equals(assetType, that.assetType); } + @Override + public int hashCode() { + int result = Objects.hashCode(attributes); + result = 31 * result + Objects.hashCode(relationships); + result = 31 * result + Objects.hashCode(assetType); + return result; + } } diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java index ed2a7c0..0de9e48 100644 --- a/src/main/java/life/qbic/model/isa/ISAAssay.java +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; /** * Model class for ISA Assays. Contains all mandatory and some optional properties and attributes @@ -25,8 +26,10 @@ public class ISAAssay extends AbstractISAObject { private Attributes attributes; private Relationships relationships; + private String title; public ISAAssay(String title, String studyId, String assayClass, URI assayType) { + this.title = title; this.attributes = new Attributes(title, assayClass, assayType); this.relationships = new Relationships(studyId); } @@ -175,7 +178,7 @@ private void generateListJSON(JsonGenerator generator, String name, List tags = new ArrayList<>(); public String description = ""; @@ -245,4 +248,27 @@ public String getUri() { } } + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ISAAssay)) { + return false; + } + + ISAAssay isaAssay = (ISAAssay) o; + return Objects.equals(attributes, + isaAssay.attributes) && Objects.equals(relationships, isaAssay.relationships) + && Objects.equals(title, isaAssay.title); + } + + @Override + public int hashCode() { + int result = ISA_TYPE.hashCode(); + result = 31 * result + Objects.hashCode(attributes); + result = 31 * result + Objects.hashCode(relationships); + result = 31 * result + Objects.hashCode(title); + return result; + } } diff --git a/src/main/java/life/qbic/model/isa/ISASample.java b/src/main/java/life/qbic/model/isa/ISASample.java index 1663b99..b16e04f 100644 --- a/src/main/java/life/qbic/model/isa/ISASample.java +++ b/src/main/java/life/qbic/model/isa/ISASample.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Model class for ISA Samples. Contains all mandatory and some optional properties and attributes @@ -23,6 +24,7 @@ public class ISASample extends AbstractISAObject { private Attributes attributes; private Relationships relationships; private final String ISA_TYPE = "samples"; + private String id; public ISASample(String title, Map attributeMap, String sampleTypeId, List projectIds) { @@ -35,6 +37,10 @@ public ISASample withOtherCreators(String otherCreators) { return this; } + public void setSampleID(String seekID) { + this.id = seekID; + } + public void setCreatorIDs(List creatorIDs) { this.relationships.setCreatorIDs(creatorIDs); } @@ -43,6 +49,10 @@ public void setAssayIDs(List assayIDs) { this.relationships.setAssayIDs(assayIDs); } + public String getId() { + return id; + } + public String toJson() throws JsonProcessingException { SimpleModule module = new SimpleModule(); module.addSerializer(Relationships.class, new RelationshipsSerializer()); @@ -61,6 +71,10 @@ public Attributes getAttributes() { return attributes; } + public Map fetchCopyOfAttributeMap() { + return new HashMap<>(attributes.getAttribute_map()); + } + private class Relationships { private String sampleTypeId; @@ -165,4 +179,25 @@ public String getOther_creators() { } } + @Override + public final boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ISASample)) { + return false; + } + + ISASample isaSample = (ISASample) o; + return Objects.equals(attributes, isaSample.attributes) && Objects.equals( + relationships, isaSample.relationships); + } + + @Override + public int hashCode() { + int result = Objects.hashCode(attributes); + result = 31 * result + Objects.hashCode(relationships); + result = 31 * result + ISA_TYPE.hashCode(); + return result; + } } diff --git a/src/main/java/life/qbic/model/isa/ISASampleType.java b/src/main/java/life/qbic/model/isa/ISASampleType.java index 201a263..3ccee05 100644 --- a/src/main/java/life/qbic/model/isa/ISASampleType.java +++ b/src/main/java/life/qbic/model/isa/ISASampleType.java @@ -216,5 +216,15 @@ public String getId() { public String getTitle() { return title; } + + @Override + public String toString() { + return "SampleAttributeType{" + + "id='" + id + '\'' + + ", title='" + title + '\'' + + ", baseType='" + baseType + '\'' + + '}'; + } } + } diff --git a/src/main/java/life/qbic/model/isa/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java index 9dd45b4..29caeef 100644 --- a/src/main/java/life/qbic/model/isa/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -1,28 +1,44 @@ package life.qbic.model.isa; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; - +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Stores newly created ISA objects for SEEK, as well as their respective openBIS reference. It is + * assumed that these references are Sample and Experiment Identifiers. PermIds of datasets are taken + * from stored DataSetFiles + */ public class SeekStructure { - private ISAAssay assay; - private List samples; - private Map isaToOpenBISFile; + private final Pair assayAndOpenBISReference; + private final Map samplesWithOpenBISReference; + private final Map isaToOpenBISFile; + + public SeekStructure(ISAAssay assay, String openBISReference) { + this.assayAndOpenBISReference = new ImmutablePair<>(assay, openBISReference); + this.samplesWithOpenBISReference = new HashMap<>(); + this.isaToOpenBISFile = new HashMap<>(); + } + + public void addSample(ISASample sample, String openBISReference) { + samplesWithOpenBISReference.put(sample, openBISReference); + } - public SeekStructure(ISAAssay assay, List samples, - Map isaToOpenBISFile) { - this.assay = assay; - this.samples = samples; - this.isaToOpenBISFile = isaToOpenBISFile; + public void addAsset(GenericSeekAsset asset, DataSetFile file) { + isaToOpenBISFile.put(asset, file); } - public ISAAssay getAssay() { - return assay; + public Pair getAssayWithOpenBISReference() { + return assayAndOpenBISReference; } - public List getSamples() { - return samples; + public Map getSamplesWithOpenBISReference() { + return samplesWithOpenBISReference; } public Map getISAFileToDatasetFiles() { From e2c91a5de90b4e2efe5682e182771c46006b4211 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Fri, 11 Oct 2024 16:57:45 +0200 Subject: [PATCH 32/63] fix config name --- src/main/java/life/qbic/model/OpenbisSeekTranslator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 99fb500..745b7d3 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -66,7 +66,7 @@ private void parseConfigs() throws IOException, ParserConfigurationException, SA final String dataTypeToAttributeType = "openbis_datatype_to_seek_attributetype.xml"; final String datasetToAssaytype = "dataset_type_to_assaytype.properties"; final String experimentTypeToAssayClass = "experiment_type_to_assay_class.properties"; - final String experimentTypeToAssayType = "more.properties"; + final String experimentTypeToAssayType = "openbis_experiment_type_to_seek_assaytype.properties"; this.experimentTypeToAssayType = PropertyReader.getProperties(experimentTypeToAssayType); this.datasetTypeToAssetType = PropertyReader.getProperties(datasetToAssaytype); this.experimentTypeToAssayClass = PropertyReader.getProperties(experimentTypeToAssayClass); From f86c027e6bd8c90d36832ffb0533cf9738af7e5c Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 14 Oct 2024 17:31:35 +0200 Subject: [PATCH 33/63] fix issue caused by mandatory assaytype --- .../life/qbic/model/OpenbisSeekTranslator.java | 7 +++++-- src/main/java/life/qbic/model/isa/ISAAssay.java | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 745b7d3..f83393c 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -160,8 +160,11 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, Experiment exp = experiment.getExperiment(); String expType = exp.getType().getCode(); String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; - ISAAssay assay = new ISAAssay(title, STUDY_ID, experimentTypeToAssayClass.get(expType), - new URI(experimentTypeToAssayType.get(expType))); + ISAAssay assay = new ISAAssay(title, STUDY_ID, experimentTypeToAssayClass.get(expType)); + String assayType = experimentTypeToAssayType.get(expType); + if(assayType !=null && !assayType.isBlank()) { + assay.withAssayType(new URI(assayType)); + } SeekStructure result = new SeekStructure(assay, exp.getIdentifier().getIdentifier()); diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java index 0de9e48..14a5ad5 100644 --- a/src/main/java/life/qbic/model/isa/ISAAssay.java +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -7,9 +7,7 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -28,12 +26,17 @@ public class ISAAssay extends AbstractISAObject { private Relationships relationships; private String title; - public ISAAssay(String title, String studyId, String assayClass, URI assayType) { + public ISAAssay(String title, String studyId, String assayClass) { this.title = title; - this.attributes = new Attributes(title, assayClass, assayType); + this.attributes = new Attributes(title, assayClass); this.relationships = new Relationships(studyId); } + public ISAAssay withAssayType(URI assayType) { + this.attributes.withAssayType(assayType.toString()); + return this; + } + public ISAAssay withOtherCreators(String otherCreators) { this.attributes.otherCreators = otherCreators; return this; @@ -188,7 +191,7 @@ protected class Attributes { private AssayType assayType; private String otherCreators = ""; - public Attributes(String title, String assayClass, URI assayType) { + public Attributes(String title, String assayClass) { this.title = title; this.assayClass = new AssayClass(assayClass); this.assayType = new AssayType(assayType.toString()); @@ -222,6 +225,10 @@ public String getOther_creators() { return otherCreators; } + public void withAssayType(String assayType) { + this.assayType = new AssayType(assayType); + } + private class AssayClass { String key; From a6b814643019fd6a65a1dcc400849be89ff4e9df Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:01:48 +0200 Subject: [PATCH 34/63] enable search for samples and identifiers --- .../io/commandline/FindDatasetsCommand.java | 46 +++++++++++++++---- .../qbic/model/AssayWithQueuedAssets.java | 23 ---------- 2 files changed, 36 insertions(+), 33 deletions(-) delete mode 100644 src/main/java/life/qbic/model/AssayWithQueuedAssets.java diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index bb2072f..3f9f747 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -9,7 +9,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import life.qbic.App; @@ -24,15 +23,19 @@ description = "lists datasets and their details for a given experiment code") public class FindDatasetsCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "experiment", description = "The code of the experiment data is attached to") - private String experimentCode; - @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter samples", names = {"-s", "--space"}) + @Parameters(arity = "1", paramLabel = "openBIS obejct", description = + "The code of the experiment " + + "or sample data is attached to.") + private String objectCode; + @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter " + + "found datasets by", names = {"-s", "--space"}) private String space; @Mixin OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); - @Override + @Override public void run() { + App.readConfig(); List spaces = new ArrayList<>(); if (space != null) { System.out.println("Querying experiment in space: " + space + "..."); @@ -40,13 +43,36 @@ public void run() { } else { System.out.println("Querying experiment in all available spaces..."); } + if(objectCode.contains("/")) { + String[] splt = objectCode.split("/"); + objectCode = splt[splt.length-1]; + System.out.println("Query is not an object code, querying for: " + objectCode+" instead."); + } OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); - List datasets = openbis.listDatasetsOfExperiment(spaces, experimentCode).stream() - .sorted(Comparator.comparing( - (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())).collect( - Collectors.toList()); + + List datasetsOfExp = openbis.listDatasetsOfExperiment(spaces, objectCode).stream() + .sorted(Comparator.comparing( + (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())) + .collect(Collectors.toList()); + + List datasets = new ArrayList<>(); + + if(!datasetsOfExp.isEmpty()) { + System.err.println("Found "+datasetsOfExp.size()+" datasets for experiment "+objectCode); + datasets.addAll(datasetsOfExp); + } + List datasetsOfSample = openbis.listDatasetsOfSample(spaces, objectCode).stream() + .sorted(Comparator.comparing( + (DataSet d) -> d.getExperiment().getProject().getSpace().getCode())) + .collect(Collectors.toList()); + + if(!datasetsOfSample.isEmpty()) { + System.err.println("Found "+datasetsOfSample.size()+" datasets for sample "+objectCode); + datasets.addAll(datasetsOfSample); + } + Map properties = new HashMap<>(); if (!datasets.isEmpty()) { Set patientIDs = openbis.findPropertiesInSampleHierarchy("PATIENT_DKFZ_ID", @@ -64,7 +90,7 @@ public void run() { }).collect(Collectors.toList()); int datasetIndex = 0; System.out.println(); - System.out.printf("Found %s datasets for experiment %s:%n", datasets.size(), experimentCode); + System.out.printf("Found %s datasets for %s:%n", datasets.size(), objectCode); for (DatasetWithProperties dataSet : datasetWithProperties) { datasetIndex++; System.out.println("["+datasetIndex+"]"); diff --git a/src/main/java/life/qbic/model/AssayWithQueuedAssets.java b/src/main/java/life/qbic/model/AssayWithQueuedAssets.java deleted file mode 100644 index 0186cfc..0000000 --- a/src/main/java/life/qbic/model/AssayWithQueuedAssets.java +++ /dev/null @@ -1,23 +0,0 @@ -package life.qbic.model; - -import java.util.List; -import life.qbic.model.download.SEEKConnector.AssetToUpload; - -public class AssayWithQueuedAssets { - - private String assayEndpoint; - private List assetsToUpload; - - public AssayWithQueuedAssets(String assayEndpoint, List assetsToUpload) { - this.assayEndpoint = assayEndpoint; - this.assetsToUpload = assetsToUpload; - } - - public String getAssayEndpoint() { - return assayEndpoint; - } - - public List getAssets() { - return assetsToUpload; - } -} From ba5c72800f08c8cb25b9c9466e506caccef1e278 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:02:38 +0200 Subject: [PATCH 35/63] make assay type mandatory again --- src/main/java/life/qbic/model/isa/ISAAssay.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/life/qbic/model/isa/ISAAssay.java b/src/main/java/life/qbic/model/isa/ISAAssay.java index 14a5ad5..b632b55 100644 --- a/src/main/java/life/qbic/model/isa/ISAAssay.java +++ b/src/main/java/life/qbic/model/isa/ISAAssay.java @@ -26,17 +26,12 @@ public class ISAAssay extends AbstractISAObject { private Relationships relationships; private String title; - public ISAAssay(String title, String studyId, String assayClass) { + public ISAAssay(String title, String studyId, String assayClass, URI assayType) { this.title = title; - this.attributes = new Attributes(title, assayClass); + this.attributes = new Attributes(title, assayClass, assayType); this.relationships = new Relationships(studyId); } - public ISAAssay withAssayType(URI assayType) { - this.attributes.withAssayType(assayType.toString()); - return this; - } - public ISAAssay withOtherCreators(String otherCreators) { this.attributes.otherCreators = otherCreators; return this; @@ -191,7 +186,7 @@ protected class Attributes { private AssayType assayType; private String otherCreators = ""; - public Attributes(String title, String assayClass) { + public Attributes(String title, String assayClass, URI assayType) { this.title = title; this.assayClass = new AssayClass(assayClass); this.assayType = new AssayType(assayType.toString()); From c324b3c937621c5d836fa5e991bf5d129e89d6bb Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:03:29 +0200 Subject: [PATCH 36/63] fix check for config and command line parameters --- .../io/commandline/OpenbisAuthenticationOptions.java | 12 ++++++++++-- .../io/commandline/SeekAuthenticationOptions.java | 9 +++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java index 9228d0b..03a56f8 100644 --- a/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/OpenbisAuthenticationOptions.java @@ -7,7 +7,6 @@ import java.net.URL; import java.util.StringJoiner; import life.qbic.App; -import life.qbic.io.PropertyReader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import picocli.CommandLine; @@ -38,7 +37,8 @@ public class OpenbisAuthenticationOptions { public String getOpenbisUser() { if(openbisUser == null && App.configProperties.containsKey("user")) { openbisUser = App.configProperties.get("user"); - } else { + } + if(openbisUser == null) { log.error("No openBIS user provided."); System.exit(2); } @@ -49,6 +49,10 @@ public String getOpenbisDSS() { if(dss_url == null && App.configProperties.containsKey("dss")) { dss_url = App.configProperties.get("dss"); } + if(dss_url == null) { + log.error("No openBIS datastore server URL provided."); + System.exit(2); + } return dss_url; } @@ -56,6 +60,10 @@ public String getOpenbisAS() { if(as_url == null && App.configProperties.containsKey("as")) { as_url = App.configProperties.get("as"); } + if(as_url == null) { + log.error("No openBIS application server URL provided."); + System.exit(2); + } return as_url; } diff --git a/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java b/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java index 8a58d6e..545949f 100644 --- a/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java +++ b/src/main/java/life/qbic/io/commandline/SeekAuthenticationOptions.java @@ -5,7 +5,6 @@ import java.util.StringJoiner; import life.qbic.App; -import life.qbic.io.PropertyReader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import picocli.CommandLine; @@ -31,7 +30,8 @@ public class SeekAuthenticationOptions { public String getSeekUser() { if(seekUser == null && App.configProperties.containsKey("seek_user")) { seekUser = App.configProperties.get("seek_user"); - } else { + } + if (seekUser == null) { log.error("No SEEK user/email provided."); System.exit(2); } @@ -39,9 +39,10 @@ public String getSeekUser() { } public String getSeekURL() { - if(seek_url == null && App.configProperties.containsKey("seek_url")) { + if(seek_url != null || App.configProperties.containsKey("seek_url")) { seek_url = App.configProperties.get("seek_url"); - } else { + } + if(seek_url == null) { log.error("No URL to the SEEK address provided."); System.exit(2); } From 19d580a3a992f0a3404a1e06c695f7ebd7aab2a0 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:04:13 +0200 Subject: [PATCH 37/63] add checks for assay type translation, enable translation without experiment --- .../qbic/model/OpenbisSeekTranslator.java | 74 ++++++++++++++++++- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index f83393c..e1fc762 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -32,6 +32,7 @@ import life.qbic.model.isa.ISASampleType.SampleAttribute; import life.qbic.model.isa.ISASampleType.SampleAttributeType; import life.qbic.model.isa.SeekStructure; +import org.apache.commons.lang3.tuple.Pair; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -160,11 +161,14 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, Experiment exp = experiment.getExperiment(); String expType = exp.getType().getCode(); String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; - ISAAssay assay = new ISAAssay(title, STUDY_ID, experimentTypeToAssayClass.get(expType)); String assayType = experimentTypeToAssayType.get(expType); - if(assayType !=null && !assayType.isBlank()) { - assay.withAssayType(new URI(assayType)); + + if(assayType ==null || assayType.isBlank()) { + throw new RuntimeException("Could not find assay type for " + expType+". A mapping needs to " + + "be added to the respective properties file."); } + ISAAssay assay = new ISAAssay(title, STUDY_ID, experimentTypeToAssayClass.get(expType), + new URI(assayType)); SeekStructure result = new SeekStructure(assay, exp.getIdentifier().getIdentifier()); @@ -214,6 +218,69 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, return result; } + public SeekStructure translate(OpenbisSampleWithDatasets sampleWithDatasets, + Map sampleTypesToIds, Set blacklist, boolean transferData) { + Sample sample = sampleWithDatasets.getSample(); + SampleType sampleType = sample.getType(); + //try to put all attributes into sample properties, as they should be a 1:1 mapping + Map typeCodesToNames = new HashMap<>(); + Set propertiesLinkingSamples = new HashSet<>(); + for (PropertyAssignment a : sampleType.getPropertyAssignments()) { + String code = a.getPropertyType().getCode(); + String label = a.getPropertyType().getLabel(); + DataType type = a.getPropertyType().getDataType(); + typeCodesToNames.put(code, label); + if (type.equals(DataType.SAMPLE)) { + propertiesLinkingSamples.add(code); + } + } + Map attributes = new HashMap<>(); + for (String code : sample.getProperties().keySet()) { + String value = sample.getProperty(code); + if (propertiesLinkingSamples.contains(code)) { + value = generateOpenBISLinkFromPermID("SAMPLE", value); + } + attributes.put(typeCodesToNames.get(code), value); + } + + String sampleID = sample.getIdentifier().getIdentifier(); + attributes.put(App.configProperties.get("seek_openbis_sample_title"), sampleID); + String sampleTypeId = sampleTypesToIds.get(sampleType.getCode()); + ISASample isaSample = new ISASample(sample.getIdentifier().getIdentifier(), attributes, + sampleTypeId, Collections.singletonList(DEFAULT_PROJECT_ID)); + SeekStructure result = new SeekStructure(isaSample, sampleID); + + //create ISA files for assets. If actual data is to be uploaded is determined later based on flag + List datasets = sampleWithDatasets.getDatasets(); + for (DatasetWithProperties dataset : datasets) { + String permID = dataset.getCode(); + if (!blacklist.contains(permID)) { + for (DataSetFile file : sampleWithDatasets.getFilesForDataset(permID)) { + String datasetType = getDatasetTypeOfFile(file, datasets); + datasetFileToSeekAsset(file, datasetType, transferData) + .ifPresent(seekAsset -> result.addAsset(seekAsset, file)); + } + } + } + return result; + } + + public SeekStructure translate(Pair> datasetWithFiles, + Set blacklist, boolean transferData) { + Map assetToDatasetFiles = new HashMap<>(); + DatasetWithProperties dataset = datasetWithFiles.getLeft(); + String permID = dataset.getCode(); + String datasetType = dataset.getType().getCode(); + + if (!blacklist.contains(permID)) { + for (DataSetFile file : datasetWithFiles.getRight()) { + datasetFileToSeekAsset(file, datasetType, transferData) + .ifPresent(seekAsset -> assetToDatasetFiles.put(seekAsset, file)); + } + } + return new SeekStructure(assetToDatasetFiles); + } + private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { String permId = file.getDataSetPermId().getPermId(); for(DatasetWithProperties dataset : dataSets) { @@ -260,4 +327,5 @@ public void setDefaultStudy(String studyID) { public void setDefaultInvestigation(String investigationID) { this.INVESTIGATION_ID = investigationID; } + } From a9446b233fa5425100932f80de3cc20463cbe14f Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:05:45 +0200 Subject: [PATCH 38/63] fix recursion for list of parent properties --- .../qbic/model/download/OpenbisConnector.java | 91 +++++++++++++++---- 1 file changed, 74 insertions(+), 17 deletions(-) diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 386e494..f4906d6 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -47,14 +47,15 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; +import life.qbic.model.OpenbisSampleWithDatasets; import life.qbic.model.SampleTypeConnection; import life.qbic.model.SampleTypesAndMaterials; import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; +import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -81,8 +82,8 @@ public List getSpaces() { } public DataSetPermId registerDatasetForExperiment(Path uploadPath, String experimentID, - List parentCodes) { - UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, parentCodes); + String datasetType, List parentCodes) { + UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, datasetType, parentCodes); creation.setExperimentId(new ExperimentIdentifier(experimentID)); try { @@ -94,8 +95,8 @@ public DataSetPermId registerDatasetForExperiment(Path uploadPath, String experi } public DataSetPermId registerDatasetForSample(Path uploadPath, String sampleID, - List parentCodes) { - UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, parentCodes); + String datasetType, List parentCodes) { + UploadedDataSetCreation creation = prepareDataSetCreation(uploadPath, datasetType, parentCodes); creation.setSampleId(new SampleIdentifier(sampleID)); try { @@ -106,14 +107,15 @@ public DataSetPermId registerDatasetForSample(Path uploadPath, String sampleID, return null; } - private UploadedDataSetCreation prepareDataSetCreation(Path uploadPath, List parentCodes) { + private UploadedDataSetCreation prepareDataSetCreation(Path uploadPath, String datasetType, + List parentCodes) { final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); final UploadedDataSetCreation creation = new UploadedDataSetCreation(); creation.setUploadId(uploadId); creation.setParentIds(parentCodes.stream().map(DataSetPermId::new).collect( Collectors.toList())); - creation.setTypeId(new EntityTypePermId("UNKNOWN", EntityKind.DATA_SET)); + creation.setTypeId(new EntityTypePermId(datasetType, EntityKind.DATA_SET)); return creation; } @@ -143,6 +145,20 @@ public List listDatasetsOfExperiment(List spaces, String experi return openBIS.searchDataSets(criteria, options).getObjects(); } + public List listDatasetsOfSample(List spaces, String sample) { + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withSample().withCode().thatEquals(sample); + if (!spaces.isEmpty()) { + criteria.withAndOperator(); + criteria.withExperiment().withProject().withSpace().withCodes().thatIn(spaces); + } + DataSetFetchOptions options = new DataSetFetchOptions(); + options.withType(); + options.withRegistrator(); + options.withExperiment().withProject().withSpace(); + return openBIS.searchDataSets(criteria, options).getObjects(); + } + public File downloadDataset(String targetPath, String datasetID, String filePath) { DataSetFileDownloadOptions options = new DataSetFileDownloadOptions(); IDataSetFileId fileToDownload = new DataSetFilePermId(new DataSetPermId(datasetID), @@ -238,14 +254,17 @@ public Map queryFullSampleHierarchy(List private Set getPropertiesFromSampleHierarchy(String propertyName, List samples, Set foundProperties) { + if(samples.isEmpty()) { + return foundProperties; + } for(Sample s : samples) { if(s.getProperties().containsKey(propertyName)) { foundProperties.add(s.getProperties().get(propertyName)); - return foundProperties; } - return getPropertiesFromSampleHierarchy(propertyName, s.getParents(), foundProperties); } - return foundProperties; + return getPropertiesFromSampleHierarchy(propertyName, + samples.stream().map(Sample::getParents).flatMap(List::stream).collect(Collectors.toList()), + foundProperties); } public Set findPropertiesInSampleHierarchy(String propertyName, @@ -502,13 +521,15 @@ public SampleTypesAndMaterials getSampleTypesWithMaterials() { return new SampleTypesAndMaterials(sampleTypes, sampleTypesAsMaterials); } - public void createSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { - Pair experimentInfo = postRegistrationInformation.getExperimentIDWithEndpoint(); - ExperimentIdentifier id = new ExperimentIdentifier(experimentInfo.getLeft()); - String endpoint = experimentInfo.getRight(); - Map props = new HashMap<>(); - props.put(EXPERIMENT_LINK_PROPERTY, endpoint); - updateExperimentProperties(id, props, false); + public void createSeekLinks(SeekStructurePostRegistrationInformation postRegInformation) { + Optional> experimentInfo = postRegInformation.getExperimentIDWithEndpoint(); + if(experimentInfo.isPresent()) { + ExperimentIdentifier id = new ExperimentIdentifier(experimentInfo.get().getLeft()); + String endpoint = experimentInfo.get().getRight(); + Map props = new HashMap<>(); + props.put(EXPERIMENT_LINK_PROPERTY, endpoint); + updateExperimentProperties(id, props, false); + } } public void updateSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { @@ -583,4 +604,40 @@ private void updateDatasetProperties(DataSetPermId id, Map prope openBIS.updateDataSets(Arrays.asList(update)); } + public OpenbisSampleWithDatasets getSampleWithDatasets(String sampleID) { + SampleSearchCriteria criteria = new SampleSearchCriteria(); + criteria.withIdentifier().thatEquals(sampleID); + + DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); + dataSetFetchOptions.withType(); + dataSetFetchOptions.withRegistrator(); + SampleFetchOptions fetchOptions = new SampleFetchOptions(); + fetchOptions.withProperties(); + fetchOptions.withType().withPropertyAssignments().withPropertyType(); + fetchOptions.withDataSetsUsing(dataSetFetchOptions); + fetchOptions.withDataSetsUsing(dataSetFetchOptions); + + Sample sample = openBIS.searchSamples(criteria, fetchOptions).getObjects().get(0); + + List datasets = new ArrayList<>(); + Map> datasetCodeToFiles = new HashMap<>(); + for(DataSet dataset : sample.getDataSets()) { + datasets.add(new DatasetWithProperties(dataset)); + datasetCodeToFiles.put(dataset.getPermId().getPermId(), getDatasetFiles(dataset)); + } + + return new OpenbisSampleWithDatasets(sample, datasets, datasetCodeToFiles); + } + + public Pair> getDataSetWithFiles(String datasetID) { + DataSetSearchCriteria criteria = new DataSetSearchCriteria(); + criteria.withPermId().thatEquals(datasetID); + + DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); + dataSetFetchOptions.withType(); + dataSetFetchOptions.withRegistrator(); + + DataSet dataset = openBIS.searchDataSets(criteria, dataSetFetchOptions).getObjects().get(0); + return new ImmutablePair<>(new DatasetWithProperties(dataset), getDatasetFiles(dataset)); + } } From 983ff91f23ca5aebcea3d7bcfc24b3e746c795e0 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:06:50 +0200 Subject: [PATCH 39/63] add dataset type as optional parameter --- .../io/commandline/UploadDatasetCommand.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 98e9f11..c9756fd 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -25,6 +25,9 @@ public class UploadDatasetCommand implements Runnable { @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) private List parents = new ArrayList<>(); + @Option(arity = "1", paramLabel = "dataset type", description = "The openBIS dataset type code the " + + "data should be stored as. UNKNOWN if no type is chosen.", names = {"-t", "--type"}) + private String datasetType = "UNKNOWN"; @Mixin OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @@ -32,6 +35,8 @@ public class UploadDatasetCommand implements Runnable { @Override public void run() { + App.readConfig(); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); @@ -40,8 +45,8 @@ public void run() { return; } boolean attachToSample = false; - boolean attachToExperiment = experimentExists(objectID); - if(sampleExists(objectID)) { + boolean attachToExperiment = openbis.experimentExists(objectID); + if(openbis.sampleExists(objectID)) { attachToSample = true; } if(!attachToSample && !attachToExperiment) { @@ -56,26 +61,20 @@ public void run() { System.out.println("Parameters verified, uploading dataset..."); System.out.println(); if(attachToExperiment) { - DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, parents); + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, + datasetType, parents); System.out.printf("Dataset %s was successfully attached to experiment%n", result.getPermId()); } else { - DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, parents); + DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, + datasetType, parents); System.out.printf("Dataset %s was successfully attached to sample%n", result.getPermId()); } } - private boolean sampleExists(String objectID) { - return openbis.sampleExists(objectID); - } - private boolean datasetsExist(List datasetCodes) { return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); } - private boolean experimentExists(String experimentID) { - return openbis.experimentExists(experimentID); - } - private boolean pathValid(String dataPath) { return new File(dataPath).exists(); } From 2cbf1e6b1c1ac9f83f2bd15aaf38a4dd5b344f2b Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:09:50 +0200 Subject: [PATCH 40/63] fix issue where config was not read --- .../java/life/qbic/io/commandline/SampleHierarchyCommand.java | 1 + .../java/life/qbic/io/commandline/SpaceStatisticsCommand.java | 1 + .../qbic/io/commandline/TransferSampleTypesToSeekCommand.java | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index 228d615..fbd66b8 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -35,6 +35,7 @@ public class SampleHierarchyCommand implements Runnable { @Override public void run() { + App.readConfig(); List summary = new ArrayList<>(); List spaces = new ArrayList<>(); if(space!=null) { diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index 4ae0323..5e66d24 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -41,6 +41,7 @@ public class SpaceStatisticsCommand implements Runnable { @Override public void run() { + App.readConfig(); List summary = new ArrayList<>(); List blackList = new ArrayList<>(Arrays.asList("ELN_SETTINGS", "MATERIAL.GLOBAL")); List spaces = new ArrayList<>(); diff --git a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java index 3ba3d55..53eecd1 100644 --- a/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferSampleTypesToSeekCommand.java @@ -6,7 +6,6 @@ import java.net.URISyntaxException; import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; -import life.qbic.io.PropertyReader; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.SampleTypesAndMaterials; import life.qbic.model.download.OpenbisConnector; @@ -35,6 +34,8 @@ public class TransferSampleTypesToSeekCommand implements Runnable { @Override public void run() { + App.readConfig(); + System.out.println("auth..."); OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), From 9bdf010b6185f4250f2de1c38b567c9f85cff531 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:10:32 +0200 Subject: [PATCH 41/63] add optional parameter for dataset type --- .../commandline/UploadPetabResultCommand.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 80d40bc..bffdc1e 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -11,20 +11,24 @@ import life.qbic.model.download.OpenbisConnector; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @Command(name = "upload-petab", - description = "uploads a PETab folder and attaches it to a provided experiment and any datasets referenced in the PETab metadata (e.g. for PETab results).") + description = "uploads a PETab folder and attaches it to a provided experiment and any datasets " + + "referenced in the PETab metadata (e.g. for PETab results).") public class UploadPetabResultCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") + @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder " + + "to upload") private String dataPath; - @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the experiment the data should be attached to. " + @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the " + + "+experiment the data should be attached to. " + "The identifier must be of the format: /space/project/experiment") private String experimentID; - //@Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" - // + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) - private List parents = new ArrayList<>(); + @Option(arity = "1", paramLabel = "dataset type", description = "The openBIS dataset type code the " + + "data should be stored as. UNKNOWN if no type is chosen.", names = {"-t", "--type"}) + private String datasetType = "UNKNOWN"; @Mixin OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @@ -33,6 +37,8 @@ public class UploadPetabResultCommand implements Runnable { @Override public void run() { + App.readConfig(); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); @@ -49,9 +55,9 @@ public void run() { return; } System.out.println("Looking for reference datasets in metaInformation.yaml..."); - parents = petabParser.parse(dataPath).getSourcePetabReferences(); + List parents = petabParser.parse(dataPath).getSourcePetabReferences(); if(parents.isEmpty()) { - System.out.println("No reference datasets found in openBISSourceIds property. Assuming" + System.out.println("No reference datasets found in openBISSourceIds property. Assuming " + "this is a new dataset."); } else { System.out.println("Found reference ids: " + String.join(", ", parents)); @@ -63,7 +69,8 @@ public void run() { } } System.out.println("Uploading dataset..."); - DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, parents); + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, + datasetType, parents); System.out.printf("Dataset %s was successfully created%n", result.getPermId()); } From 27eac030bc746d1f0ce77ef1e69adddce22d8714 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:11:03 +0200 Subject: [PATCH 42/63] fix issue where config was not read --- .../java/life/qbic/io/commandline/DownloadPetabCommand.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index a80f861..5deac91 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -28,6 +28,7 @@ public class DownloadPetabCommand implements Runnable { @Override public void run() { + App.readConfig(); OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); OpenbisConnector openbis = new OpenbisConnector(authentication); @@ -35,7 +36,7 @@ public void run() { List datasets = openbis.findDataSets(Collections.singletonList(datasetCode)); if(datasets.isEmpty()) { - System.out.println(datasetCode+" not found"); + System.out.println("Dataset "+datasetCode+" not found"); return; } DatasetWithProperties result = new DatasetWithProperties(datasets.get(0)); @@ -54,6 +55,7 @@ public void run() { try { System.out.println("Adding dataset identifier to metaInformation.yaml."); parser.addDatasetId(outputPath, datasetCode); + parser.addPatientIDs(outputPath, patientIDs); } catch (IOException e) { System.out.println("Could not add dataset identifier."); throw new RuntimeException(e); From 705877563601e8f63eb514308f1c3a0cabf77988 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:11:47 +0200 Subject: [PATCH 43/63] add test logging of patient id property --- src/main/java/life/qbic/io/PetabParser.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index 220e41b..25461ed 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; import life.qbic.model.petab.PetabMetadata; public class PetabParser { @@ -94,4 +95,7 @@ private File findYaml(File directory) { return null; } -} \ No newline at end of file + public void addPatientIDs(String outputPath, Set patientIDs) { + System.err.println("found patient ids: "+patientIDs); + } +} From dd3000a1ae6c855e3820eb224793516fc112ac48 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Tue, 15 Oct 2024 02:12:42 +0200 Subject: [PATCH 44/63] add partial functionality to add and update samples and assets without assay (experiment) information --- .../TransferDataToSeekCommand.java | 256 +++++++++++++----- .../qbic/model/OpenbisSampleWithDatasets.java | 32 +++ .../qbic/model/download/SEEKConnector.java | 241 +++++++++++++++-- .../life/qbic/model/isa/SeekStructure.java | 19 +- 4 files changed, 455 insertions(+), 93 deletions(-) create mode 100644 src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 4160d45..7c39790 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -2,6 +2,8 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -17,15 +19,18 @@ import java.util.stream.Stream; import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; -import life.qbic.model.AssayWithQueuedAssets; +import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; +import life.qbic.model.OpenbisSampleWithDatasets; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; +import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import life.qbic.model.download.SEEKConnector.AssetToUpload; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.tuple.Pair; import org.xml.sax.SAXException; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -48,8 +53,8 @@ public class TransferDataToSeekCommand implements Runnable { @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + "experiment, sample or dataset to transfer.") private String objectID; - @Parameters(arity = "1", paramLabel = "seek study", description = "Title of the study in SEEK" - + "where nodes should be added.") + @Option(names = "--seek-study", description = "Title of the study in SEEK where nodes should be " + + "added. Mandatory for assay creation.") private String studyTitle; @Option(names = "--seek-project", description = "Title of the project in SEEK where nodes should" + "be added. Can alternatively be provided via the config file as 'seek_default_project'.") @@ -87,7 +92,6 @@ public void run() { } else { System.out.printf("No SEEK project title provided, will search config file.%n"); } - System.out.printf("Provided SEEK study title: %s%n", studyTitle); System.out.printf("Transfer datasets to SEEK? %s%n", transferData); System.out.printf("Update existing assay if found? %s%n", !noUpdate); if(blacklistFile!=null && !blacklistFile.isBlank()) { @@ -99,7 +103,7 @@ public void run() { OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), openbisAuth.getOpenbisUser(), openbisAuth.getOpenbisAS(), openbisAuth.getOpenbisDSS()); - openbis = new OpenbisConnector(authentication); + this.openbis = new OpenbisConnector(authentication); boolean isSample = false; boolean isDataSet = false; @@ -107,6 +111,13 @@ public void run() { System.out.println("Searching for specified object in openBIS..."); boolean isExperiment = experimentExists(objectID); + + if (isExperiment && (studyTitle == null || studyTitle.isBlank())) { + System.out.printf( + "No SEEK study title was provided. This is mandatory if an openBIS experiment is to be transferred%n"); + return; + } + if (!isExperiment && sampleExists(objectID)) { isSample = true; } @@ -140,36 +151,130 @@ public void run() { ParserConfigurationException | SAXException e) { throw new RuntimeException(e); } - + SeekStructurePostRegistrationInformation postRegInfo; try { - System.out.println("Collecting information from openBIS..."); - OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); + if (isExperiment) { + postRegInfo = handleExperimentTransfer(); + } else if (isSample) { + postRegInfo = handleSampleTransfer(); + } else { + postRegInfo = handleDatasetTransfer(); + } + } catch (URISyntaxException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + System.out.println("Creating links to new SEEK objects in openBIS..."); + openbis.createSeekLinks(postRegInfo); + + System.out.println("Done"); + } + + private SeekStructurePostRegistrationInformation handleExperimentTransfer() + throws URISyntaxException, IOException, InterruptedException { + System.out.println("Collecting information from openBIS..."); + OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); + Set blacklist = parseBlackList(blacklistFile); + System.out.println("Translating openBIS property codes to SEEK names..."); + Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); + System.out.println("Creating SEEK structure..."); + SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist, + transferData); + if (!noUpdate) { System.out.println("Trying to find existing corresponding assay in SEEK..."); Optional assayID = getAssayIDForOpenBISExperiment(experiment.getExperiment()); - assayID.ifPresent(x -> System.out.println("Found assay with id "+assayID.get())); - Set blacklist = parseBlackList(blacklistFile); - System.out.println("Translating openBIS property codes to SEEK names..."); - Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); - System.out.println("Creating SEEK structure..."); - SeekStructure nodeWithChildren = translator.translate(experiment, sampleTypesToIds, blacklist, - transferData); - - if(assayID.isEmpty() || noUpdate) { + assayID.ifPresent(x -> System.out.println("Found assay with id " + assayID.get())); + if (assayID.isEmpty()) { System.out.println("Creating new node(s)..."); - SeekStructurePostRegistrationInformation postRegInfo = createNewNodes(nodeWithChildren); - System.out.println("Creating links to SEEK objects in openBIS..."); - openbis.createSeekLinks(postRegInfo); + return createNewAssayStructure(nodeWithChildren); } else { System.out.println("Updating nodes..."); - SeekStructurePostRegistrationInformation postRegInfo = updateNodes(nodeWithChildren, - assayID.get()); - System.out.println("Updating links to SEEK objects in openBIS..."); - openbis.updateSeekLinks(postRegInfo); + return updateAssayStructure(nodeWithChildren, assayID.get()); } - } catch (URISyntaxException | InterruptedException | IOException e) { - throw new RuntimeException(e); } - System.out.println("Done"); + System.out.println("Creating new node(s)..."); + return createNewAssayStructure(nodeWithChildren); + } + + private SeekStructurePostRegistrationInformation handleSampleTransfer() + throws URISyntaxException, IOException, InterruptedException { + System.out.println("Collecting information from openBIS..."); + OpenbisSampleWithDatasets sampleWithDatasets = openbis.getSampleWithDatasets(objectID); + Set blacklist = parseBlackList(blacklistFile); + System.out.println("Translating openBIS property codes to SEEK names..."); + Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); + System.out.println("Trying to find existing corresponding sample in SEEK..."); + Optional sampleID = getSampleIDForOpenBISSample(sampleWithDatasets.getSample()); + sampleID.ifPresent(x -> System.out.println("Found sample with id "+sampleID.get())); + SeekStructure nodeWithChildren = translator.translate(sampleWithDatasets, sampleTypesToIds, + blacklist, transferData); + + System.out.println("Creating SEEK structure..."); + if(sampleID.isEmpty() || noUpdate) { + System.out.println("Creating new node(s)..."); + return createNewSampleStructure(nodeWithChildren); + } else { + System.out.println("Updating nodes..."); + return updateSampleStructure(nodeWithChildren, sampleID.get()); + } + } + + private SeekStructurePostRegistrationInformation handleDatasetTransfer() + throws URISyntaxException, IOException, InterruptedException { + System.out.println("Collecting information from openBIS..."); + Pair> datasetWithFiles = openbis.getDataSetWithFiles( + objectID); + Set blacklist = parseBlackList(blacklistFile); + //TODO is this necessary? + //System.out.println("Trying to find existing corresponding assets in SEEK..."); + //List assetIDs = seek.searchAssetsContainingKeyword(datasetWithFiles.getLeft().getCode()); + //System.out.println("Found existing asset ids: "+assetIDs); + SeekStructure nodeWithChildren = translator.translate(datasetWithFiles, blacklist, transferData); + + System.out.println("Creating new asset(s)..."); + return createNewAssetsForDataset(nodeWithChildren.getISAFileToDatasetFiles()); + } + + private SeekStructurePostRegistrationInformation updateSampleStructure( + SeekStructure nodeWithChildren, String sampleID) + throws URISyntaxException, IOException, InterruptedException { + SeekStructurePostRegistrationInformation postRegInfo = + seek.updateSampleNode(nodeWithChildren, sampleID); + List assetsToUpload = postRegInfo.getAssetsToUpload(); + if (transferData) { + handleDataTransfer(assetsToUpload); + } + postRegInfo.getExperimentIDWithEndpoint().ifPresentOrElse( + (value) -> System.out.printf("%s was successfully updated.%n", value.getRight()), + () -> System.out.printf("Update performed, but assay id not found in post update info.%n") + ); + return postRegInfo; + } + + private SeekStructurePostRegistrationInformation createNewSampleStructure( + SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { + SeekStructurePostRegistrationInformation postRegistrationInformation = + seek.createSampleWithAssets(nodeWithChildren); + List assetsOfAssayToUpload = postRegistrationInformation.getAssetsToUpload(); + if (transferData) { + handleDataTransfer(assetsOfAssayToUpload); + } + System.out.printf("Sample was successfully created.%n"); + return postRegistrationInformation; + } + + private SeekStructurePostRegistrationInformation createNewAssetsForDataset( + Map assets) throws URISyntaxException, IOException, + InterruptedException { + + SeekStructurePostRegistrationInformation postRegistrationInformation = + seek.createStandaloneAssets(assets); + List assetsOfAssayToUpload = postRegistrationInformation.getAssetsToUpload(); + if (transferData) { + handleDataTransfer(assetsOfAssayToUpload); + } + System.out.printf("Assets were successfully created.%n"); + return postRegistrationInformation; } private Set parseBlackList(String blacklistFile) { @@ -195,50 +300,64 @@ private Set parseBlackList(String blacklistFile) { } } - private SeekStructurePostRegistrationInformation updateNodes(SeekStructure nodeWithChildren, + private SeekStructurePostRegistrationInformation updateAssayStructure(SeekStructure nodeWithChildren, String assayID) throws URISyntaxException, IOException, InterruptedException { - SeekStructurePostRegistrationInformation postRegInfo = seek.updateNode(nodeWithChildren, assayID); - System.out.printf("%s was successfully updated.%n", postRegInfo.getExperimentIDWithEndpoint().getRight()); + SeekStructurePostRegistrationInformation postRegInfo = seek.updateAssayNode(nodeWithChildren, + assayID); + List assetsToUpload = postRegInfo.getAssetsToUpload(); + if (transferData) { + handleDataTransfer(assetsToUpload); + } + postRegInfo.getExperimentIDWithEndpoint().ifPresentOrElse( + (value) -> System.out.printf("%s was successfully updated.%n", value.getRight()), + () -> System.out.printf("Update performed, but assay id not found in post update info.%n") + ); return postRegInfo; } - private SeekStructurePostRegistrationInformation createNewNodes(SeekStructure nodeWithChildren) + private SeekStructurePostRegistrationInformation createNewAssayStructure( + SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - SeekStructurePostRegistrationInformation postRegistrationInformation = + SeekStructurePostRegistrationInformation postRegInfo = seek.createNode(nodeWithChildren); - AssayWithQueuedAssets assetsOfAssayToUpload = postRegistrationInformation.getAssayWithQueuedAssets(); - if(transferData) { - final String tmpFolderPath = "tmp/"; - for(AssetToUpload asset : assetsOfAssayToUpload.getAssets()) { - String filePath = asset.getFilePath(); - String dsCode = asset.getDataSetCode(); - if(asset.getFileSizeInBytes() > 1000*1024*1024) { - System.out.printf("Skipping %s due to size...%n", - filePath); - } else if (asset.getFileSizeInBytes() > 300 * 1024 * 1024) { - System.out.printf("File is %s MB...streaming might take a while%n", - asset.getFileSizeInBytes() / (1024 * 1024)); - System.out.printf("Downloading file %s from openBIS to tmp folder due to size...%n", - filePath); - File tmpFile = openbis.downloadDataset(tmpFolderPath, dsCode, filePath); - - System.out.printf("Uploading file to SEEK...%n"); - String fileURL = seek.uploadFileContent(asset.getBlobEndpoint(), tmpFile.getAbsolutePath()); - System.out.printf("File stored here: %s%n", fileURL); - } else { - System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + List assetsToUpload = postRegInfo.getAssetsToUpload(); + if (transferData) { + handleDataTransfer(assetsToUpload); + } + System.out.printf("Assay was successfully created.%n"); + return postRegInfo; + } + + private void handleDataTransfer(List assets) + throws URISyntaxException, IOException, InterruptedException { + final String tmpFolderPath = "tmp/"; + for(AssetToUpload asset : assets) { + String filePath = asset.getFilePath(); + String dsCode = asset.getDataSetCode(); + if(asset.getFileSizeInBytes() > 1000*1024*1024) { + System.out.printf("Skipping %s due to size...%n", + filePath); + } else if (asset.getFileSizeInBytes() > 300 * 1024 * 1024) { + System.out.printf("File is %s MB...streaming might take a while%n", + asset.getFileSizeInBytes() / (1024 * 1024)); + System.out.printf("Downloading file %s from openBIS to tmp folder due to size...%n", + filePath); + File tmpFile = openbis.downloadDataset(tmpFolderPath, dsCode, filePath); + + System.out.printf("Uploading file to SEEK...%n"); + String fileURL = seek.uploadFileContent(asset.getBlobEndpoint(), tmpFile.getAbsolutePath()); + System.out.printf("File stored here: %s%n", fileURL); + } else { + System.out.printf("Streaming file %s from openBIS to SEEK...%n", asset.getFilePath()); + + String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), + () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); + System.out.printf("File stored here: %s%n", fileURL); - String fileURL = seek.uploadStreamContent(asset.getBlobEndpoint(), - () -> openbis.streamDataset(asset.getDataSetCode(), asset.getFilePath())); - System.out.printf("File stored here: %s%n", fileURL); - } } System.out.printf("Cleaning up temp folder%n"); cleanupTemp(new File(tmpFolderPath)); } - - System.out.printf("%s was successfully created.%n", assetsOfAssayToUpload.getAssayEndpoint()); - return postRegistrationInformation; } private boolean sampleExists(String objectID) { @@ -262,13 +381,26 @@ private Optional getAssayIDForOpenBISExperiment(Experiment experiment) String permID = experiment.getPermId().getPermId(); List assayIDs = seek.searchAssaysContainingKeyword(permID); if(assayIDs.isEmpty()) { - System.err.println("no assay found containing "+permID); return Optional.empty(); } if(assayIDs.size() == 1) { return Optional.of(assayIDs.get(0)); } - throw new RuntimeException("Experiment identifier "+permID+ " was found in more than one assay: "+assayIDs); + throw new RuntimeException("Experiment identifier "+permID+ " was found in more than one assay: " + +assayIDs+". Don't know which assay to update."); + } + + private Optional getSampleIDForOpenBISSample(Sample sample) + throws URISyntaxException, IOException, InterruptedException { + String id = sample.getIdentifier().getIdentifier(); + List sampleIDs = seek.searchSamplesContainingKeyword(id); + if(sampleIDs.isEmpty()) { + return Optional.empty(); + } + if(sampleIDs.size() == 1) { + return Optional.of(sampleIDs.get(0)); + } + throw new RuntimeException("Experiment identifier "+id+ " was found in more than one sample: "+sampleIDs); } private void cleanupTemp(File tmpFolder) { diff --git a/src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java b/src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java new file mode 100644 index 0000000..431d66c --- /dev/null +++ b/src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java @@ -0,0 +1,32 @@ +package life.qbic.model; + +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.util.List; +import java.util.Map; + +public class OpenbisSampleWithDatasets { + + private Sample sample; + private List datasets; + private Map> datasetCodeToFiles; + + public OpenbisSampleWithDatasets(Sample sample, + List datasets, Map> datasetCodeToFiles) { + this.sample = sample; + this.datasets = datasets; + this.datasetCodeToFiles = datasetCodeToFiles; + } + + public Sample getSample() { + return sample; + } + + public List getDatasets() { + return datasets; + } + + public List getFilesForDataset(String permID) { + return datasetCodeToFiles.get(permID); + } +} diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 6214c7e..10195bf 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -27,7 +27,6 @@ import java.util.function.Supplier; import java.util.regex.Matcher; import javax.xml.parsers.ParserConfigurationException; -import life.qbic.model.AssayWithQueuedAssets; import life.qbic.model.AssetInformation; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.SampleInformation; @@ -51,6 +50,8 @@ public class SEEKConnector { private byte[] credentials; private OpenbisSeekTranslator translator; private final String DEFAULT_PROJECT_ID; + private final List ASSET_TYPES = new ArrayList<>(Arrays.asList("data_files", "models", + "sops", "documents", "publications")); public SEEKConnector(String seekURL, byte[] httpCredentials, String openBISBaseURL, String defaultProjectTitle) throws URISyntaxException, IOException, @@ -136,6 +137,8 @@ public String addAssay(ISAAssay assay) .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), BodyHandlers.ofString()); + System.err.println(assay.toJson()); + if(response.statusCode()!=200) { throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } @@ -376,12 +379,14 @@ public boolean endPointExists(String endpoint) * @throws URISyntaxException * @throws InterruptedException */ - public List createAssetsForAssays(Map isaToOpenBISFile, - List assays) + public List createAssetsForAssays(Map isaToOpenBISFile, List assays) throws IOException, URISyntaxException, InterruptedException { List result = new ArrayList<>(); for (GenericSeekAsset isaFile : isaToOpenBISFile.keySet()) { - isaFile.withAssays(assays); + if(!assays.isEmpty()) { + isaFile.withAssays(assays); + } result.add(createAsset(isaToOpenBISFile.get(isaFile).getDataSetPermId().getPermId(), isaFile)); } @@ -511,6 +516,45 @@ public List searchAssaysContainingKeyword(String searchTerm) return assayIDs; } + /** + * Searches for samples containing a search term and returns a list of found sample ids + * @param searchTerm the search term that should be in the assay properties - e.g. an openBIS id + * @return + * @throws URISyntaxException + * @throws IOException + * @throws InterruptedException + */ + public List searchSamplesContainingKeyword(String searchTerm) + throws URISyntaxException, IOException, InterruptedException { + + JsonNode result = genericSearch("samples", "*"+searchTerm+"*"); + + JsonNode hits = result.path("data"); + List assayIDs = new ArrayList<>(); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assayIDs.add(hit.get("id").asText()); + } + return assayIDs; + } + + + public List searchAssetsContainingKeyword(String searchTerm) + throws URISyntaxException, IOException, InterruptedException { + List assetIDs = new ArrayList<>(); + for(String type : ASSET_TYPES) { + JsonNode result = genericSearch(type, "*"+searchTerm+"*"); + + JsonNode hits = result.path("data"); + for (Iterator it = hits.elements(); it.hasNext(); ) { + JsonNode hit = it.next(); + assetIDs.add(hit.get("id").asText()); + } + } + return assetIDs; + } + + /** * Updates information of an existing assay, its samples and attached assets. Missing samples and * assets are created, but nothing missing from the new structure is deleted from SEEK. @@ -520,7 +564,7 @@ public List searchAssaysContainingKeyword(String searchTerm) * data to newly created assets. In the case of the update use case, only newly created objects * will be contained in the return object. */ - public SeekStructurePostRegistrationInformation updateNode(SeekStructure nodeWithChildren, + public SeekStructurePostRegistrationInformation updateAssayNode(SeekStructure nodeWithChildren, String assayID) throws URISyntaxException, IOException, InterruptedException { JsonNode assayData = fetchAssayData(assayID).get("data"); Map sampleInfos = collectSampleInformation(assayData); @@ -593,22 +637,25 @@ public SeekStructurePostRegistrationInformation updateNode(SeekStructure nodeWit } String assayEndpoint = apiURL + "/assays/" + assayID; - AssayWithQueuedAssets assayWithQueuedAssets = - new AssayWithQueuedAssets(assayEndpoint, assetsToUpload); - String expID = nodeWithChildren.getAssayWithOpenBISReference().getRight(); + if(nodeWithChildren.getAssayWithOpenBISReference().isEmpty()) { + throw new RuntimeException("No assay and openBIS reference found. Object has not been " + + "initialized using an assay object and openBIS experiment reference."); + } + String expID = nodeWithChildren.getAssayWithOpenBISReference().get().getRight(); Pair experimentIDWithEndpoint = new ImmutablePair<>(expID, assayEndpoint); - return new SeekStructurePostRegistrationInformation(assayWithQueuedAssets, - experimentIDWithEndpoint, sampleIDsWithEndpoints, datasetIDsWithEndpoints); + SeekStructurePostRegistrationInformation postRegInfo = + new SeekStructurePostRegistrationInformation(assetsToUpload, sampleIDsWithEndpoints, + datasetIDsWithEndpoints); + postRegInfo.setExperimentIDWithEndpoint(experimentIDWithEndpoint); + return postRegInfo; } private Map collectAssetInformation(JsonNode assayData) throws URISyntaxException, IOException, InterruptedException { - List assetTypes = new ArrayList<>(Arrays.asList("data_files", "models", "sops", - "documents", "publications")); Map assets = new HashMap<>(); JsonNode relationships = assayData.get("relationships"); - for(String type : assetTypes) { + for(String type : ASSET_TYPES) { for (Iterator it = relationships.get(type).get("data").elements(); it.hasNext(); ) { String assetID = it.next().get("id").asText(); AssetInformation assetInfo = fetchAssetInformation(assetID, type); @@ -674,6 +721,71 @@ private Optional tryParseDatasetPermID(String input) { return Optional.empty(); } + public SeekStructurePostRegistrationInformation updateSampleNode(SeekStructure nodeWithChildren, + String sampleID) throws URISyntaxException, IOException, InterruptedException { + SampleInformation existingSampleInfo = fetchSampleInformation(sampleID); + //TODO to be able to connect samples with assets, we need to create a new assay, here + + // compare samples + Map newSamplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + + List samplesToCreate = new ArrayList<>(); + for (ISASample newSample : newSamplesWithReferences.keySet()) { + String openBisID = newSamplesWithReferences.get(newSample); + if (!existingSampleInfo.getOpenBisIdentifier().equals(openBisID)) { + samplesToCreate.add(newSample); + System.out.printf("%s not found in SEEK. It will be created.%n", openBisID); + } else { + Map newAttributes = newSample.fetchCopyOfAttributeMap(); + for (String key : newAttributes.keySet()) { + Object newValue = newAttributes.get(key); + Object oldValue = existingSampleInfo.getAttributes().get(key); + + boolean oldEmpty = oldValue == null || oldValue.toString().isEmpty(); + boolean newEmpty = newValue == null || newValue.toString().isEmpty(); + if ((!oldEmpty && !newEmpty) && !newValue.equals(oldValue)) { + System.out.printf("Mismatch found in attributes of %s. Sample will be updated.%n", + openBisID); + updateSample(newSample, sampleID); + } + } + } + } + + // compare assets + Map newAssetsToFiles = nodeWithChildren.getISAFileToDatasetFiles(); + + //TODO follow creation of assets for assay, no way to be sure these are attached to similar samples + List assetsToCreate = new ArrayList<>(); + + Map sampleIDsWithEndpoints = new HashMap<>(); + for (ISASample sample : samplesToCreate) { + String sampleEndpoint = createSample(sample); + sampleIDsWithEndpoints.put(newSamplesWithReferences.get(sample), sampleEndpoint); + } + List assetsToUpload = new ArrayList<>(); + + for (GenericSeekAsset asset : assetsToCreate) { + assetsToUpload.add(createAsset(newAssetsToFiles.get(asset).getDataSetPermId().getPermId(), + asset)); + } + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for (AssetToUpload asset : assetsToUpload) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if (datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + + return new SeekStructurePostRegistrationInformation(assetsToUpload, sampleIDsWithEndpoints, + datasetIDsWithEndpoints); + } + private SampleInformation fetchSampleInformation(String sampleID) throws URISyntaxException, IOException, InterruptedException { String endpoint = apiURL+"/samples/"+sampleID; @@ -723,13 +835,21 @@ private JsonNode fetchAssayData(String assayID) public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference(); + if(nodeWithChildren.getAssayWithOpenBISReference().isEmpty()) { + throw new RuntimeException("No assay and openBIS reference found. Object has not been " + + "initialized using an assay object and openBIS experiment reference."); + } + + Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference().get(); String assayID = addAssay(assayIDPair.getKey()); String assayEndpoint = apiURL+"/assays/"+assayID; Pair experimentIDWithEndpoint = new ImmutablePair<>(assayIDPair.getValue(), assayEndpoint); + //wait for a bit, so we can be sure the assay that will be referenced by the samples has been created + Thread.sleep(3000); + Map sampleIDsWithEndpoints = new HashMap<>(); Map samplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); for(ISASample sample : samplesWithReferences.keySet()) { @@ -740,12 +860,12 @@ public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWit Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); - AssayWithQueuedAssets assayWithQueuedAssets = new AssayWithQueuedAssets(assayEndpoint, - createAssetsForAssays(isaToFileMap, Collections.singletonList(assayID))); + List assetsToUpload = createAssetsForAssays(isaToFileMap, + Collections.singletonList(assayID)); Map> datasetIDsWithEndpoints = new HashMap<>(); - for(AssetToUpload asset : assayWithQueuedAssets.getAssets()) { + for(AssetToUpload asset : assetsToUpload) { String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); String dsCode = asset.getDataSetCode(); if(datasetIDsWithEndpoints.containsKey(dsCode)) { @@ -755,8 +875,61 @@ public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWit List.of(endpointWithoutBlob))); } } - return new SeekStructurePostRegistrationInformation(assayWithQueuedAssets, - experimentIDWithEndpoint, sampleIDsWithEndpoints, datasetIDsWithEndpoints); + SeekStructurePostRegistrationInformation postRegInfo = + new SeekStructurePostRegistrationInformation(assetsToUpload, sampleIDsWithEndpoints, + datasetIDsWithEndpoints); + postRegInfo.setExperimentIDWithEndpoint(experimentIDWithEndpoint); + return postRegInfo; + } + + public SeekStructurePostRegistrationInformation createSampleWithAssets(SeekStructure nodeWithChildren) + throws URISyntaxException, IOException, InterruptedException { + Map sampleIDsWithEndpoints = new HashMap<>(); + Map samplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + for(ISASample sample : samplesWithReferences.keySet()) { + String sampleEndpoint = createSample(sample); + sampleIDsWithEndpoints.put(samplesWithReferences.get(sample), sampleEndpoint); + } + + Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); + + List assetsToUpload = createAssetsForAssays(isaToFileMap, new ArrayList<>()); + + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for(AssetToUpload asset : assetsToUpload) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if(datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + return new SeekStructurePostRegistrationInformation(assetsToUpload, sampleIDsWithEndpoints, + datasetIDsWithEndpoints); + } + + public SeekStructurePostRegistrationInformation createStandaloneAssets( + Map isaToFileMap) + throws IOException, URISyntaxException, InterruptedException { + + List assetsToUpload = createAssetsForAssays(isaToFileMap, new ArrayList<>()); + + Map> datasetIDsWithEndpoints = new HashMap<>(); + + for(AssetToUpload asset : assetsToUpload) { + String endpointWithoutBlob = blobEndpointToAssetURL(asset.getBlobEndpoint()); + String dsCode = asset.getDataSetCode(); + if(datasetIDsWithEndpoints.containsKey(dsCode)) { + datasetIDsWithEndpoints.get(dsCode).add(endpointWithoutBlob); + } else { + datasetIDsWithEndpoints.put(dsCode, new HashSet<>( + List.of(endpointWithoutBlob))); + } + } + return new SeekStructurePostRegistrationInformation(assetsToUpload, datasetIDsWithEndpoints); } public OpenbisSeekTranslator getTranslator() { @@ -797,25 +970,37 @@ public String getDataSetCode() { public class SeekStructurePostRegistrationInformation { - private final AssayWithQueuedAssets assayWithQueuedAssets; - private final Pair experimentIDWithEndpoint; + private final List assetsToUpload; + private Optional> experimentIDWithEndpoint; private final Map sampleIDsWithEndpoints; private final Map> datasetIDsWithEndpoints; - public SeekStructurePostRegistrationInformation(AssayWithQueuedAssets assayWithQueuedAssets, - Pair experimentIDWithEndpoint, Map sampleIDsWithEndpoints, + public SeekStructurePostRegistrationInformation(List assetsToUpload, + Map sampleIDsWithEndpoints, Map> datasetIDsWithEndpoints) { - this.assayWithQueuedAssets = assayWithQueuedAssets; - this.experimentIDWithEndpoint = experimentIDWithEndpoint; + this.assetsToUpload = assetsToUpload; this.sampleIDsWithEndpoints = sampleIDsWithEndpoints; this.datasetIDsWithEndpoints = datasetIDsWithEndpoints; + this.experimentIDWithEndpoint = Optional.empty(); + } + + public SeekStructurePostRegistrationInformation(List assetsToUpload, + Map> datasetIDsWithEndpoints) { + this.sampleIDsWithEndpoints = new HashMap<>(); + this.datasetIDsWithEndpoints = datasetIDsWithEndpoints; + this.assetsToUpload = assetsToUpload; + this.experimentIDWithEndpoint = Optional.empty(); + } + + public void setExperimentIDWithEndpoint(Pair experimentIDWithEndpoint) { + this.experimentIDWithEndpoint = Optional.of(experimentIDWithEndpoint); } - public AssayWithQueuedAssets getAssayWithQueuedAssets() { - return assayWithQueuedAssets; + public List getAssetsToUpload() { + return assetsToUpload; } - public Pair getExperimentIDWithEndpoint() { + public Optional> getExperimentIDWithEndpoint() { return experimentIDWithEndpoint; } diff --git a/src/main/java/life/qbic/model/isa/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java index 29caeef..858e30e 100644 --- a/src/main/java/life/qbic/model/isa/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -15,16 +16,28 @@ */ public class SeekStructure { - private final Pair assayAndOpenBISReference; + private final Optional> assayAndOpenBISReference; private final Map samplesWithOpenBISReference; private final Map isaToOpenBISFile; public SeekStructure(ISAAssay assay, String openBISReference) { - this.assayAndOpenBISReference = new ImmutablePair<>(assay, openBISReference); + this.assayAndOpenBISReference = Optional.of(new ImmutablePair<>(assay, openBISReference)); this.samplesWithOpenBISReference = new HashMap<>(); this.isaToOpenBISFile = new HashMap<>(); } + public SeekStructure(ISASample isaSample, String sampleID) { + this.samplesWithOpenBISReference = new HashMap<>(); + this.isaToOpenBISFile = new HashMap<>(); + this.assayAndOpenBISReference = Optional.empty(); + } + + public SeekStructure(Map isaToOpenBISFile) { + this.isaToOpenBISFile = isaToOpenBISFile; + this.samplesWithOpenBISReference = new HashMap<>(); + this.assayAndOpenBISReference = Optional.empty(); + } + public void addSample(ISASample sample, String openBISReference) { samplesWithOpenBISReference.put(sample, openBISReference); } @@ -33,7 +46,7 @@ public void addAsset(GenericSeekAsset asset, DataSetFile file) { isaToOpenBISFile.put(asset, file); } - public Pair getAssayWithOpenBISReference() { + public Optional> getAssayWithOpenBISReference() { return assayAndOpenBISReference; } From fc9b79c647548711b913426f435dcfe453444108 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 09:45:23 +0200 Subject: [PATCH 45/63] Create README.md --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..897ce1b --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +empty readme From b6ac491aa52ca28ad2539dd87a72af4645c10a05 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 09:52:55 +0200 Subject: [PATCH 46/63] always transfer experiment with samples --- .../TransferDataToSeekCommand.java | 26 ++++--- .../OpenbisExperimentWithDescendants.java | 1 - .../qbic/model/OpenbisSampleWithDatasets.java | 32 -------- .../qbic/model/OpenbisSeekTranslator.java | 3 + .../qbic/model/download/OpenbisConnector.java | 74 +++++++++++++++---- .../qbic/model/download/SEEKConnector.java | 3 + 6 files changed, 80 insertions(+), 59 deletions(-) delete mode 100644 src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 7c39790..4fef582 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -3,7 +3,6 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -19,18 +18,14 @@ import java.util.stream.Stream; import javax.xml.parsers.ParserConfigurationException; import life.qbic.App; -import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; -import life.qbic.model.OpenbisSampleWithDatasets; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; -import life.qbic.model.isa.GenericSeekAsset; import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; import life.qbic.model.download.SEEKConnector.AssetToUpload; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.tuple.Pair; import org.xml.sax.SAXException; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -152,13 +147,18 @@ public void run() { throw new RuntimeException(e); } SeekStructurePostRegistrationInformation postRegInfo; + OpenbisExperimentWithDescendants structure; try { + System.out.println("Collecting information from openBIS..."); if (isExperiment) { - postRegInfo = handleExperimentTransfer(); + structure = openbis.getExperimentWithDescendants(objectID); + postRegInfo = handleExperimentTransfer(structure); } else if (isSample) { - postRegInfo = handleSampleTransfer(); + structure = openbis.getExperimentAndDataFromSample(objectID); + postRegInfo = handleExperimentTransfer(structure); } else { - postRegInfo = handleDatasetTransfer(); + structure = openbis.getExperimentStructureFromDataset(objectID); + postRegInfo = handleExperimentTransfer(structure); } } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); @@ -170,10 +170,9 @@ public void run() { System.out.println("Done"); } - private SeekStructurePostRegistrationInformation handleExperimentTransfer() + private SeekStructurePostRegistrationInformation handleExperimentTransfer( + OpenbisExperimentWithDescendants experiment) throws URISyntaxException, IOException, InterruptedException { - System.out.println("Collecting information from openBIS..."); - OpenbisExperimentWithDescendants experiment = openbis.getExperimentWithDescendants(objectID); Set blacklist = parseBlackList(blacklistFile); System.out.println("Translating openBIS property codes to SEEK names..."); Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); @@ -196,10 +195,10 @@ private SeekStructurePostRegistrationInformation handleExperimentTransfer() return createNewAssayStructure(nodeWithChildren); } + /* private SeekStructurePostRegistrationInformation handleSampleTransfer() throws URISyntaxException, IOException, InterruptedException { System.out.println("Collecting information from openBIS..."); - OpenbisSampleWithDatasets sampleWithDatasets = openbis.getSampleWithDatasets(objectID); Set blacklist = parseBlackList(blacklistFile); System.out.println("Translating openBIS property codes to SEEK names..."); Map sampleTypesToIds = seek.getSampleTypeNamesToIDs(); @@ -235,6 +234,7 @@ private SeekStructurePostRegistrationInformation handleDatasetTransfer() return createNewAssetsForDataset(nodeWithChildren.getISAFileToDatasetFiles()); } + private SeekStructurePostRegistrationInformation updateSampleStructure( SeekStructure nodeWithChildren, String sampleID) throws URISyntaxException, IOException, InterruptedException { @@ -277,6 +277,8 @@ private SeekStructurePostRegistrationInformation createNewAssetsForDataset( return postRegistrationInformation; } + */ + private Set parseBlackList(String blacklistFile) { if(blacklistFile == null) { return new HashSet<>(); diff --git a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java index 0782d55..1c8baec 100644 --- a/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java +++ b/src/main/java/life/qbic/model/OpenbisExperimentWithDescendants.java @@ -1,6 +1,5 @@ package life.qbic.model; -import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; diff --git a/src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java b/src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java deleted file mode 100644 index 431d66c..0000000 --- a/src/main/java/life/qbic/model/OpenbisSampleWithDatasets.java +++ /dev/null @@ -1,32 +0,0 @@ -package life.qbic.model; - -import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; -import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; -import java.util.List; -import java.util.Map; - -public class OpenbisSampleWithDatasets { - - private Sample sample; - private List datasets; - private Map> datasetCodeToFiles; - - public OpenbisSampleWithDatasets(Sample sample, - List datasets, Map> datasetCodeToFiles) { - this.sample = sample; - this.datasets = datasets; - this.datasetCodeToFiles = datasetCodeToFiles; - } - - public Sample getSample() { - return sample; - } - - public List getDatasets() { - return datasets; - } - - public List getFilesForDataset(String permID) { - return datasetCodeToFiles.get(permID); - } -} diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index e1fc762..354ad66 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -218,6 +218,7 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, return result; } + /* public SeekStructure translate(OpenbisSampleWithDatasets sampleWithDatasets, Map sampleTypesToIds, Set blacklist, boolean transferData) { Sample sample = sampleWithDatasets.getSample(); @@ -281,6 +282,8 @@ public SeekStructure translate(Pair> da return new SeekStructure(assetToDatasetFiles); } + */ + private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { String permId = file.getDataSetPermId().getPermId(); for(DatasetWithProperties dataset : dataSets) { diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index f4906d6..2cf951c 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -3,9 +3,12 @@ import ch.ethz.sis.openbis.generic.OpenBIS; import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.search.SearchResult; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetType; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetTypeFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetSearchCriteria; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.search.DataSetTypeSearchCriteria; import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.update.DataSetUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind; import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId; @@ -51,11 +54,9 @@ import java.util.stream.Collectors; import life.qbic.model.DatasetWithProperties; import life.qbic.model.OpenbisExperimentWithDescendants; -import life.qbic.model.OpenbisSampleWithDatasets; import life.qbic.model.SampleTypeConnection; import life.qbic.model.SampleTypesAndMaterials; import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -109,6 +110,10 @@ public DataSetPermId registerDatasetForSample(Path uploadPath, String sampleID, private UploadedDataSetCreation prepareDataSetCreation(Path uploadPath, String datasetType, List parentCodes) { + if(listDatasetTypes().stream().map(DataSetType::getCode).noneMatch(x -> x.equals(datasetType))) { + throw new RuntimeException("Dataset type " + datasetType + + " is not supported by this instance of openBIS."); + } final String uploadId = openBIS.uploadFileWorkspaceDSS(uploadPath); final UploadedDataSetCreation creation = new UploadedDataSetCreation(); @@ -494,6 +499,14 @@ public List getDatasetFiles(DataSet dataset) { return result.getObjects(); } + public List listDatasetTypes() { + DataSetTypeSearchCriteria criteria = new DataSetTypeSearchCriteria(); + DataSetTypeFetchOptions fetchOptions = new DataSetTypeFetchOptions(); + fetchOptions.withPropertyAssignments().withPropertyType(); + fetchOptions.withPropertyAssignments().withEntityType(); + return openBIS.searchDataSetTypes(criteria, fetchOptions).getObjects(); + } + public SampleTypesAndMaterials getSampleTypesWithMaterials() { SampleTypeSearchCriteria criteria = new SampleTypeSearchCriteria(); SampleTypeFetchOptions typeOptions = new SampleTypeFetchOptions(); @@ -585,26 +598,27 @@ private void updateDatasetProperties(DataSetPermId id, Map prope boolean overwrite) { DataSetUpdate update = new DataSetUpdate(); update.setDataSetId(id); - if(overwrite) { + if (overwrite) { update.setProperties(properties); } else { DataSetFetchOptions options = new DataSetFetchOptions(); options.withProperties(); DataSet oldDataset = openBIS.getDataSets(Arrays.asList(id), options).get(id); - for(String property : properties.keySet()) { + for (String property : properties.keySet()) { String newValue = properties.get(property); String oldValue = oldDataset.getProperty(property); - if(oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { + if (oldValue == null || oldValue.isEmpty() || oldValue.equals(newValue)) { update.setProperty(property, newValue); } else { - update.setProperty(property, oldValue+", "+newValue);//TODO this can be changed to any other strategy + update.setProperty(property, + oldValue + ", " + newValue);//TODO this can be changed to any other strategy } } } openBIS.updateDataSets(Arrays.asList(update)); } - public OpenbisSampleWithDatasets getSampleWithDatasets(String sampleID) { + public OpenbisExperimentWithDescendants getExperimentAndDataFromSample(String sampleID) { SampleSearchCriteria criteria = new SampleSearchCriteria(); criteria.withIdentifier().thatEquals(sampleID); @@ -615,29 +629,61 @@ public OpenbisSampleWithDatasets getSampleWithDatasets(String sampleID) { fetchOptions.withProperties(); fetchOptions.withType().withPropertyAssignments().withPropertyType(); fetchOptions.withDataSetsUsing(dataSetFetchOptions); - fetchOptions.withDataSetsUsing(dataSetFetchOptions); - Sample sample = openBIS.searchSamples(criteria, fetchOptions).getObjects().get(0); + ExperimentFetchOptions expFetchOptions = new ExperimentFetchOptions(); + expFetchOptions.withType(); + expFetchOptions.withProject(); + expFetchOptions.withProperties(); + fetchOptions.withExperimentUsing(expFetchOptions); + + List samples = openBIS.searchSamples(criteria, fetchOptions).getObjects(); + Sample sample = samples.get(0); List datasets = new ArrayList<>(); Map> datasetCodeToFiles = new HashMap<>(); - for(DataSet dataset : sample.getDataSets()) { + for (DataSet dataset : sample.getDataSets()) { datasets.add(new DatasetWithProperties(dataset)); datasetCodeToFiles.put(dataset.getPermId().getPermId(), getDatasetFiles(dataset)); } - - return new OpenbisSampleWithDatasets(sample, datasets, datasetCodeToFiles); + return new OpenbisExperimentWithDescendants(sample.getExperiment(), samples, datasets, + datasetCodeToFiles); } - public Pair> getDataSetWithFiles(String datasetID) { + public OpenbisExperimentWithDescendants getExperimentStructureFromDataset(String datasetID) { DataSetSearchCriteria criteria = new DataSetSearchCriteria(); criteria.withPermId().thatEquals(datasetID); + SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); + sampleFetchOptions.withProperties(); + sampleFetchOptions.withType().withPropertyAssignments().withPropertyType(); + + ExperimentFetchOptions expFetchOptions = new ExperimentFetchOptions(); + expFetchOptions.withType(); + expFetchOptions.withProject(); + expFetchOptions.withProperties(); + DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); dataSetFetchOptions.withType(); dataSetFetchOptions.withRegistrator(); + dataSetFetchOptions.withSampleUsing(sampleFetchOptions); + dataSetFetchOptions.withExperimentUsing(expFetchOptions); DataSet dataset = openBIS.searchDataSets(criteria, dataSetFetchOptions).getObjects().get(0); - return new ImmutablePair<>(new DatasetWithProperties(dataset), getDatasetFiles(dataset)); + + List samples = new ArrayList<>(); + if(dataset.getSample() != null) { + samples.add(dataset.getSample()); + } + + List datasets = new ArrayList<>(); + Map> datasetCodeToFiles = new HashMap<>(); + datasets.add(new DatasetWithProperties(dataset)); + datasetCodeToFiles.put(dataset.getPermId().getPermId(), getDatasetFiles(dataset)); + + if(dataset.getExperiment() == null) { + System.err.println("No experiment found for dataset "+datasetID); + } + return new OpenbisExperimentWithDescendants(dataset.getExperiment(), samples, datasets, + datasetCodeToFiles); } } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 10195bf..152dc5a 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -882,6 +882,7 @@ public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWit return postRegInfo; } + /* public SeekStructurePostRegistrationInformation createSampleWithAssets(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { Map sampleIDsWithEndpoints = new HashMap<>(); @@ -932,6 +933,8 @@ public SeekStructurePostRegistrationInformation createStandaloneAssets( return new SeekStructurePostRegistrationInformation(assetsToUpload, datasetIDsWithEndpoints); } + */ + public OpenbisSeekTranslator getTranslator() { return translator; } From 91b3b5340345fcb68dfd464d2a2c308b4ee8b263 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 09:53:29 +0200 Subject: [PATCH 47/63] allow for slightly more variable metaInformation.yaml names --- src/main/java/life/qbic/io/PetabParser.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index 25461ed..e4aeb93 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -18,7 +18,7 @@ public class PetabParser { - private final String META_INFO_YAML = "metaInformation.yaml"; + private final String META_INFO_YAML_NAME = "metaInformation"; public PetabMetadata parse(String dataPath) { @@ -84,14 +84,15 @@ public void addDatasetId(String outputPath, String datasetCode) throws IOExcepti private File findYaml(File directory) { for (File file : Objects.requireNonNull(directory.listFiles())) { - if (file.isFile() && file.getName().equalsIgnoreCase(META_INFO_YAML)) { + String fileName = file.getName(); + if (file.isFile() && fileName.contains(META_INFO_YAML_NAME) && fileName.endsWith(".yaml")) { return file; } if (file.isDirectory()) { return findYaml(file); } } - System.out.println(META_INFO_YAML + " not found"); + System.out.println(META_INFO_YAML_NAME + " yaml not found."); return null; } From a1382c4d78e30da06cc7f7a760d5c54c4d08807d Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 09:54:13 +0200 Subject: [PATCH 48/63] add description for each command --- .../io/commandline/DownloadPetabCommand.java | 19 +++++++++++---- .../io/commandline/FindDatasetsCommand.java | 14 +++++++---- .../commandline/SampleHierarchyCommand.java | 23 +++++++++++++------ .../commandline/SpaceStatisticsCommand.java | 17 +++++++++++--- .../io/commandline/UploadDatasetCommand.java | 11 +++++++++ .../commandline/UploadPetabResultCommand.java | 22 +++++++++++++++--- 6 files changed, 85 insertions(+), 21 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java index 5deac91..2b0f972 100644 --- a/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java +++ b/src/main/java/life/qbic/io/commandline/DownloadPetabCommand.java @@ -5,7 +5,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Set; import life.qbic.App; import life.qbic.io.PetabParser; @@ -15,13 +14,25 @@ import picocli.CommandLine.Mixin; import picocli.CommandLine.Parameters; +/** + * The Download PEtab command can be used to download a PEtab Dataset from openBIS and store some + * additional information from openBIS in the metaInformation.yaml file (or a respective yaml file + * containing 'metaInformation' in its name). + * The Dataset to download is specified by providing the openBIS dataset identifier (code) and the + * PEtab is downloaded to the download path parameter provided. + * By design, the Dataset Identifier is added to the downloaded metaInformation.yaml as 'openBISId' + * in order to keep track of the source of this PEtab. + */ @Command(name = "download-petab", - description = "Downloads PEtab dataset and stores some additional information from openbis in the petab.yaml") + description = "Downloads PEtab dataset and stores some additional information from openbis in " + + "the petab.yaml") public class DownloadPetabCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset to download. Can be found via list-data.") + @Parameters(arity = "1", paramLabel = "dataset id", description = "The code of the dataset to " + + "download. Can be found via list-data.") private String datasetCode; - @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to store the downloaded data") + @Parameters(arity = "1", paramLabel = "download path", description = "The local path where to " + + "store the downloaded data") private String outputPath; @Mixin OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index 3f9f747..b1644a6 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -19,16 +19,22 @@ import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; +/** + * List Data + * The list-data command can be used to list Datasets in openBIS and some of their metadata based + * on the experiment or sample they are attached to. Experiments or samples are specified by their + * openBIS code or identifier. + * The optional 'space' parameter can be used to only show datasets found in the provided space. + */ @Command(name = "list-data", description = "lists datasets and their details for a given experiment code") public class FindDatasetsCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "openBIS obejct", description = - "The code of the experiment " - + "or sample data is attached to.") + @Parameters(arity = "1", paramLabel = "openBIS object", description = + "The code of the experiment or sample data is attached to.") private String objectCode; @Option(arity = "1", paramLabel = "", description = "Optional openBIS spaces to filter " - + "found datasets by", names = {"-s", "--space"}) + + "found datasets by.", names = {"-s", "--space"}) private String space; @Mixin OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); diff --git a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java index fbd66b8..04fd913 100644 --- a/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java +++ b/src/main/java/life/qbic/io/commandline/SampleHierarchyCommand.java @@ -22,13 +22,22 @@ import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; +/** + * The Sample Types command queries all sample types and prints which types are connected and how + * often (via samples existing in the queried openBIS instance), creating a sample type hierarchy. + * The --space command can be used to only show the sample-types used in a specific openBIS space. + * An output file for the resulting hierarchy can be specified using the --out command. + */ @Command(name = "sample-types", - description = "lists sample types with children sample types and how often they are found in the openbis instance") + description = "lists sample types with children sample types and how often they are found in " + + "the openbis instance") public class SampleHierarchyCommand implements Runnable { - @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) + @Option(arity = "1", paramLabel = "", description = "optional openBIS space to filter " + + "samples", names = {"-s", "--space"}) private String space; - @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) + @Option(arity = "1", paramLabel = "", description = "optional output path", + names = {"-o", "--out"}) private String outpath; @Mixin OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); @@ -44,8 +53,8 @@ public void run() { } else { summary.add("Querying samples in all available spaces..."); } - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), - auth.getOpenbisAS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), + auth.getOpenbisUser(), auth.getOpenbisAS()); OpenbisConnector openbis = new OpenbisConnector(authentication); Map hierarchy = openbis.queryFullSampleHierarchy(spaces); @@ -57,7 +66,7 @@ public void run() { System.out.println(s); } Path outputPath = Paths.get(Configuration.LOG_PATH.toString(), - "sample_model_summary"+getTimeStamp()+".txt"); + "sample_model_summary" + getTimeStamp() + ".txt"); if(outpath!=null) { outputPath = Paths.get(outpath); } @@ -72,6 +81,6 @@ public void run() { private String getTimeStamp() { final String PATTERN_FORMAT = "YYYY-MM-dd_HHmmss"; DateTimeFormatter formatter = DateTimeFormatter.ofPattern(PATTERN_FORMAT); - return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter).toString(); + return LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC).format(formatter); } } diff --git a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java index 5e66d24..e4cb97e 100644 --- a/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java +++ b/src/main/java/life/qbic/io/commandline/SpaceStatisticsCommand.java @@ -24,16 +24,27 @@ import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; +/** + * The Statistics command can be used to list the number of collections, sample objects and attached + * datasets by type for one or all spaces accessible by the user. + * The --space command can be used to only show the objects in a specific openBIS space. + * An output file for the resulting list can be specified using the --out command. + * By default, openBIS settings objects and material spaces are ignored. This can be overwritten + * using --show-settings. + */ @Command(name = "statistics", description = "lists the number of collections, sample objects and attached datasets (by type)" + "for one or all spaces accessible by the user") public class SpaceStatisticsCommand implements Runnable { - @Option(arity = "1", paramLabel = "", description = "optional openBIS spaces to filter samples", names = {"-s", "--space"}) + @Option(arity = "1", paramLabel = "", description = "optional openBIS space to filter " + + "samples", names = {"-s", "--space"}) private String space; - @Option(arity = "1", paramLabel = "", description = "optional output path", names = {"-o", "--out"}) + @Option(arity = "1", paramLabel = "", description = "optional output path", + names = {"-o", "--out"}) private String outpath; - @Option(arity = "0", description = "shows results for openBIS settings and material spaces. Ignored if a specific space is selected.", + @Option(arity = "0", description = "shows results for openBIS settings and material spaces. " + + "Ignored if a specific space is selected.", names = {"--show-settings"}) private boolean allSpaces; @Mixin diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index c9756fd..3cabf78 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -13,6 +13,17 @@ import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; +/** + * The Upload Dataset command can be used to upload a Dataset to openBIS and connect it to existing + * datasets. + * To upload a dataset, the path to the file or folder and the object ID to which it should + * be attached need to be provided. Objects can be experiments or samples. + * Parent datasets can be specified using the --parents command. + * If the specified object ID or any of the specified parent datasets cannot be found, the script + * will stop and return an error message. + * The dataset type of the new dataset in openBIS can be specified using the --type option, + * otherwise the type "UNKNOWN" will be used. + */ @Command(name = "upload-data", description = "uploads a dataset and attaches it to an experiment and (optionally) other datasets") public class UploadDatasetCommand implements Runnable { diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index bffdc1e..55ec1fa 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -4,7 +4,6 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId; import java.io.File; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import life.qbic.App; import life.qbic.io.PetabParser; @@ -14,12 +13,29 @@ import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; +/** + * The Upload PEtab command can be used to upload a PEtab Dataset to openBIS and connect it to its + * source files if these are stored in the same openBIS instance and referenced in the PEtabs meta- + * data. + * To upload a PEtab dataset, the path to the PEtab folder and the experiment ID to which it should + * be attached need to be provided. + * The dataset type of the new dataset in openBIS can be specified using the --type option, + * otherwise the type "UNKNOWN" will be used. + * The script will search the metaInformation.yaml for the entry "openBISSourceIds:" and attach the + * new dataset to all the datasets with ids in the following blocks found in this instance of + * openBIS: + * openBISSourceIds: + * - 20210702093837370-184137 + * - 20220702100912333-189138 + * If one or more dataset identifiers are not found, the script will stop without uploading the data + * and inform the user. + */ @Command(name = "upload-petab", description = "uploads a PETab folder and attaches it to a provided experiment and any datasets " + "referenced in the PETab metadata (e.g. for PETab results).") public class UploadPetabResultCommand implements Runnable { - @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder " + @Parameters(arity = "1", paramLabel = "PEtab folder", description = "The path to the PEtab folder " + "to upload") private String dataPath; @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the " @@ -33,7 +49,7 @@ public class UploadPetabResultCommand implements Runnable { OpenbisAuthenticationOptions auth = new OpenbisAuthenticationOptions(); private OpenbisConnector openbis; - private PetabParser petabParser = new PetabParser(); + private final PetabParser petabParser = new PetabParser(); @Override public void run() { From e1e663eefbcdf7d2e00076cd2f2c73594dec6bff Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 12:55:09 +0200 Subject: [PATCH 49/63] provide documentation for several commands --- README.md | 155 +++++++++++++++++- .../io/commandline/FindDatasetsCommand.java | 7 +- 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 897ce1b..fbffa2b 100644 --- a/README.md +++ b/README.md @@ -1 +1,154 @@ -empty readme +# OpenBIS Scripts + +## User Documentation + +### OpenBIS Statistics and Data Model + +In order to interact with openBIS, the parameters for the application server URL, the user and their password, have to be provided. + +In order to interact with data in openBIS (upload or download), a data store (dss) URL needs to be provided, as well. + +Everything but the password can be provided via the config file (config.txt): + +* as=https://my-openbis-instance.de/openbis/openbis +* dss=https://my-openbis-instance.de/datastore_server +* user=your-username + +Keep in mind that you have to edit the config file or provide these parameters via command line, if you want to use different users or connect to different openBIS instances. + +Refer to the help of the respective command or the examples below for more details. + +#### Finding Datasets + +The list-data command can be used to list Datasets in openBIS and some of their metadata based on the experiment or sample they are attached to. Experiments or samples are specified by their openBIS code or identifier. + +The optional 'space' parameter can be used to only show datasets found in the provided space. + +Example command: + +`java -jar scripts.jar list-data /SPACY/PROJECTX/TEST_PATIENTS1 -config config.txt --openbis-pw` + +Example output: + + reading config + Querying experiment in all available spaces... + Query is not an object code, querying for: TEST_PATIENTS1 instead. + [main] INFO org.eclipse.jetty.util.log - Logging initialized @6997ms to org.eclipse.jetty.util.log.Slf4jLog + Found 4 datasets for experiment TEST_PATIENTS1: + [1] + patientIDs: ID 4930-72,ID 4931-79 + ID: 20241014000813459-189089 (/TEST_PATIENTS1) + Type: UNKNOWN + Uploaded by Friedrich Andreas (10-14-24 20:58:13) + + [2] + patientIDs: ID 4930-72,ID 4931-79 + ID: 20241000010001025-189090 (/SPACY/RPOJECTX/TEST_PATIENTS1) + Type: UNKNOWN + Uploaded by Friedrich Andreas (10-14-24 21:00:01) + +#### Showing Space Statistics + +The Statistics command can be used to list the number of collections, sample objects and attached datasets by type for one or all spaces accessible by the user. + +The --space command can be used to only show the objects in a specific openBIS space. + +An output file for the resulting list can be specified using the --out command. + +By default, openBIS settings objects and material spaces are ignored. This can be overwritten using --show-settings. + +Example command: + +`java -jar scripts.jar statistics -config config.txt --openbis-pw` + +Example output: + + Querying samples in all available spaces... + ----- + Summary for TEMP_PLAYGROUND + ----- + Experiments (9): + + 02_MASSSPECTROMETRY_EXPERIMENT: 1 + 00_STANDARD_OPERATING_PROTOCOLS: 1 + 00_PATIENT_DATABASE: 3 + 01_BIOLOGICAL_EXPERIMENT: 4 + + Samples (316): + + 00_PRIMARY_BLOOD: 128 + 03_SDS_PAGE_SETUP: 1 + 00_PRIMARY_TISSUE: 24 + PLASMID: 161 + 02_EXPERIMENT_TREATMENT: 2 + + Attached datasets (30): + + IB_DATA: 4 + SOURCE_CODE: 2 + RAW_DATA: 1 + EXPERIMENT_RESULT: 3 + MS_DATA_RAW: 1 + UNKNOWN: 18 + EXPERIMENT_PROTOCOL: 1 + +#### Showing Sample Hierarchy + +The Sample Types command queries all sample types and prints which types are connected and how often (via samples existing in the queried openBIS instance), creating a sample type hierarchy. + +The --space option can be used to only show the sample-types used in a specific openBIS space. + +An output file for the resulting hierarchy can be specified using the --out command. + +Example command: + +`java -jar scripts.jar statistics -config config.txt --openbis-pw` + +Example output: + + Querying samples in all available spaces... + MATERIAL.CHEMICALS (1) + PATIENT_ID (1) + PATIENT_ID -> PATIENT_ID (1) + 05_MS_RUN (1) + 00_PATIENT_INFO -> 01_EXPERIMENT_PRIMARYCULTURE (3) + 04_IMMUNOBLOT (4) + 03_SDS_PAGE_SETUP -> 04_IMMUNOBLOT (4) + +### Upload/Download and Interaction with PEtab + +#### Uploading general data + +The Upload Dataset command can be used to upload a Dataset to openBIS and connect it to existing datasets. + +To upload a dataset, the path to the file or folder and the object ID to which it should be attached need to be provided. Objects can be experiments or samples. + +Parent datasets can be specified using the --parents command. + +If the specified object ID or any of the specified parent datasets cannot be found, the script will stop and return an error message. + +The dataset type of the new dataset in openBIS can be specified using the --type option, otherwise the type "UNKNOWN" will be used. + +Example command: + +`java -jar scripts.jar upload-data README.md /SPACY/PROJECTX/MY_SAMPLE -t ATTACHMENT -config config.txt --openbis-pw` + +Example output: + + Parameters verified, uploading dataset... + + Dataset 20241021125328024-689105 was successfully attached to experiment` + +#### Downloading a PEtab dataset + +#### Uploading a PEtab dataset + +### Interaction with SEEK instances + +#### Transferring Sample Types to SEEK + +#### Transferring openBIS objects and files to SEEK + +#### Updating nodes in SEEK based on updates in openBIS + +### Caveats and Future Options \ No newline at end of file diff --git a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java index b1644a6..d359a18 100644 --- a/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java +++ b/src/main/java/life/qbic/io/commandline/FindDatasetsCommand.java @@ -66,7 +66,7 @@ public void run() { List datasets = new ArrayList<>(); if(!datasetsOfExp.isEmpty()) { - System.err.println("Found "+datasetsOfExp.size()+" datasets for experiment "+objectCode); + System.out.printf("Found %s datasets for experiment %s:%n", datasetsOfExp.size(), objectCode); datasets.addAll(datasetsOfExp); } List datasetsOfSample = openbis.listDatasetsOfSample(spaces, objectCode).stream() @@ -75,7 +75,8 @@ public void run() { .collect(Collectors.toList()); if(!datasetsOfSample.isEmpty()) { - System.err.println("Found "+datasetsOfSample.size()+" datasets for sample "+objectCode); + System.out.printf("Found %s datasets for sample %s:%n", datasetsOfSample.size(), objectCode); + datasets.addAll(datasetsOfSample); } @@ -95,8 +96,6 @@ public void run() { return ds; }).collect(Collectors.toList()); int datasetIndex = 0; - System.out.println(); - System.out.printf("Found %s datasets for %s:%n", datasets.size(), objectCode); for (DatasetWithProperties dataSet : datasetWithProperties) { datasetIndex++; System.out.println("["+datasetIndex+"]"); From 9b59e2c19f9f8dd17c29d2c3067812393432ffca Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 16:05:55 +0200 Subject: [PATCH 50/63] update readme to latest features --- README.md | 186 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 157 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index fbffa2b..3f2642b 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,12 @@ -# OpenBIS Scripts +# OpenBIS Scripts User Documentation -## User Documentation - -### OpenBIS Statistics and Data Model +## OpenBIS Statistics and Data Model In order to interact with openBIS, the parameters for the application server URL, the user and their password, have to be provided. In order to interact with data in openBIS (upload or download), a data store (dss) URL needs to be provided, as well. -Everything but the password can be provided via the config file (config.txt): +**Everything but the password can be provided via the config file (config.txt):** * as=https://my-openbis-instance.de/openbis/openbis * dss=https://my-openbis-instance.de/datastore_server @@ -18,17 +16,17 @@ Keep in mind that you have to edit the config file or provide these parameters v Refer to the help of the respective command or the examples below for more details. -#### Finding Datasets +### Finding Datasets The list-data command can be used to list Datasets in openBIS and some of their metadata based on the experiment or sample they are attached to. Experiments or samples are specified by their openBIS code or identifier. The optional 'space' parameter can be used to only show datasets found in the provided space. -Example command: +**Example command:** `java -jar scripts.jar list-data /SPACY/PROJECTX/TEST_PATIENTS1 -config config.txt --openbis-pw` -Example output: +**Example output:** reading config Querying experiment in all available spaces... @@ -47,7 +45,7 @@ Example output: Type: UNKNOWN Uploaded by Friedrich Andreas (10-14-24 21:00:01) -#### Showing Space Statistics +### Showing Space Statistics The Statistics command can be used to list the number of collections, sample objects and attached datasets by type for one or all spaces accessible by the user. @@ -57,11 +55,11 @@ An output file for the resulting list can be specified using the --out command. By default, openBIS settings objects and material spaces are ignored. This can be overwritten using --show-settings. -Example command: +**Example command:** `java -jar scripts.jar statistics -config config.txt --openbis-pw` -Example output: +**Example output:** Querying samples in all available spaces... ----- @@ -92,7 +90,7 @@ Example output: UNKNOWN: 18 EXPERIMENT_PROTOCOL: 1 -#### Showing Sample Hierarchy +### Showing Sample Hierarchy The Sample Types command queries all sample types and prints which types are connected and how often (via samples existing in the queried openBIS instance), creating a sample type hierarchy. @@ -100,11 +98,11 @@ The --space option can be used to only show the sample-types used in a specific An output file for the resulting hierarchy can be specified using the --out command. -Example command: +**Example command:** `java -jar scripts.jar statistics -config config.txt --openbis-pw` -Example output: +**Example output:** Querying samples in all available spaces... MATERIAL.CHEMICALS (1) @@ -115,40 +113,170 @@ Example output: 04_IMMUNOBLOT (4) 03_SDS_PAGE_SETUP -> 04_IMMUNOBLOT (4) -### Upload/Download and Interaction with PEtab +## Upload/Download and Interaction with PEtab -#### Uploading general data +### Uploading general data -The Upload Dataset command can be used to upload a Dataset to openBIS and connect it to existing datasets. +The Upload Dataset command can be used to upload a Dataset to openBIS and connect it to existing +datasets. -To upload a dataset, the path to the file or folder and the object ID to which it should be attached need to be provided. Objects can be experiments or samples. +To upload a dataset, the path to the file or folder and the object ID to which it should be attached +need to be provided. Objects can be experiments or samples. Parent datasets can be specified using the --parents command. -If the specified object ID or any of the specified parent datasets cannot be found, the script will stop and return an error message. +If the specified object ID or any of the specified parent datasets cannot be found, the script will +stop and return an error message. -The dataset type of the new dataset in openBIS can be specified using the --type option, otherwise the type "UNKNOWN" will be used. +The dataset type of the new dataset in openBIS can be specified using the --type option, otherwise +the type "UNKNOWN" will be used. -Example command: +**Example command:** `java -jar scripts.jar upload-data README.md /SPACY/PROJECTX/MY_SAMPLE -t ATTACHMENT -config config.txt --openbis-pw` -Example output: +**Example output:** Parameters verified, uploading dataset... Dataset 20241021125328024-689105 was successfully attached to experiment` -#### Downloading a PEtab dataset +### Downloading a PEtab dataset + +The Download PEtab command can be used to download a PEtab Dataset from openBIS and store some +additional information from openBIS in the metaInformation.yaml file (or a respective yaml file +containing 'metaInformation' in its name). + +The Dataset to download is specified by providing the openBIS dataset identifier (code) and the +PEtab is downloaded to the download path parameter provided. + +By design, the Dataset Identifier is added to the downloaded metaInformation.yaml as 'openBISId' +in order to keep track of the source of this PEtab. + +### Uploading a PEtab dataset + +The Upload PEtab command can be used to upload a PEtab Dataset to openBIS and connect it to its +source files if these are stored in the same openBIS instance and referenced in the PEtabs metadata. + +To upload a PEtab dataset, the path to the PEtab folder and the experiment ID to which it should be +attached need to be provided. + +The dataset type of the new dataset in openBIS can be specified using the --type option, otherwise +the type "UNKNOWN" will be used. + +The script will search the **metaInformation.yaml** for the entry "**openBISSourceIds:**" and attach +the new dataset to all the datasets with ids in the following blocks found in this instance of +openBIS: + + openBISSourceIds: + - 20210702093837370-184137 + - 20220702100912333-189138 + +If one or more dataset identifiers are not found, the script will stop without uploading the data +and inform the user. + +## Interaction with SEEK instances + +In order to interact with SEEK, the parameters for the server URL, the user (usually an email +address) and their password, have to be provided. + +In order to interact with openBIS (transfer of data and metadata), the respective credentials need +to be provided, as well. + +**Everything but the passwords can be provided via the config file (config.txt):** + +* as=https://my-openbis-instance.de/openbis/openbis +* dss=https://my-openbis-instance.de/datastore_server +* user=your-openbis-username +* seek_user=your@email.de +* seek_url=http://localhost:3000 + +**Furthermore, names of default project and investigation in SEEK can be provided:** + +* seek_default_project=seek_test +* seek_default_investigation=default_investigation + +In order to keep track of samples transferred from openBIS, the script will try to transfer the +openBIS identifier of each sample to an additional SEEK sample type attribute (more details in the +section **Transferring Sample Types to SEEK**). + +**Its name can be specified in the config:** + +* seek_openbis_sample_title=openBIS Name + +Refer to the help of the respective command or the examples below for more details. + +### Transferring Sample Types to SEEK + +The Sample Type Transfer command transfers sample types and their attributes from an openBIS to a +SEEK instance. + +In order to do this, a mapping of the respective data types/sample attribute types needs to be +specified. This can be found (and changed) in the provided property file +**openbis_datatype_to_seek_attributetype.xml**: + + + 3 + Real number + Float + + +Each **entry type** denotes the **data type** in openBIS. Note that SEEK needs the identifier of the +respective sample attribute type (here: *3* for *Float*) in its database. Any changes to the file +need to reflect this. + +The available types and their identifiers can be queried at the endpoint **sample_attribute_types**. +For example: +`http://localhost:3000/sample_attribute_types` + +When transferring openBIS sample types, the command will automatically add a mandatory title +attribute to the sample type in SEEK. This title will be filled with the identifier of the openBIS +**sample object** (not sample type!) will be added. The attribute name is specified in the config +file and should selected before sample types are transferred to the respective instance: +* seek_openbis_sample_title=openBIS Name + +By default, only sample types (not samples!) with names not already found in SEEK will be +transferred and the user will be informed if duplicates are found. The option **--ignore-existing** +can be used to transfer existing sample types a second time, although it is recommended to only use +this option for testing purposes. + +### Transferring openBIS objects and files to SEEK + +The OpenBIS to Seek command transfers metadata and (optionally) data from openBIS to SEEK. +Experiments, samples and dataset information are always transferred together (as assays, samples and +one of several **asset types** in SEEK). + +The script will try to find the provided **openbis ID** in experiments, samples or datasets and +fetch any missing information to create a SEEK node containing at least one assay (when an +experiment without samples and datasets is specified). + +The seek-study needs to be provides to attach the assay. TODO: This information is also used to +decide if the node(s) should be updated (if they exist for the provided study) or created anew. + +Similarly, the title of the project in SEEK where nodes should be added, can either be provided via +the config file as **'seek_default_project'** or via the command line using the **--seek-project** +option. -#### Uploading a PEtab dataset +Info in the created asset(s) always links back to the openBIS path of the respective dataset. +The data itself can be transferred and stored in SEEK using the '-d' flag. -### Interaction with SEEK instances +To completely exclude some dataset information from being transferred, a file ('--blacklist') +containing dataset codes (from openBIS) can be specified. //TODO do this for samples/sample types -#### Transferring Sample Types to SEEK +### Updating nodes in SEEK based on updates in openBIS -#### Transferring openBIS objects and files to SEEK +Updating nodes in SEEK uses the same general command, parameters and options. Unless otherwise +specified (**--no-update** flag), the command will try to update existing nodes in SEEK (recognized +by openBIS identifiers in their metadata, as well as the provided study name). -#### Updating nodes in SEEK based on updates in openBIS +The updating of a node-structure is done based on the following rules: +1. if an assay contains the openBIS permID of the experiment AND is attached to specified study, +its samples and assets are updated +2. samples are created, if the openBIS name of the respective sample is not found in a sample +attached to the assay in question +3. samples are updated, if their openBIS name is found in a sample attached to the asset and at +least one sample attribute is different in openBIS and SEEK +4. assets attached to the experiment or samples will be created, if they are missing from this assay +5. no existing sample or assets are deleted from SEEK, even if they are missing from openBIS -### Caveats and Future Options \ No newline at end of file +## Caveats and Future Options From 6365958935bfc6761bbe9a05601922a23d74b0d5 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 16:06:17 +0200 Subject: [PATCH 51/63] make study mandatory --- .../life/qbic/io/commandline/TransferDataToSeekCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index 4fef582..a36c056 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -48,8 +48,8 @@ public class TransferDataToSeekCommand implements Runnable { @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + "experiment, sample or dataset to transfer.") private String objectID; - @Option(names = "--seek-study", description = "Title of the study in SEEK where nodes should be " - + "added. Mandatory for assay creation.") + @Parameters(arity = "1", paramLabel = "seek-study", description = "Title of the study in SEEK where " + + "nodes should be added. Mandatory, as an assay is always needed and attached to a study.") private String studyTitle; @Option(names = "--seek-project", description = "Title of the project in SEEK where nodes should" + "be added. Can alternatively be provided via the config file as 'seek_default_project'.") From 225af2ca9ccc881607cec0b6a9d24521534e28a3 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 18:05:02 +0200 Subject: [PATCH 52/63] fix addition of openbis ID if no keyword is part of yaml yet --- src/main/java/life/qbic/io/PetabParser.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/life/qbic/io/PetabParser.java b/src/main/java/life/qbic/io/PetabParser.java index e4aeb93..ce83c50 100644 --- a/src/main/java/life/qbic/io/PetabParser.java +++ b/src/main/java/life/qbic/io/PetabParser.java @@ -73,12 +73,22 @@ public void addDatasetId(String outputPath, String datasetCode) throws IOExcepti Path path = Paths.get(Objects.requireNonNull(findYaml(new File(outputPath))).getPath()); Charset charset = StandardCharsets.UTF_8; - final String keyWord = "openBISId"; + final String idKeyWord = "openBISId"; - String idInLine = keyWord+":(.*)?(\\r\\n|[\\r\\n])"; + final String endOfLine = ":(.*)?(\\r\\n|[\\r\\n])"; + final String idInLine = idKeyWord+endOfLine; String content = Files.readString(path, charset); - content = content.replaceAll(idInLine, keyWord+": "+datasetCode+"\n"); + // existing property found, fill/replace with relevant dataset code + if(content.contains(idKeyWord)) { + content = content.replaceAll(idInLine, idKeyWord+": "+datasetCode+"\n"); + // no existing property found, create it above the dateOfExperiment property + } else { + String dateKeyword = "dateOfExperiment"; + String dateLine = dateKeyword+endOfLine; + String newLines = idKeyWord+": "+datasetCode+"\n "+dateKeyword+":\n"; + content = content.replaceAll(dateLine, newLines); + } Files.write(path, content.getBytes(charset)); } From b63c22ed7d4f02162d7caba8f9a455abbf267ab8 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 18:05:23 +0200 Subject: [PATCH 53/63] clean up print --- src/main/java/life/qbic/model/download/SEEKConnector.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 152dc5a..dd35eea 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -137,8 +137,6 @@ public String addAssay(ISAAssay assay) .send(buildAuthorizedPOSTRequest(endpoint, assay.toJson()), BodyHandlers.ofString()); - System.err.println(assay.toJson()); - if(response.statusCode()!=200) { throw new RuntimeException("Failed : HTTP error code : " + response.statusCode()); } From 7de18ad745df9f6e6cbece048fba998bfb36cd5b Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 18:05:58 +0200 Subject: [PATCH 54/63] prepare for addition of link sample to store SEEK links --- .../qbic/model/download/OpenbisConnector.java | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 2cf951c..3dd134e 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -19,6 +19,7 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.update.ExperimentUpdate; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.SampleType; +import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.create.SampleCreation; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleTypeFetchOptions; import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier; @@ -536,15 +537,51 @@ public SampleTypesAndMaterials getSampleTypesWithMaterials() { public void createSeekLinks(SeekStructurePostRegistrationInformation postRegInformation) { Optional> experimentInfo = postRegInformation.getExperimentIDWithEndpoint(); + //TODO link sample type not implemented? + final String SAMPLE_TYPE = "EXTERNAL_LINK"; + + SampleTypeSearchCriteria criteria = new SampleTypeSearchCriteria(); + criteria.withCode().thatEquals(SAMPLE_TYPE); + SampleTypeFetchOptions typeOptions = new SampleTypeFetchOptions(); + typeOptions.withPropertyAssignments().withPropertyType(); + typeOptions.withPropertyAssignments().withEntityType(); + if(openBIS.searchSampleTypes(criteria, typeOptions).getObjects().size() == 0) { + System.out.printf( + "This is where links would be put into openBIS, but EXTERNAL_LINK sample was " + + "not yet added to openBIS instance.%n"); + return; + } + if(experimentInfo.isPresent()) { ExperimentIdentifier id = new ExperimentIdentifier(experimentInfo.get().getLeft()); String endpoint = experimentInfo.get().getRight(); - Map props = new HashMap<>(); - props.put(EXPERIMENT_LINK_PROPERTY, endpoint); - updateExperimentProperties(id, props, false); + SampleCreation sample = createNewLinkSample(endpoint); + sample.setExperimentId(id); + openBIS.createSamples(Arrays.asList(sample)); + } + Map sampleInfos = postRegInformation.getSampleIDsWithEndpoints(); + for(String sampleID : sampleInfos.keySet()) { + SampleIdentifier id = new SampleIdentifier(sampleID); + String endpoint = sampleInfos.get(sampleID); + SampleCreation sample = createNewLinkSample(endpoint); + sample.setParentIds(Arrays.asList(id)); + openBIS.createSamples(Arrays.asList(sample)); } } + private SampleCreation createNewLinkSample(String endpoint) { + final String SAMPLE_TYPE = "EXTERNAL_LINK"; + SampleCreation sample = new SampleCreation(); + sample.setTypeId(new EntityTypePermId(SAMPLE_TYPE, EntityKind.SAMPLE)); + + Map properties = new HashMap<>(); + properties.put("LINK_TYPE", "SEEK"); + properties.put("URL", endpoint); + + sample.setProperties(properties); + return sample; + } + public void updateSeekLinks(SeekStructurePostRegistrationInformation postRegistrationInformation) { } From f8ec586b76f42be6976a8cc4e8078e28bdd1e3f0 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 18:06:24 +0200 Subject: [PATCH 55/63] allow uploading petab to sample objects --- .../commandline/UploadPetabResultCommand.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java index 55ec1fa..1e7d5b8 100644 --- a/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadPetabResultCommand.java @@ -17,7 +17,7 @@ * The Upload PEtab command can be used to upload a PEtab Dataset to openBIS and connect it to its * source files if these are stored in the same openBIS instance and referenced in the PEtabs meta- * data. - * To upload a PEtab dataset, the path to the PEtab folder and the experiment ID to which it should + * To upload a PEtab dataset, the path to the PEtab folder and the object ID to which it should * be attached need to be provided. * The dataset type of the new dataset in openBIS can be specified using the --type option, * otherwise the type "UNKNOWN" will be used. @@ -38,10 +38,10 @@ public class UploadPetabResultCommand implements Runnable { @Parameters(arity = "1", paramLabel = "PEtab folder", description = "The path to the PEtab folder " + "to upload") private String dataPath; - @Parameters(arity = "1", paramLabel = "experiment ID", description = "The full identifier of the " - + "+experiment the data should be attached to. " - + "The identifier must be of the format: /space/project/experiment") - private String experimentID; + @Parameters(arity = "1", paramLabel = "object ID", description = "The full identifier of the " + + "experiment or sample the data should be attached to. The identifier must be of the format: " + + "/space/project/experiment for experiments or /space/sample for samples") + private String objectID; @Option(arity = "1", paramLabel = "dataset type", description = "The openBIS dataset type code the " + "data should be stored as. UNKNOWN if no type is chosen.", names = {"-t", "--type"}) private String datasetType = "UNKNOWN"; @@ -55,7 +55,8 @@ public class UploadPetabResultCommand implements Runnable { public void run() { App.readConfig(); - OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), auth.getOpenbisAS(), auth.getOpenbisDSS()); + OpenBIS authentication = App.loginToOpenBIS(auth.getOpenbisPassword(), auth.getOpenbisUser(), + auth.getOpenbisAS(), auth.getOpenbisDSS()); openbis = new OpenbisConnector(authentication); if(!pathValid(dataPath)) { @@ -66,8 +67,13 @@ public void run() { System.out.printf("%s is not a directory. Please specify the PETab directory root%n", dataPath); return; } - if(!experimentExists(experimentID)) { - System.out.printf("Experiment %s could not be found%n", experimentID); + boolean attachToSample = false; + boolean attachToExperiment = openbis.experimentExists(objectID); + if(openbis.sampleExists(objectID)) { + attachToSample = true; + } + if(!attachToSample && !attachToExperiment) { + System.out.printf("%s could not be found in openBIS.%n", objectID); return; } System.out.println("Looking for reference datasets in metaInformation.yaml..."); @@ -85,19 +91,21 @@ public void run() { } } System.out.println("Uploading dataset..."); - DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), experimentID, - datasetType, parents); - System.out.printf("Dataset %s was successfully created%n", result.getPermId()); + if(attachToExperiment) { + DataSetPermId result = openbis.registerDatasetForExperiment(Path.of(dataPath), objectID, + datasetType, parents); + System.out.printf("Dataset %s was successfully attached to experiment%n", result.getPermId()); + } else { + DataSetPermId result = openbis.registerDatasetForSample(Path.of(dataPath), objectID, + datasetType, parents); + System.out.printf("Dataset %s was successfully attached to sample%n", result.getPermId()); + } } private boolean datasetsExist(List datasetCodes) { return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); } - private boolean experimentExists(String experimentID) { - return openbis.experimentExists(experimentID); - } - private boolean pathValid(String dataPath) { return new File(dataPath).exists(); } From c9447ab0113967482e9a9f01a5241484bbf09599 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 18:06:30 +0200 Subject: [PATCH 56/63] spacing --- .../life/qbic/io/commandline/UploadDatasetCommand.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java index 3cabf78..dc4532c 100644 --- a/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java +++ b/src/main/java/life/qbic/io/commandline/UploadDatasetCommand.java @@ -25,13 +25,15 @@ * otherwise the type "UNKNOWN" will be used. */ @Command(name = "upload-data", - description = "uploads a dataset and attaches it to an experiment and (optionally) other datasets") + description = "uploads a dataset and attaches it to an experiment or sample and (optionally) " + + "other datasets") public class UploadDatasetCommand implements Runnable { @Parameters(arity = "1", paramLabel = "file/folder", description = "The path to the file or folder to upload") private String dataPath; - @Parameters(arity = "1", paramLabel = "object ID", description = "The full identifier of the experiment or sample the data should be attached to. " - + "The identifier must be of the format: /space/project/experiment for experiments or /space/sample for samples") + @Parameters(arity = "1", paramLabel = "object ID", description = "The full identifier of the " + + "experiment or sample the data should be attached to. The identifier must be of the format: " + + "/space/project/experiment for experiments or /space/sample for samples") private String objectID; @Option(arity = "1..*", paramLabel = "", description = "Optional list of dataset codes to act" + " as parents for the upload. E.g. when this dataset has been generated using these datasets as input.", names = {"-pa", "--parents"}) From 4e48a4b72409404378ce3ad306c485c656a58f0d Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 19:44:57 +0200 Subject: [PATCH 57/63] test and finish update method --- README.md | 25 +++++++++ .../TransferDataToSeekCommand.java | 52 +++++++++---------- .../qbic/model/download/SEEKConnector.java | 36 +++++++++---- 3 files changed, 77 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 3f2642b..be9a4f8 100644 --- a/README.md +++ b/README.md @@ -279,4 +279,29 @@ least one sample attribute is different in openBIS and SEEK 4. assets attached to the experiment or samples will be created, if they are missing from this assay 5. no existing sample or assets are deleted from SEEK, even if they are missing from openBIS +**Example command:** + +`java -jar target/openbis-scripts-1.0.0-jar-with-dependencies.jar openbis-to-seek /MYSPACE/PROJECTY/00_P_INFO_691 mystudy -d -config config.txt --openbis-pw --seek-pw` + +**Example output:** + + Transfer openBIS -> SEEK started. + Provided openBIS object: /MYSPACE/PROJECTY/00_P_INFO_691 + Provided SEEK study title: mystudy + No SEEK project title provided, will search config file. + Transfer datasets to SEEK? true + Update existing nodes if found? true + Connecting to openBIS... + Searching for specified object in openBIS... + Search successful. + Connecting to SEEK... + Collecting information from openBIS... + Translating openBIS property codes to SEEK names... + Creating SEEK structure... + Trying to find existing corresponding assay in SEEK... + Found assay with id 64 + Updating nodes... + Mismatch found in Gender attribute of /MYSPACE/PROJECTY/00_P_INFO_691. Sample will be updated. + http://localhost:3000/assays/64 was successfully updated. + ## Caveats and Future Options diff --git a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java index a36c056..5fbd964 100644 --- a/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java +++ b/src/main/java/life/qbic/io/commandline/TransferDataToSeekCommand.java @@ -21,6 +21,7 @@ import life.qbic.model.OpenbisExperimentWithDescendants; import life.qbic.model.OpenbisSeekTranslator; import life.qbic.model.download.SEEKConnector.SeekStructurePostRegistrationInformation; +import life.qbic.model.isa.NodeType; import life.qbic.model.isa.SeekStructure; import life.qbic.model.download.OpenbisConnector; import life.qbic.model.download.SEEKConnector; @@ -88,7 +89,7 @@ public void run() { System.out.printf("No SEEK project title provided, will search config file.%n"); } System.out.printf("Transfer datasets to SEEK? %s%n", transferData); - System.out.printf("Update existing assay if found? %s%n", !noUpdate); + System.out.printf("Update existing nodes if found? %s%n", !noUpdate); if(blacklistFile!=null && !blacklistFile.isBlank()) { System.out.printf("File with datasets codes that won't be transferred: %s%n", blacklistFile); } @@ -100,28 +101,20 @@ public void run() { this.openbis = new OpenbisConnector(authentication); - boolean isSample = false; - boolean isDataSet = false; - System.out.println("Searching for specified object in openBIS..."); boolean isExperiment = experimentExists(objectID); - - if (isExperiment && (studyTitle == null || studyTitle.isBlank())) { - System.out.printf( - "No SEEK study title was provided. This is mandatory if an openBIS experiment is to be transferred%n"); - return; - } + NodeType nodeType = NodeType.ASSAY; if (!isExperiment && sampleExists(objectID)) { - isSample = true; + nodeType = NodeType.SAMPLE; } - if (!isExperiment && !isSample && datasetsExist(Arrays.asList(objectID))) { - isDataSet = true; + if (!isExperiment && !nodeType.equals(NodeType.SAMPLE) && datasetsExist(Arrays.asList(objectID))) { + nodeType = NodeType.ASSET; } - if (!isSample && !isExperiment && !isDataSet) { + if (nodeType.equals(NodeType.ASSAY) && !isExperiment) { System.out.printf( "%s could not be found in openBIS. Make sure you either specify an experiment, sample or dataset%n", objectID); @@ -150,16 +143,20 @@ public void run() { OpenbisExperimentWithDescendants structure; try { System.out.println("Collecting information from openBIS..."); - if (isExperiment) { - structure = openbis.getExperimentWithDescendants(objectID); - postRegInfo = handleExperimentTransfer(structure); - } else if (isSample) { - structure = openbis.getExperimentAndDataFromSample(objectID); - postRegInfo = handleExperimentTransfer(structure); - } else { - structure = openbis.getExperimentStructureFromDataset(objectID); - postRegInfo = handleExperimentTransfer(structure); + switch (nodeType) { + case ASSAY: + structure = openbis.getExperimentWithDescendants(objectID); + break; + case SAMPLE: + structure = openbis.getExperimentAndDataFromSample(objectID); + break; + case ASSET: + structure = openbis.getExperimentStructureFromDataset(objectID); + break; + default: + throw new RuntimeException("Handling of node type " + nodeType + " is not supported."); } + postRegInfo = handleExperimentTransfer(structure, nodeType); } catch (URISyntaxException | IOException | InterruptedException e) { throw new RuntimeException(e); } @@ -171,7 +168,7 @@ public void run() { } private SeekStructurePostRegistrationInformation handleExperimentTransfer( - OpenbisExperimentWithDescendants experiment) + OpenbisExperimentWithDescendants experiment, NodeType nodeType) throws URISyntaxException, IOException, InterruptedException { Set blacklist = parseBlackList(blacklistFile); System.out.println("Translating openBIS property codes to SEEK names..."); @@ -302,8 +299,9 @@ private Set parseBlackList(String blacklistFile) { } } - private SeekStructurePostRegistrationInformation updateAssayStructure(SeekStructure nodeWithChildren, - String assayID) throws URISyntaxException, IOException, InterruptedException { + private SeekStructurePostRegistrationInformation updateAssayStructure( + SeekStructure nodeWithChildren, String assayID) throws URISyntaxException, + IOException, InterruptedException { SeekStructurePostRegistrationInformation postRegInfo = seek.updateAssayNode(nodeWithChildren, assayID); List assetsToUpload = postRegInfo.getAssetsToUpload(); @@ -381,7 +379,7 @@ private Optional getAssayIDForOpenBISExperiment(Experiment experiment) // because if a perm id is found in the wrong SEEK node, meta-information in SEEK could be // overwritten or samples/data added to the wrong assay. String permID = experiment.getPermId().getPermId(); - List assayIDs = seek.searchAssaysContainingKeyword(permID); + List assayIDs = seek.searchAssaysInStudyContainingKeyword(permID); if(assayIDs.isEmpty()) { return Optional.empty(); } diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index dd35eea..3ad017e 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -50,6 +50,7 @@ public class SEEKConnector { private byte[] credentials; private OpenbisSeekTranslator translator; private final String DEFAULT_PROJECT_ID; + private String currentStudy; private final List ASSET_TYPES = new ArrayList<>(Arrays.asList("data_files", "models", "sops", "documents", "publications")); @@ -75,7 +76,8 @@ public void setDefaultInvestigation(String investigationTitle) public void setDefaultStudy(String studyTitle) throws URISyntaxException, IOException, InterruptedException { - translator.setDefaultStudy(searchNodeWithTitle("studies", studyTitle)); + this.currentStudy = searchNodeWithTitle("studies", studyTitle); + translator.setDefaultStudy(currentStudy); } /** @@ -500,18 +502,24 @@ private String searchNodeWithTitle(String nodeType, String title) * @throws IOException * @throws InterruptedException */ - public List searchAssaysContainingKeyword(String searchTerm) + public List searchAssaysInStudyContainingKeyword(String searchTerm) throws URISyntaxException, IOException, InterruptedException { JsonNode result = genericSearch("assays", "*"+searchTerm+"*"); JsonNode hits = result.path("data"); - List assayIDs = new ArrayList<>(); + List assayIDsInStudy = new ArrayList<>(); for (Iterator it = hits.elements(); it.hasNext(); ) { JsonNode hit = it.next(); - assayIDs.add(hit.get("id").asText()); + String assayID = hit.get("id").asText(); + JsonNode assayData = fetchAssayData(assayID).get("data"); + JsonNode relationships = assayData.get("relationships"); + String studyID = relationships.get("study").get("data").get("id").asText(); + if(studyID.equals(currentStudy)) { + assayIDsInStudy.add(assayID); + } } - return assayIDs; + return assayIDsInStudy; } /** @@ -556,8 +564,10 @@ public List searchAssetsContainingKeyword(String searchTerm) /** * Updates information of an existing assay, its samples and attached assets. Missing samples and * assets are created, but nothing missing from the new structure is deleted from SEEK. + * * @param nodeWithChildren the translated Seek structure as it should be once the update is done - * @param assayID the assay id of the existing assay, that should be compared to the new structure + * @param assayID the assay id of the existing assay, that should be compared to the new + * structure * @return information necessary to make post registration updates in openBIS and upload missing * data to newly created assets. In the case of the update use case, only newly created objects * will be contained in the return object. @@ -585,9 +595,9 @@ public SeekStructurePostRegistrationInformation updateAssayNode(SeekStructure no boolean oldEmpty = oldValue == null || oldValue.toString().isEmpty(); boolean newEmpty = newValue == null || newValue.toString().isEmpty(); - if ((!oldEmpty && !newEmpty) && !newValue.equals(oldValue)) { - System.out.printf("Mismatch found in attributes of %s. Sample will be updated.%n", - openBisID); + if ((!oldEmpty && !newEmpty) && !newValue.toString().equals(oldValue.toString())) { + System.out.printf("Mismatch found in %s attribute of %s. Sample will be updated.%n", + key, openBisID); newSample.setAssayIDs(List.of(assayID)); updateSample(newSample, existingSample.getSeekID()); } @@ -840,6 +850,7 @@ public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWit Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference().get(); + System.out.println("Creating assay..."); String assayID = addAssay(assayIDPair.getKey()); String assayEndpoint = apiURL+"/assays/"+assayID; Pair experimentIDWithEndpoint = @@ -850,6 +861,9 @@ public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWit Map sampleIDsWithEndpoints = new HashMap<>(); Map samplesWithReferences = nodeWithChildren.getSamplesWithOpenBISReference(); + if(!samplesWithReferences.isEmpty()) { + System.out.println("Creating samples..."); + } for(ISASample sample : samplesWithReferences.keySet()) { sample.setAssayIDs(Collections.singletonList(assayID)); String sampleEndpoint = createSample(sample); @@ -858,6 +872,10 @@ public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWit Map isaToFileMap = nodeWithChildren.getISAFileToDatasetFiles(); + if(!isaToFileMap.isEmpty()) { + System.out.println("Creating assets..."); + } + List assetsToUpload = createAssetsForAssays(isaToFileMap, Collections.singletonList(assayID)); From e1c5c3d51ceb6bc12b3013092ea80531c5c4f81e Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 19:45:50 +0200 Subject: [PATCH 58/63] add node type enum --- src/main/java/life/qbic/model/isa/NodeType.java | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/main/java/life/qbic/model/isa/NodeType.java diff --git a/src/main/java/life/qbic/model/isa/NodeType.java b/src/main/java/life/qbic/model/isa/NodeType.java new file mode 100644 index 0000000..e426dd3 --- /dev/null +++ b/src/main/java/life/qbic/model/isa/NodeType.java @@ -0,0 +1,5 @@ +package life.qbic.model.isa; + +public enum NodeType { + ASSAY, SAMPLE, ASSET +} From e6b5e2ddb3ee878dad6f48397a23c80ea810c14d Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Mon, 21 Oct 2024 20:00:49 +0200 Subject: [PATCH 59/63] explain EXTERNAL_LINK sample type --- README.md | 10 ++++++++++ .../life/qbic/model/download/OpenbisConnector.java | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index be9a4f8..4cb945b 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,16 @@ The data itself can be transferred and stored in SEEK using the '-d' flag. To completely exclude some dataset information from being transferred, a file ('--blacklist') containing dataset codes (from openBIS) can be specified. //TODO do this for samples/sample types +In order to store links to the newly created SEEK objects in the source openBIS instance, the +following sample type is needed: + + Sample Type Code: EXTERNAL_LINK + Property: LINK_TYPE (VARCHAR) + Property: URL (VARCHAR) + +EXTERNAL_LINK samples are added to transferred experiments and samples and point to their respective +counterparts in SEEK. If the sample type is not available, this will be logged. + ### Updating nodes in SEEK based on updates in openBIS Updating nodes in SEEK uses the same general command, parameters and options. Unless otherwise diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index 3dd134e..a622259 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -545,7 +545,7 @@ public void createSeekLinks(SeekStructurePostRegistrationInformation postRegInfo SampleTypeFetchOptions typeOptions = new SampleTypeFetchOptions(); typeOptions.withPropertyAssignments().withPropertyType(); typeOptions.withPropertyAssignments().withEntityType(); - if(openBIS.searchSampleTypes(criteria, typeOptions).getObjects().size() == 0) { + if(openBIS.searchSampleTypes(criteria, typeOptions).getObjects().isEmpty()) { System.out.printf( "This is where links would be put into openBIS, but EXTERNAL_LINK sample was " + "not yet added to openBIS instance.%n"); From 42670a2b8cae185267666d86d060b7c01580d8d8 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 23 Oct 2024 16:19:47 +0200 Subject: [PATCH 60/63] add ro-crate use case --- README.md | 72 +++++- .../io/commandline/CommandLineOptions.java | 3 +- .../qbic/io/commandline/CreateROCrate.java | 238 ++++++++++++++++++ .../qbic/model/DatasetWithProperties.java | 11 + .../qbic/model/OpenbisSeekTranslator.java | 141 ++++++----- .../qbic/model/download/OpenbisConnector.java | 5 +- .../qbic/model/download/SEEKConnector.java | 15 +- .../life/qbic/model/isa/SeekStructure.java | 21 +- 8 files changed, 405 insertions(+), 101 deletions(-) create mode 100644 src/main/java/life/qbic/io/commandline/CreateROCrate.java diff --git a/README.md b/README.md index 4cb945b..691ed65 100644 --- a/README.md +++ b/README.md @@ -250,7 +250,7 @@ The script will try to find the provided **openbis ID** in experiments, samples fetch any missing information to create a SEEK node containing at least one assay (when an experiment without samples and datasets is specified). -The seek-study needs to be provides to attach the assay. TODO: This information is also used to +The seek-study needs to be provided to attach the assay. TODO: This information is also used to decide if the node(s) should be updated (if they exist for the provided study) or created anew. Similarly, the title of the project in SEEK where nodes should be added, can either be provided via @@ -291,7 +291,7 @@ least one sample attribute is different in openBIS and SEEK **Example command:** -`java -jar target/openbis-scripts-1.0.0-jar-with-dependencies.jar openbis-to-seek /MYSPACE/PROJECTY/00_P_INFO_691 mystudy -d -config config.txt --openbis-pw --seek-pw` +`java -jar scripts.jar openbis-to-seek /MYSPACE/PROJECTY/00_P_INFO_691 mystudy -d -config config.txt --openbis-pw --seek-pw` **Example output:** @@ -314,4 +314,72 @@ least one sample attribute is different in openBIS and SEEK Mismatch found in Gender attribute of /MYSPACE/PROJECTY/00_P_INFO_691. Sample will be updated. http://localhost:3000/assays/64 was successfully updated. +#### RO-Crates + +While the creation of RO-Crates is not fully implemented, the command creates a folder and metadata +structure based on OpenBIS experiment, sample and dataset information. The command works similarly +to the OpenBIS to Seek command, with the difference that no SEEK instance and fewer mapping +parameters need to be provided (there will be no references to existing study or project objects in +SEEK). + +The script will try to find the provided **openbis ID** in experiments, samples or datasets and +fetch any missing information to create a folder structure containing at least one assay (when an +experiment without samples and datasets is specified). + +Info in the created asset .jsons always links back to the openBIS path of the respective dataset. +The data itself can be downloaded into the structure using the '-d' flag. + +To completely exclude some dataset information from being transferred, a file ('--blacklist') +containing dataset codes (from openBIS) can be specified. //TODO do this for samples/sample types + +**Example command:** + +`java -jar scripts.jar ro-crate /TEMP_PLAYGROUND/TEMP_PLAYGROUND/TEST_PATIENTS1 my-ro-crate -config config.txt --openbis-pw -d` + +**Example output:** + + reading config + Transfer openBIS -> RO-crate started. + Provided openBIS object: /TEMP_PLAYGROUND/TEMP_PLAYGROUND/TEST_PATIENTS1 + Pack datasets into crate? true + Connecting to openBIS... + Searching for specified object in openBIS... + Search successful. + Collecting information from openBIS... + Translating openBIS structure to ISA structure... + Writing assay json for /TEMP_PLAYGROUND/TEMP_PLAYGROUND/TEST_PATIENTS1. + Writing sample json for /TEMP_PLAYGROUND/TEMP_PLAYGROUND/00_P_INFO_670490. + Writing sample json for /TEMP_PLAYGROUND/TEMP_PLAYGROUND/00_P_INFO_670491. + Writing asset json for file in dataset 20241014205813459-689089. + Downloading dataset file to asset folder. + Writing asset json for file in dataset 20241014210001025-689090. + Downloading dataset file to asset folder. + Writing asset json for file in dataset 20241014205813459-689089. + Downloading dataset file to asset folder. + Writing asset json for file in dataset 20241021191109163-689109. + Downloading dataset file to asset folder. + +**Creates structure:** + + my-ro-crate + └── TEMP_PLAYGROUND_TEMP_PLAYGROUND_TEST_PATIENTS1 + β”œβ”€β”€ 20241021125328024-689105 + β”‚ β”œβ”€β”€ README.md + β”‚ └── README.md.json + β”œβ”€β”€ TEMP_PLAYGROUND_TEMP_PLAYGROUND_00_P_INFO_670490 + β”‚ └── TEMP_PLAYGROUND_TEMP_PLAYGROUND_00_P_INFO_670490.json + β”œβ”€β”€ TEMP_PLAYGROUND_TEMP_PLAYGROUND_00_P_INFO_670491 + β”‚ β”œβ”€β”€ 20241014210317842-689092 + β”‚ β”‚ β”œβ”€β”€ scripts-new.jar + β”‚ β”‚ └── scripts-new.jar.json + β”‚ β”œβ”€β”€ 20241021173011602-689108 + β”‚ β”‚ └── smol_petab + β”‚ β”‚ β”œβ”€β”€ metaInformation.yaml + β”‚ β”‚ └── metaInformation.yaml.json + β”‚ β”œβ”€β”€ 20241021191109163-689109 + β”‚ β”‚ β”œβ”€β”€ testfile_100 + β”‚ β”‚ └── testfile_100.json + β”‚ └── TEMP_PLAYGROUND_TEMP_PLAYGROUND_00_P_INFO_670491.json + └── TEMP_PLAYGROUND_TEMP_PLAYGROUND_TEST_PATIENTS1.json + ## Caveats and Future Options diff --git a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java index 332bd68..b44ea07 100644 --- a/src/main/java/life/qbic/io/commandline/CommandLineOptions.java +++ b/src/main/java/life/qbic/io/commandline/CommandLineOptions.java @@ -10,7 +10,8 @@ @Command(name = "openbis-scripts", subcommands = {SampleHierarchyCommand.class, TransferSampleTypesToSeekCommand.class, DownloadPetabCommand.class, UploadPetabResultCommand.class, UploadDatasetCommand.class, - SpaceStatisticsCommand.class, TransferDataToSeekCommand.class, FindDatasetsCommand.class}, + SpaceStatisticsCommand.class, TransferDataToSeekCommand.class, FindDatasetsCommand.class, + CreateROCrate.class}, description = "A client software for querying openBIS.", mixinStandardHelpOptions = true, versionProvider = ManifestVersionProvider.class) public class CommandLineOptions { diff --git a/src/main/java/life/qbic/io/commandline/CreateROCrate.java b/src/main/java/life/qbic/io/commandline/CreateROCrate.java new file mode 100644 index 0000000..9478ce5 --- /dev/null +++ b/src/main/java/life/qbic/io/commandline/CreateROCrate.java @@ -0,0 +1,238 @@ +package life.qbic.io.commandline; + +import ch.ethz.sis.openbis.generic.OpenBIS; +import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.xml.parsers.ParserConfigurationException; +import life.qbic.App; +import life.qbic.model.DatasetWithProperties; +import life.qbic.model.OpenbisExperimentWithDescendants; +import life.qbic.model.OpenbisSeekTranslator; +import life.qbic.model.download.OpenbisConnector; +import life.qbic.model.isa.GenericSeekAsset; +import life.qbic.model.isa.ISAAssay; +import life.qbic.model.isa.ISASample; +import life.qbic.model.isa.NodeType; +import life.qbic.model.isa.SeekStructure; +import org.xml.sax.SAXException; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@Command(name = "ro-crate", + description = + "Transfers metadata and (optionally) data from openBIS to an RO-Crate-like structure that is " + + "based on assays, samples and one of several data types in SEEK). The data itself can " + + "be put into the crate using the '-d' flag. To completely exclude some dataset " + + "information from being transferred, a file ('--blacklist') containing dataset codes " + + "can be specified. The crate is not zipped at the moment.") +public class CreateROCrate implements Runnable { + + @Parameters(arity = "1", paramLabel = "openbis id", description = "The identifier of the " + + "experiment, sample or dataset to transfer.") + private String objectID; + @Parameters(arity = "1", paramLabel = "ro-path", description = "Path to the output folder") + private String roPath; + @Option(names = "--blacklist", description = "Path to file specifying by dataset " + + "dataset code which openBIS datasets not to transfer to SEEK. The file must contain one code " + + "per line.") + private String blacklistFile; + @Option(names = {"-d", "--data"}, description = + "Transfers the data itself to SEEK along with the metadata. " + + "Otherwise only the link(s) to the openBIS object will be created in SEEK.") + private boolean transferData; + @Mixin + OpenbisAuthenticationOptions openbisAuth = new OpenbisAuthenticationOptions(); + OpenbisConnector openbis; + OpenbisSeekTranslator translator; + + @Override + public void run() { + App.readConfig(); + System.out.printf("Transfer openBIS -> RO-crate started.%n"); + System.out.printf("Provided openBIS object: %s%n", objectID); + System.out.printf("Pack datasets into crate? %s%n", transferData); + if(blacklistFile!=null && !blacklistFile.isBlank()) { + System.out.printf("File with datasets codes that won't be transferred: %s%n", blacklistFile); + } + + System.out.println("Connecting to openBIS..."); + + OpenBIS authentication = App.loginToOpenBIS(openbisAuth.getOpenbisPassword(), + openbisAuth.getOpenbisUser(), openbisAuth.getOpenbisAS(), openbisAuth.getOpenbisDSS()); + + this.openbis = new OpenbisConnector(authentication); + + System.out.println("Searching for specified object in openBIS..."); + + boolean isExperiment = experimentExists(objectID); + NodeType nodeType = NodeType.ASSAY; + + if (!isExperiment && sampleExists(objectID)) { + nodeType = NodeType.SAMPLE; + } + + if (!isExperiment && !nodeType.equals(NodeType.SAMPLE) && datasetsExist( + Arrays.asList(objectID))) { + nodeType = NodeType.ASSET; + } + + if (nodeType.equals(NodeType.ASSAY) && !isExperiment) { + System.out.printf( + "%s could not be found in openBIS. Make sure you either specify an experiment, sample or dataset%n", + objectID); + return; + } + System.out.println("Search successful."); + + try { + translator = new OpenbisSeekTranslator(openbisAuth.getOpenbisBaseURL()); + } catch (IOException | ParserConfigurationException | SAXException e) { + throw new RuntimeException(e); + } + OpenbisExperimentWithDescendants structure; + System.out.println("Collecting information from openBIS..."); + switch (nodeType) { + case ASSAY: + structure = openbis.getExperimentWithDescendants(objectID); + break; + case SAMPLE: + structure = openbis.getExperimentAndDataFromSample(objectID); + break; + case ASSET: + structure = openbis.getExperimentStructureFromDataset(objectID); + break; + default: + throw new RuntimeException("Handling of node type " + nodeType + " is not supported."); + } + Set blacklist = parseBlackList(blacklistFile); + System.out.println("Translating openBIS structure to ISA structure..."); + try { + SeekStructure nodeWithChildren = translator.translateForRO(structure, blacklist, transferData); + String experimentID = nodeWithChildren.getAssayWithOpenBISReference().getRight(); + ISAAssay assay = nodeWithChildren.getAssayWithOpenBISReference().getLeft(); + String assayFileName = openbisIDToFileName(experimentID); + + String assayPath = Path.of(roPath, assayFileName).toString(); + new File(assayPath).mkdirs(); + + System.out.printf("Writing assay json for %s.%n", experimentID); + writeFile(Path.of(assayPath, assayFileName)+".json", assay.toJson()); + + for(ISASample sample : nodeWithChildren.getSamplesWithOpenBISReference().keySet()) { + String sampleID = nodeWithChildren.getSamplesWithOpenBISReference().get(sample); + String sampleFileName = openbisIDToFileName(sampleID); + String samplePath = Path.of(assayPath, sampleFileName).toString(); + new File(samplePath).mkdirs(); + + System.out.printf("Writing sample json for %s.%n", sampleID); + writeFile(Path.of(samplePath, sampleFileName)+".json", sample.toJson()); + } + + Map datasetIDToDataFolder = new HashMap<>(); + + for(DatasetWithProperties dwp : structure.getDatasets()) { + String sourceID = dwp.getClosestSourceID(); + String code = dwp.getCode(); + if(sourceID.equals(experimentID)) { + Path folderPath = Path.of(assayPath, code); + File dataFolder = new File(folderPath.toString()); + datasetIDToDataFolder.put(dwp.getCode(), dataFolder.getAbsolutePath()); + } else { + Path samplePath = Path.of(assayPath, openbisIDToFileName(sourceID), code); + File dataFolder = new File(samplePath.toString()); + datasetIDToDataFolder.put(dwp.getCode(), dataFolder.getAbsolutePath()); + } + } + + for(GenericSeekAsset asset : nodeWithChildren.getISAFileToDatasetFiles().keySet()) { + DataSetFile file = nodeWithChildren.getISAFileToDatasetFiles().get(asset); + String datasetID = file.getDataSetPermId().getPermId(); + String dataFolderPath = datasetIDToDataFolder.get(datasetID); + String assetJson = asset.toJson(); + String assetWithoutOriginFolder = asset.getFileName().replace("original",""); + File assetFolder = Path.of(dataFolderPath, assetWithoutOriginFolder).getParent().toFile(); + assetFolder.mkdirs(); + + String assetPath = Path.of(dataFolderPath, assetWithoutOriginFolder+".json").toString(); + System.out.printf("Writing asset json for file in dataset %s.%n", datasetID); + writeFile(assetPath, assetJson); + if(transferData) { + System.out.printf("Downloading dataset file to asset folder.%n"); + openbis.downloadDataset(dataFolderPath, datasetID, asset.getFileName()); + } + } + } catch (URISyntaxException | IOException e) { + throw new RuntimeException(e); + } + + System.out.println("Done"); + } + + private String openbisIDToFileName(String id) { + id = id.replace("/","_"); + if(id.startsWith("_")) { + return id.substring(1); + } else { + return id; + } + } + + private void writeFile(String path, String content) throws IOException { + FileWriter file = new FileWriter(path); + file.write(content); + file.close(); + } + + private Set parseBlackList(String blacklistFile) { + if(blacklistFile == null) { + return new HashSet<>(); + } + // trim whitespace, skip empty lines + try (Stream lines = Files.lines(Paths.get(blacklistFile)) + .map(String::trim) + .filter(s -> !s.isBlank())) { + + Set codes = lines.collect(Collectors.toSet()); + + for(String code : codes) { + if(!OpenbisConnector.datasetCodePattern.matcher(code).matches()) { + throw new RuntimeException("Invalid dataset code: " + code+". Please make sure to use valid" + + " dataset codes in the blacklist file."); + } + } + return codes; + } catch (IOException e) { + throw new RuntimeException(blacklistFile+" could not be found or read."); + } + } + + private boolean sampleExists(String objectID) { + return openbis.sampleExists(objectID); + } + + private boolean datasetsExist(List datasetCodes) { + return openbis.findDataSets(datasetCodes).size() == datasetCodes.size(); + } + + private boolean experimentExists(String experimentID) { + return openbis.experimentExists(experimentID); + } + +} diff --git a/src/main/java/life/qbic/model/DatasetWithProperties.java b/src/main/java/life/qbic/model/DatasetWithProperties.java index d9a1534..a203d34 100644 --- a/src/main/java/life/qbic/model/DatasetWithProperties.java +++ b/src/main/java/life/qbic/model/DatasetWithProperties.java @@ -57,4 +57,15 @@ public Person getRegistrator() { public Date getRegistrationDate() { return dataset.getRegistrationDate(); } + + /** + * Returns sample ID or experiment ID, if Dataset has no sample. + */ + public String getClosestSourceID() { + if(dataset.getSample()!=null) { + return dataset.getSample().getIdentifier().getIdentifier(); + } else { + return getExperiment().getIdentifier().getIdentifier(); + } + } } diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 354ad66..7ae298b 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -32,7 +32,6 @@ import life.qbic.model.isa.ISASampleType.SampleAttribute; import life.qbic.model.isa.ISASampleType.SampleAttributeType; import life.qbic.model.isa.SeekStructure; -import org.apache.commons.lang3.tuple.Pair; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -60,6 +59,80 @@ public OpenbisSeekTranslator(String openBISBaseURL, String defaultProjectID) } } + /** + * Used for translation to RO-Crate, without SEEK connection + * @param openbisBaseURL + */ + public OpenbisSeekTranslator(String openbisBaseURL) + throws IOException, ParserConfigurationException, SAXException { + this.openBISBaseURL = openbisBaseURL; + this.DEFAULT_PROJECT_ID = null; + parseConfigs(); + } + + //Used for translation to RO-Crate, without SEEK connection + public SeekStructure translateForRO(OpenbisExperimentWithDescendants experiment, + Set blacklist, boolean transferData) throws URISyntaxException { + + Experiment exp = experiment.getExperiment(); + String expType = exp.getType().getCode(); + String title = exp.getCode()+" ("+exp.getPermId().getPermId()+")"; + String assayType = experimentTypeToAssayType.get(expType); + + if(assayType ==null || assayType.isBlank()) { + throw new RuntimeException("Could not find assay type for " + expType+". A mapping needs to " + + "be added to the respective properties file."); + } + ISAAssay assay = new ISAAssay(title, "-1", experimentTypeToAssayClass.get(expType), + new URI(assayType)); + + SeekStructure result = new SeekStructure(assay, exp.getIdentifier().getIdentifier()); + + for(Sample sample : experiment.getSamples()) { + SampleType sampleType = sample.getType(); + + //try to put all attributes into sample properties, as they should be a 1:1 mapping + Map typeCodesToNames = new HashMap<>(); + Set propertiesLinkingSamples = new HashSet<>(); + for (PropertyAssignment a : sampleType.getPropertyAssignments()) { + String code = a.getPropertyType().getCode(); + String label = a.getPropertyType().getLabel(); + DataType type = a.getPropertyType().getDataType(); + typeCodesToNames.put(code, label); + if(type.equals(DataType.SAMPLE)) { + propertiesLinkingSamples.add(code); + } + } + Map attributes = new HashMap<>(); + for(String code : sample.getProperties().keySet()) { + String value = sample.getProperty(code); + if(propertiesLinkingSamples.contains(code)) { + value = generateOpenBISLinkFromPermID("SAMPLE", value); + } + attributes.put(typeCodesToNames.get(code), value); + } + + String sampleID = sample.getIdentifier().getIdentifier(); + attributes.put(App.configProperties.get("seek_openbis_sample_title"), sampleID); + ISASample isaSample = new ISASample(sample.getIdentifier().getIdentifier(), attributes, + "-1", Collections.singletonList(DEFAULT_PROJECT_ID)); + result.addSample(isaSample, sampleID); + } + + //create ISA files for assets. If actual data is to be uploaded is determined later based on flag + for(DatasetWithProperties dataset : experiment.getDatasets()) { + String permID = dataset.getCode(); + if(!blacklist.contains(permID)) { + for(DataSetFile file : experiment.getFilesForDataset(permID)) { + String datasetType = getDatasetTypeOfFile(file, experiment.getDatasets()); + datasetFileToSeekAsset(file, datasetType, transferData) + .ifPresent(seekAsset -> result.addAsset(seekAsset, file)); + } + } + } + return result; + } + /** * Parses mandatory mapping information from mandatory config files. Other files may be added. */ @@ -218,72 +291,6 @@ public SeekStructure translate(OpenbisExperimentWithDescendants experiment, return result; } - /* - public SeekStructure translate(OpenbisSampleWithDatasets sampleWithDatasets, - Map sampleTypesToIds, Set blacklist, boolean transferData) { - Sample sample = sampleWithDatasets.getSample(); - SampleType sampleType = sample.getType(); - //try to put all attributes into sample properties, as they should be a 1:1 mapping - Map typeCodesToNames = new HashMap<>(); - Set propertiesLinkingSamples = new HashSet<>(); - for (PropertyAssignment a : sampleType.getPropertyAssignments()) { - String code = a.getPropertyType().getCode(); - String label = a.getPropertyType().getLabel(); - DataType type = a.getPropertyType().getDataType(); - typeCodesToNames.put(code, label); - if (type.equals(DataType.SAMPLE)) { - propertiesLinkingSamples.add(code); - } - } - Map attributes = new HashMap<>(); - for (String code : sample.getProperties().keySet()) { - String value = sample.getProperty(code); - if (propertiesLinkingSamples.contains(code)) { - value = generateOpenBISLinkFromPermID("SAMPLE", value); - } - attributes.put(typeCodesToNames.get(code), value); - } - - String sampleID = sample.getIdentifier().getIdentifier(); - attributes.put(App.configProperties.get("seek_openbis_sample_title"), sampleID); - String sampleTypeId = sampleTypesToIds.get(sampleType.getCode()); - ISASample isaSample = new ISASample(sample.getIdentifier().getIdentifier(), attributes, - sampleTypeId, Collections.singletonList(DEFAULT_PROJECT_ID)); - SeekStructure result = new SeekStructure(isaSample, sampleID); - - //create ISA files for assets. If actual data is to be uploaded is determined later based on flag - List datasets = sampleWithDatasets.getDatasets(); - for (DatasetWithProperties dataset : datasets) { - String permID = dataset.getCode(); - if (!blacklist.contains(permID)) { - for (DataSetFile file : sampleWithDatasets.getFilesForDataset(permID)) { - String datasetType = getDatasetTypeOfFile(file, datasets); - datasetFileToSeekAsset(file, datasetType, transferData) - .ifPresent(seekAsset -> result.addAsset(seekAsset, file)); - } - } - } - return result; - } - - public SeekStructure translate(Pair> datasetWithFiles, - Set blacklist, boolean transferData) { - Map assetToDatasetFiles = new HashMap<>(); - DatasetWithProperties dataset = datasetWithFiles.getLeft(); - String permID = dataset.getCode(); - String datasetType = dataset.getType().getCode(); - - if (!blacklist.contains(permID)) { - for (DataSetFile file : datasetWithFiles.getRight()) { - datasetFileToSeekAsset(file, datasetType, transferData) - .ifPresent(seekAsset -> assetToDatasetFiles.put(seekAsset, file)); - } - } - return new SeekStructure(assetToDatasetFiles); - } - - */ - private String getDatasetTypeOfFile(DataSetFile file, List dataSets) { String permId = file.getDataSetPermId().getPermId(); for(DatasetWithProperties dataset : dataSets) { diff --git a/src/main/java/life/qbic/model/download/OpenbisConnector.java b/src/main/java/life/qbic/model/download/OpenbisConnector.java index a622259..7d872eb 100644 --- a/src/main/java/life/qbic/model/download/OpenbisConnector.java +++ b/src/main/java/life/qbic/model/download/OpenbisConnector.java @@ -127,7 +127,6 @@ private UploadedDataSetCreation prepareDataSetCreation(Path uploadPath, String d private static void copyInputStreamToFile(InputStream inputStream, File file) throws IOException { - System.err.println(file.getPath()); try (FileOutputStream outputStream = new FileOutputStream(file, false)) { int read; byte[] bytes = new byte[8192]; @@ -469,6 +468,8 @@ public OpenbisExperimentWithDescendants getExperimentWithDescendants(String expe DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); dataSetFetchOptions.withType(); dataSetFetchOptions.withRegistrator(); + dataSetFetchOptions.withExperiment(); + dataSetFetchOptions.withSample(); SampleFetchOptions sampleFetchOptions = new SampleFetchOptions(); sampleFetchOptions.withProperties(); sampleFetchOptions.withType().withPropertyAssignments().withPropertyType(); @@ -662,6 +663,8 @@ public OpenbisExperimentWithDescendants getExperimentAndDataFromSample(String sa DataSetFetchOptions dataSetFetchOptions = new DataSetFetchOptions(); dataSetFetchOptions.withType(); dataSetFetchOptions.withRegistrator(); + dataSetFetchOptions.withExperiment(); + dataSetFetchOptions.withSample(); SampleFetchOptions fetchOptions = new SampleFetchOptions(); fetchOptions.withProperties(); fetchOptions.withType().withPropertyAssignments().withPropertyType(); diff --git a/src/main/java/life/qbic/model/download/SEEKConnector.java b/src/main/java/life/qbic/model/download/SEEKConnector.java index 3ad017e..157fbf6 100644 --- a/src/main/java/life/qbic/model/download/SEEKConnector.java +++ b/src/main/java/life/qbic/model/download/SEEKConnector.java @@ -645,11 +645,8 @@ public SeekStructurePostRegistrationInformation updateAssayNode(SeekStructure no } String assayEndpoint = apiURL + "/assays/" + assayID; - if(nodeWithChildren.getAssayWithOpenBISReference().isEmpty()) { - throw new RuntimeException("No assay and openBIS reference found. Object has not been " - + "initialized using an assay object and openBIS experiment reference."); - } - String expID = nodeWithChildren.getAssayWithOpenBISReference().get().getRight(); + + String expID = nodeWithChildren.getAssayWithOpenBISReference().getRight(); Pair experimentIDWithEndpoint = new ImmutablePair<>(expID, assayEndpoint); SeekStructurePostRegistrationInformation postRegInfo = @@ -842,13 +839,7 @@ private JsonNode fetchAssayData(String assayID) public SeekStructurePostRegistrationInformation createNode(SeekStructure nodeWithChildren) throws URISyntaxException, IOException, InterruptedException { - - if(nodeWithChildren.getAssayWithOpenBISReference().isEmpty()) { - throw new RuntimeException("No assay and openBIS reference found. Object has not been " - + "initialized using an assay object and openBIS experiment reference."); - } - - Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference().get(); + Pair assayIDPair = nodeWithChildren.getAssayWithOpenBISReference(); System.out.println("Creating assay..."); String assayID = addAssay(assayIDPair.getKey()); diff --git a/src/main/java/life/qbic/model/isa/SeekStructure.java b/src/main/java/life/qbic/model/isa/SeekStructure.java index 858e30e..b3826bf 100644 --- a/src/main/java/life/qbic/model/isa/SeekStructure.java +++ b/src/main/java/life/qbic/model/isa/SeekStructure.java @@ -1,11 +1,8 @@ package life.qbic.model.isa; import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.DataSetFile; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Optional; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -16,28 +13,16 @@ */ public class SeekStructure { - private final Optional> assayAndOpenBISReference; + private final Pair assayAndOpenBISReference; private final Map samplesWithOpenBISReference; private final Map isaToOpenBISFile; public SeekStructure(ISAAssay assay, String openBISReference) { - this.assayAndOpenBISReference = Optional.of(new ImmutablePair<>(assay, openBISReference)); + this.assayAndOpenBISReference = new ImmutablePair<>(assay, openBISReference); this.samplesWithOpenBISReference = new HashMap<>(); this.isaToOpenBISFile = new HashMap<>(); } - public SeekStructure(ISASample isaSample, String sampleID) { - this.samplesWithOpenBISReference = new HashMap<>(); - this.isaToOpenBISFile = new HashMap<>(); - this.assayAndOpenBISReference = Optional.empty(); - } - - public SeekStructure(Map isaToOpenBISFile) { - this.isaToOpenBISFile = isaToOpenBISFile; - this.samplesWithOpenBISReference = new HashMap<>(); - this.assayAndOpenBISReference = Optional.empty(); - } - public void addSample(ISASample sample, String openBISReference) { samplesWithOpenBISReference.put(sample, openBISReference); } @@ -46,7 +31,7 @@ public void addAsset(GenericSeekAsset asset, DataSetFile file) { isaToOpenBISFile.put(asset, file); } - public Optional> getAssayWithOpenBISReference() { + public Pair getAssayWithOpenBISReference() { return assayAndOpenBISReference; } From d3085f684fbc0f3ff697cf95c0210601bc361864 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 23 Oct 2024 16:25:36 +0200 Subject: [PATCH 61/63] improve readme --- README.md | 9 +++++++-- .../java/life/qbic/io/commandline/CreateROCrate.java | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 691ed65..460a317 100644 --- a/README.md +++ b/README.md @@ -323,8 +323,12 @@ parameters need to be provided (there will be no references to existing study or SEEK). The script will try to find the provided **openbis ID** in experiments, samples or datasets and -fetch any missing information to create a folder structure containing at least one assay (when an -experiment without samples and datasets is specified). +fetch any missing information to create a folder structure in the provided **ro-path** containing at +least one assay's information (when an experiment without samples and datasets is specified). + +Assets (files and their ISA metadata) are stored in a folder named like the openBIS dataset code +they are part of, which is either the subfolder of the experiment (assay), or the subfolder of a +sample, depending on where the dataset was attached in openBIS. Info in the created asset .jsons always links back to the openBIS path of the respective dataset. The data itself can be downloaded into the structure using the '-d' flag. @@ -358,6 +362,7 @@ containing dataset codes (from openBIS) can be specified. //TODO do this for sam Downloading dataset file to asset folder. Writing asset json for file in dataset 20241021191109163-689109. Downloading dataset file to asset folder. + ... **Creates structure:** diff --git a/src/main/java/life/qbic/io/commandline/CreateROCrate.java b/src/main/java/life/qbic/io/commandline/CreateROCrate.java index 9478ce5..b7f56f1 100644 --- a/src/main/java/life/qbic/io/commandline/CreateROCrate.java +++ b/src/main/java/life/qbic/io/commandline/CreateROCrate.java @@ -14,7 +14,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; From d258d070c16aa75a36c1141640688323959a7100 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 30 Oct 2024 21:08:06 +0100 Subject: [PATCH 62/63] upate readme, rename property files, provide examples --- README.md | 30 ++++++++- dataset_type_to_asset_type.properties | 3 + experiment_type_to_assay_class.properties | 2 + experiment_type_to_assay_type.properties | 1 + openbis_datatype_to_seek_attributetype.xml | 64 +++++++++++++++++++ .../qbic/model/OpenbisSeekTranslator.java | 4 +- 6 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 dataset_type_to_asset_type.properties create mode 100644 experiment_type_to_assay_class.properties create mode 100644 experiment_type_to_assay_type.properties create mode 100644 openbis_datatype_to_seek_attributetype.xml diff --git a/README.md b/README.md index 460a317..eedc8cc 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ address) and their password, have to be provided. In order to interact with openBIS (transfer of data and metadata), the respective credentials need to be provided, as well. -**Everything but the passwords can be provided via the config file (config.txt):** +**Everything but the passwords can be provided via the config file (e.g. config.txt):** * as=https://my-openbis-instance.de/openbis/openbis * dss=https://my-openbis-instance.de/datastore_server @@ -204,6 +204,34 @@ section **Transferring Sample Types to SEEK**). * seek_openbis_sample_title=openBIS Name +In order to create certain objects in SEEK, user-provided mappings need to be provided via property +files placed in the same folder as the .jar. These are: + +**experiment_type_to_assay_class.properties** + +Here, openBIS experiment type codes are mapped to the assay class needed to create assay objects in +SEEK. Example entries: + + PARAMETER_ESTIMATION=EXP + MASS_SPECTROMETRY_EXPERIMENT=EXP + +**experiment_type_to_assay_type.properties** + +Here, openBIS experiment type codes are mapped to the assay type property needed to create assay +objects in SEEK. Example entry: + + MASS_SPECTROMETRY_EXPERIMENT=http://jermontology.org/ontology/JERMOntology#Proteomics + +Other fitting assay types can be found using the JERM ontology browser: +https://bioportal.bioontology.org/ontologies/JERM/?p=classes&conceptid=http%3A%2F%2Fjermontology.org%2Fontology%2FJERMOntology%23Experimental_assay_type&lang=en + +**dataset_type_to_asset_type.properties** + +Here, openBIS dataset type codes are mapped to the asset type created in SEEK. Example entries: + + SOURCE_CODE=documents + UNKNOWN=data_files + Refer to the help of the respective command or the examples below for more details. ### Transferring Sample Types to SEEK diff --git a/dataset_type_to_asset_type.properties b/dataset_type_to_asset_type.properties new file mode 100644 index 0000000..44bb45f --- /dev/null +++ b/dataset_type_to_asset_type.properties @@ -0,0 +1,3 @@ +SOURCE_CODE=documents +UNKNOWN=data_files +TEST_DAT=data_files diff --git a/experiment_type_to_assay_class.properties b/experiment_type_to_assay_class.properties new file mode 100644 index 0000000..a37900e --- /dev/null +++ b/experiment_type_to_assay_class.properties @@ -0,0 +1,2 @@ +PARAMETER_ESTIMATION=EXP +MASS_SPECTROMETRY_EXPERIMENT=EXP diff --git a/experiment_type_to_assay_type.properties b/experiment_type_to_assay_type.properties new file mode 100644 index 0000000..5ee82f3 --- /dev/null +++ b/experiment_type_to_assay_type.properties @@ -0,0 +1 @@ +MASS_SPECTROMETRY_EXPERIMENT=http://jermontology.org/ontology/JERMOntology#Proteomics diff --git a/openbis_datatype_to_seek_attributetype.xml b/openbis_datatype_to_seek_attributetype.xml new file mode 100644 index 0000000..0a29815 --- /dev/null +++ b/openbis_datatype_to_seek_attributetype.xml @@ -0,0 +1,64 @@ + + + + 4 + Integer + Integer + + + 8 + String + String + + + 7 + Text + Text + + + 3 + Real number + Float + + + 1 + Date time + DateTime + + + 16 + Boolean + Boolean + + + 8 + String + String + + + 8 + String + String + + + 8 + String + String + + + 7 + Text + Text + + + 5 + Web link + String + + + 2 + Date time + Date + + + diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 7ae298b..482cf23 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -138,9 +138,9 @@ public SeekStructure translateForRO(OpenbisExperimentWithDescendants experiment, */ private void parseConfigs() throws IOException, ParserConfigurationException, SAXException { final String dataTypeToAttributeType = "openbis_datatype_to_seek_attributetype.xml"; - final String datasetToAssaytype = "dataset_type_to_assaytype.properties"; + final String datasetToAssaytype = "dataset_type_to_asset_type.properties"; final String experimentTypeToAssayClass = "experiment_type_to_assay_class.properties"; - final String experimentTypeToAssayType = "openbis_experiment_type_to_seek_assaytype.properties"; + final String experimentTypeToAssayType = "experiment_type_to_seek_assay_type.properties"; this.experimentTypeToAssayType = PropertyReader.getProperties(experimentTypeToAssayType); this.datasetTypeToAssetType = PropertyReader.getProperties(datasetToAssaytype); this.experimentTypeToAssayClass = PropertyReader.getProperties(experimentTypeToAssayClass); From 1c55331fc8572e217cacb85eb3d541722767d613 Mon Sep 17 00:00:00 2001 From: wow-such-code Date: Wed, 30 Oct 2024 21:20:55 +0100 Subject: [PATCH 63/63] fix property file name --- src/main/java/life/qbic/model/OpenbisSeekTranslator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java index 482cf23..2c83354 100644 --- a/src/main/java/life/qbic/model/OpenbisSeekTranslator.java +++ b/src/main/java/life/qbic/model/OpenbisSeekTranslator.java @@ -140,7 +140,7 @@ private void parseConfigs() throws IOException, ParserConfigurationException, SA final String dataTypeToAttributeType = "openbis_datatype_to_seek_attributetype.xml"; final String datasetToAssaytype = "dataset_type_to_asset_type.properties"; final String experimentTypeToAssayClass = "experiment_type_to_assay_class.properties"; - final String experimentTypeToAssayType = "experiment_type_to_seek_assay_type.properties"; + final String experimentTypeToAssayType = "experiment_type_to_assay_type.properties"; this.experimentTypeToAssayType = PropertyReader.getProperties(experimentTypeToAssayType); this.datasetTypeToAssetType = PropertyReader.getProperties(datasetToAssaytype); this.experimentTypeToAssayClass = PropertyReader.getProperties(experimentTypeToAssayClass);