terra_mhub/papermill_notebooks/combine_tasks_output.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "l1oiPWf94zzP"
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "VZDJNMuf41eQ"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BA-Lmt-oAWjy"
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "jyP6HFYidMa_"
   },
   "outputs": [],
   "source": [
    "# from google.colab import auth\n",
    "# auth.authenticate_user()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "u6IZUFle7apI"
   },
   "outputs": [],
   "source": [
    "# !apt-get install lz4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "1O9UNZct7e_9"
   },
   "outputs": [],
   "source": [
    "# %%capture\n",
    "# !pip install --upgrade duckdb papermill s5cmd dcm2niix pyplastimatch pyradiomics \\\n",
    "# highdicom pydicom seg-metrics idc-index itk SimpleITK pyaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "qLB3dBkB7c6t"
   },
   "outputs": [],
   "source": [
    "# %%capture\n",
    "# from pyplastimatch.utils.install import install_precompiled_binaries\n",
    "# install_precompiled_binaries()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jYGs93hb7cQJ"
   },
   "outputs": [],
   "source": [
    "# dcmqi_release_url = \"https://github.com/QIICR/dcmqi/releases/download/v1.3.4/dcmqi-1.3.4-linux.tar.gz\"\n",
    "# dcmqi_download_path = \"dcmqi-1.3.4-linux.tar.gz\"\n",
    "# dcmqi_path = \"dcmqi-1.3.4-linux\"\n",
    "# !wget -O $dcmqi_download_path $dcmqi_release_url\\\n",
    "# && tar -xvf $dcmqi_download_path\\\n",
    "# && mv $dcmqi_path/bin/* /bin\\\n",
    "# && rm -r $dcmqi_download_path $dcmqi_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yJLRHiDh-7J_"
   },
   "source": [
    "# Global Variables\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "NEmPsqHP7qTZ",
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "#absolute path output\n",
    "OUTPUT_PATH = \"\"#\n",
    "#input MHUB zip files\n",
    "mhubCompressedOutputFiles = []\n",
    "#input rads ai compressed zip files\n",
    "radsAiCompressedOutputFiles = []\n",
    "#input eval compressed zip files\n",
    "evalCompressedOutputFiles = []\n",
    "#second list eval\n",
    "evalAddCompressedOutputFiles =  []\n",
    "#input IDC expert annotations zip file\n",
    "idcExpertCompressedOutputFile = \"\"\n",
    "#input IDC expert SR annotations zip file\n",
    "radsIdcExpertCompressedOutputFile = \"\"\n",
    "#second list IDC annotations files\n",
    "idcExpertAddCompressedOutputFile = \"\"\n",
    "radsIdcExpertAddCompressedOutputFile = \"\"\n",
    "#list of mhub_models\n",
    "mhub_model_name_lst = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LDvSz7sC1hE7"
   },
   "source": [
    "# Paths definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "R-Z5W8mf6ACn"
   },
   "outputs": [],
   "source": [
    "TASK_OUT_UNZIP = os.path.join(OUTPUT_PATH, \"tasks_unzip\")\n",
    "#mhub_outputs\n",
    "mhub_seg_out_unzip = os.path.join(TASK_OUT_UNZIP, \"mhub_seg\")\n",
    "mhub_sr_out_unzip = os.path.join(TASK_OUT_UNZIP, \"mhub_sr\")\n",
    "#eval outputs\n",
    "eval_seg_out_unzip = os.path.join(TASK_OUT_UNZIP, \"eval\")\n",
    "##second set of annotations\n",
    "eval_seg_add_out_unzip = os.path.join(TASK_OUT_UNZIP, \"eval_add\")\n",
    "#idc experts outputs\n",
    "idc_expert_seg_out_unzip = os.path.join(TASK_OUT_UNZIP, \"idc_expert_seg\")\n",
    "idc_expert_sr_out_unzip =os.path.join(TASK_OUT_UNZIP, \"idc_expert_sr\")\n",
    "#second set of idc experts outputs\n",
    "idc_expert_seg_add_out_unzip = os.path.join(TASK_OUT_UNZIP, \"idc_expert_seg_add\")\n",
    "idc_expert_sr_add_out_unzip =os.path.join(TASK_OUT_UNZIP, \"idc_expert_sr_add\")\n",
    "\n",
    "#final folder path\n",
    "base_out_folder = os.path.join(OUTPUT_PATH, \"aggregated_results\")\n",
    "##AI output folders\n",
    "ai_seg_out_folder = os.path.join(base_out_folder, \"ai_results\", \"dicom_seg\")\n",
    "ai_sr_out_folder = os.path.join(base_out_folder, \"ai_results\", \"dicom_sr\")\n",
    "##IDC output folders\n",
    "idc_seg_out_folder = os.path.join(base_out_folder, \"idc_expert_results\", \"dicom_seg\")\n",
    "idc_sr_out_folder = os.path.join(base_out_folder, \"idc_expert_results\", \"dicom_sr\")\n",
    "##second set of segmentations\n",
    "idc_seg_add_out_folder = os.path.join(base_out_folder, \"idc_expert_results_add\", \"dicom_seg\")\n",
    "idc_sr_add_out_folder = os.path.join(base_out_folder, \"idc_expert_results_add\", \"dicom_sr\")\n",
    "##eval folders\n",
    "eval_out_folder = os.path.join(base_out_folder, \"quant_eval_results\")\n",
    "eval_add_out_folder = os.path.join(base_out_folder, \"quant_eval_results_add\")\n",
    "##papermill notebooks log outputs\n",
    "papermill_logs_out_folder = os.path.join(base_out_folder, \"papermill_logs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "asVnhb-VqSEs"
   },
   "outputs": [],
   "source": [
    "def setup_folders():\n",
    "  \"\"\"\n",
    "  creates environment data and results folder, first deletes current ones and then creates new ones\n",
    "  \"\"\"\n",
    "  for folder_setup in [TASK_OUT_UNZIP,\n",
    "                       mhub_seg_out_unzip,\n",
    "                       mhub_sr_out_unzip,\n",
    "                       eval_seg_out_unzip,\n",
    "                       idc_expert_seg_out_unzip,\n",
    "                       idc_expert_sr_out_unzip,\n",
    "                       base_out_folder,\n",
    "                       ai_seg_out_folder,\n",
    "                       ai_sr_out_folder,\n",
    "                       idc_seg_out_folder,\n",
    "                       idc_sr_out_folder,\n",
    "                       idc_sr_add_out_folder,\n",
    "                       idc_seg_add_out_folder,\n",
    "                       eval_out_folder,\n",
    "                       eval_add_out_folder,\n",
    "                       papermill_logs_out_folder,\n",
    "                       eval_seg_add_out_unzip,\n",
    "                       idc_expert_seg_add_out_unzip,\n",
    "                       idc_expert_sr_add_out_unzip]:\n",
    "    !rm -rf $folder_setup\n",
    "    !mkdir -p $folder_setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "xKAchJCdqSEs"
   },
   "outputs": [],
   "source": [
    "setup_folders()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nZkcUku1rTF-"
   },
   "source": [
    "# Create model-specific output folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "pC9bARnDh7wv"
   },
   "outputs": [],
   "source": [
    "for model_name in mhub_model_name_lst:\n",
    "  #create models unzip paths\n",
    "  for path_model_create in [mhub_seg_out_unzip,\n",
    "                            mhub_sr_out_unzip,\n",
    "                            eval_seg_out_unzip,\n",
    "                            eval_seg_add_out_unzip]:\n",
    "    temp_path = os.path.join(path_model_create, model_name)\n",
    "    !mkdir -p $temp_path\n",
    "  ##create models out paths\n",
    "  for path_model_create in [ai_seg_out_folder, ai_sr_out_folder,\n",
    "                            eval_out_folder, eval_add_out_folder,\n",
    "                            papermill_logs_out_folder]:\n",
    "    temp_path = os.path.join(path_model_create, model_name)\n",
    "    !mkdir -p $temp_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Le7E71TFsAaY"
   },
   "source": [
    "# Unzip all outputs and aggregate into final folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Jaaby-SzsETT"
   },
   "source": [
    "mhub ai segs -- Unzip and move to output directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HjtzB1F_sVAH"
   },
   "outputs": [],
   "source": [
    "for idx_model, model_name in enumerate(mhub_model_name_lst):\n",
    "  #download archive\n",
    "  archive_temp = mhubCompressedOutputFiles[idx_model]\n",
    "  archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "  zip_ddl_path = os.path.join(mhub_seg_out_unzip, model_name)\n",
    "  # !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "  !cp $archive_temp $zip_ddl_path\n",
    "  zip_path = os.path.join(mhub_seg_out_unzip, model_name, archive_filemame)\n",
    "  !lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "  #move archive contents to output folder\n",
    "  dcm_files_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"*.dcm\"), recursive=True)\n",
    "  ##get image_serieUID\n",
    "  ##define output path for each DCM file, add to the .dcm  a prefix image_serieUID\n",
    "  for in_dicom_file_path in dcm_files_lst:\n",
    "    image_segmented_serieUID = in_dicom_file_path.split(\"/\")[-2].strip()\n",
    "    ##make sure output_path_folder is defined\n",
    "    out_folder_path = os.path.join(ai_seg_out_folder, model_name,\n",
    "                 image_segmented_serieUID)\n",
    "    !mkdir -p $out_folder_path\n",
    "    out_dcm_file_path = os.path.join(out_folder_path,\n",
    "                                     image_segmented_serieUID+\"_\"+in_dicom_file_path.split(\"/\")[-1].strip())\n",
    "    #copy to output folder\n",
    "    !cp $in_dicom_file_path $out_dcm_file_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jbHwPJNzsGFm"
   },
   "source": [
    "Unzip mhub ai srs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z5KoONTFsVlz"
   },
   "outputs": [],
   "source": [
    "for idx_model, model_name in enumerate(mhub_model_name_lst):\n",
    "  archive_temp = radsAiCompressedOutputFiles[idx_model]\n",
    "  archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "  zip_ddl_path = os.path.join(mhub_sr_out_unzip, model_name)\n",
    "  # !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "  !cp $archive_temp $zip_ddl_path\n",
    "  zip_path = os.path.join(mhub_sr_out_unzip, model_name, archive_filemame)\n",
    "  !lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "  #move archive contents to output folder\n",
    "  dcm_files_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"*.dcm\"), recursive=True)\n",
    "  ##get image_serieUID\n",
    "  ##define output path for each DCM file, add to the .dcm  a prefix image_serieUID\n",
    "  for in_dicom_file_path in dcm_files_lst:\n",
    "    image_segmented_serieUID = in_dicom_file_path.split(\"/\")[-2].strip()\n",
    "    ##make sure output_path_folder is defined\n",
    "    out_folder_path = os.path.join(ai_sr_out_folder, model_name,\n",
    "                 image_segmented_serieUID)\n",
    "    !mkdir -p $out_folder_path\n",
    "    out_dcm_file_path = os.path.join(out_folder_path,\n",
    "                                     image_segmented_serieUID+\"_\"+in_dicom_file_path.split(\"/\")[-1].strip())\n",
    "    #copy to output folder\n",
    "    !cp $in_dicom_file_path $out_dcm_file_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nUUQr4wpsJca"
   },
   "source": [
    "Unzip quant eval results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DF77mTLTsWYC",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for idx_model, model_name in enumerate(mhub_model_name_lst):\n",
    "  archive_temp = evalCompressedOutputFiles[idx_model]\n",
    "  archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "  zip_ddl_path = os.path.join(eval_seg_out_unzip, model_name)\n",
    "  # !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "  !cp $archive_temp $zip_ddl_path\n",
    "  zip_path = os.path.join(eval_seg_out_unzip, model_name, archive_filemame)\n",
    "  !lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "  #move archive contents to output folder\n",
    "  eval_file_path = glob.glob(os.path.join(zip_ddl_path, \"**\", \"output_eval.csv\"), recursive=True)[0]\n",
    "  ##get image_serieUID\n",
    "  out_folder_path = os.path.join(eval_out_folder, model_name)\n",
    "  !mkdir -p $out_folder_path\n",
    "  out_eval_file_path = os.path.join(out_folder_path, eval_file_path.split(\"/\")[-1].strip())\n",
    "  #copy to output folder\n",
    "  !cp $eval_file_path $out_eval_file_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d7B20bROJcQa"
   },
   "source": [
    "Unzip of eval results of second set of annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "OxVpPbaSJcQa"
   },
   "outputs": [],
   "source": [
    "for idx_model, model_name in enumerate(mhub_model_name_lst):\n",
    "  archive_temp = evalAddCompressedOutputFiles[idx_model]\n",
    "  archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "  zip_ddl_path = os.path.join(eval_seg_add_out_unzip, model_name)\n",
    "  # !gsutil -m cp -r {archive_temp} $zip_ddl_path\n",
    "  !cp $archive_temp $zip_ddl_path\n",
    "  zip_path = os.path.join(eval_seg_add_out_unzip, model_name, archive_filemame)\n",
    "  !lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "  eval_file_path_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"output_eval.csv\"), recursive=True)\n",
    "  print(eval_file_path_lst)\n",
    "  #move archive contents to output folder\n",
    "  if len(eval_file_path_lst) > 0:\n",
    "    eval_file_path = eval_file_path_lst[0]\n",
    "    ##get image_serieUID\n",
    "    out_folder_path = os.path.join(eval_add_out_folder, model_name)\n",
    "    !mkdir -p $out_folder_path\n",
    "    out_eval_file_path = os.path.join(out_folder_path, eval_file_path.split(\"/\")[-1].strip())\n",
    "    #copy to output folder\n",
    "    !cp $eval_file_path $out_eval_file_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nYbzo1vssMgN"
   },
   "source": [
    "Unzip idc expert annotations segs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rJJ-VKnVsW-t"
   },
   "outputs": [],
   "source": [
    "archive_temp = idcExpertCompressedOutputFile\n",
    "archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "zip_ddl_path = os.path.join(idc_expert_seg_out_unzip)\n",
    "# !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "!cp $archive_temp $zip_ddl_path\n",
    "zip_path = os.path.join(idc_expert_seg_out_unzip, archive_filemame)\n",
    "!lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "#move archive contents to output folder\n",
    "dcm_files_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"*.dcm\"), recursive=True)\n",
    "##get image_serieUID\n",
    "##define output path for each DCM file, add to the .dcm  a prefix image_serieUID\n",
    "for in_dicom_file_path in dcm_files_lst:\n",
    "  image_segmented_serieUID = in_dicom_file_path.split(\"/\")[-2].strip()\n",
    "  ##make sure output_path_folder is defined\n",
    "  out_folder_path = os.path.join(idc_seg_out_folder,\n",
    "                image_segmented_serieUID)\n",
    "  !mkdir -p $out_folder_path\n",
    "  out_dcm_file_path = os.path.join(out_folder_path,\n",
    "                                    image_segmented_serieUID+\"_\"+in_dicom_file_path.split(\"/\")[-1].strip())\n",
    "  #copy to output folder\n",
    "  !cp $in_dicom_file_path $out_dcm_file_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "LhxXSaNRJcQb"
   },
   "source": [
    "Unzip second set of IDC expert annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZT0Vv0JZJcQb"
   },
   "outputs": [],
   "source": [
    "archive_temp = idcExpertAddCompressedOutputFile\n",
    "archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "zip_ddl_path = os.path.join(idc_expert_seg_add_out_unzip)\n",
    "# !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "!cp $archive_temp $zip_ddl_path\n",
    "zip_path = os.path.join(idc_expert_seg_add_out_unzip, archive_filemame)\n",
    "!lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "#move archive contents to output folder\n",
    "dcm_files_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"*.dcm\"), recursive=True)\n",
    "##get image_serieUID\n",
    "##define output path for each DCM file, add to the .dcm  a prefix image_serieUID\n",
    "if len(dcm_files_lst) > 0:#second set of annotations might be empty\n",
    "    for in_dicom_file_path in dcm_files_lst:\n",
    "      image_segmented_serieUID = in_dicom_file_path.split(\"/\")[-2].strip()\n",
    "      ##make sure output_path_folder is defined\n",
    "      out_folder_path = os.path.join(idc_seg_add_out_folder,\n",
    "                    image_segmented_serieUID)\n",
    "      !mkdir -p $out_folder_path\n",
    "      out_dcm_file_path = os.path.join(out_folder_path,\n",
    "                                        image_segmented_serieUID+\"_\"+in_dicom_file_path.split(\"/\")[-1].strip())\n",
    "      #copy to output folder\n",
    "      !cp $in_dicom_file_path $out_dcm_file_path\n",
    "else:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "w-mtz3EWsRLD"
   },
   "source": [
    "Unzip idc expert annotations srs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lMdEh-YI-KZC"
   },
   "outputs": [],
   "source": [
    "archive_temp = radsIdcExpertCompressedOutputFile\n",
    "archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "zip_ddl_path = os.path.join(idc_expert_sr_out_unzip)\n",
    "# !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "!cp $archive_temp $zip_ddl_path\n",
    "zip_path = os.path.join(idc_expert_sr_out_unzip,archive_filemame)\n",
    "!lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "#move archive contents to output folder\n",
    "dcm_files_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"*.dcm\"), recursive=True)\n",
    "##get image_serieUID\n",
    "##define output path for each DCM file, add to the .dcm  a prefix image_serieUID\n",
    "for in_dicom_file_path in dcm_files_lst:\n",
    "  image_segmented_serieUID = in_dicom_file_path.split(\"/\")[-2].strip()\n",
    "  ##make sure output_path_folder is defined\n",
    "  out_folder_path = os.path.join(idc_sr_out_folder,\n",
    "                image_segmented_serieUID)\n",
    "  !mkdir -p $out_folder_path\n",
    "  out_dcm_file_path = os.path.join(out_folder_path,\n",
    "                                    image_segmented_serieUID+\"_\"+in_dicom_file_path.split(\"/\")[-1].strip())\n",
    "  #copy to output folder\n",
    "  !cp $in_dicom_file_path $out_dcm_file_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qvOoPLCkJcQc"
   },
   "source": [
    "Unzip second set of idc srs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "p_t4SB8FJcQc"
   },
   "outputs": [],
   "source": [
    "archive_temp = radsIdcExpertAddCompressedOutputFile\n",
    "archive_filemame = archive_temp.split(\"/\")[-1]\n",
    "zip_ddl_path = os.path.join(idc_expert_sr_add_out_unzip)\n",
    "# !gsutil -m cp -r $archive_temp $zip_ddl_path\n",
    "!cp $archive_temp $zip_ddl_path\n",
    "zip_path = os.path.join(idc_expert_sr_add_out_unzip,archive_filemame)\n",
    "!lz4 -dc < $zip_path | tar xvf - -C $zip_ddl_path\n",
    "#move archive contents to output folder\n",
    "dcm_files_lst = glob.glob(os.path.join(zip_ddl_path, \"**\", \"*.dcm\"), recursive=True)\n",
    "##get image_serieUID\n",
    "##define output path for each DCM file, add to the .dcm  a prefix image_serieUID\n",
    "if len(dcm_files_lst) > 0:#second set of annotations might be empty\n",
    "    for in_dicom_file_path in dcm_files_lst:\n",
    "      image_segmented_serieUID = in_dicom_file_path.split(\"/\")[-2].strip()\n",
    "      ##make sure output_path_folder is defined\n",
    "      out_folder_path = os.path.join(idc_sr_add_out_folder,\n",
    "                    image_segmented_serieUID)\n",
    "      !mkdir -p $out_folder_path\n",
    "      out_dcm_file_path = os.path.join(out_folder_path,\n",
    "                                        image_segmented_serieUID+\"_\"+in_dicom_file_path.split(\"/\")[-1].strip())\n",
    "      #copy to output folder\n",
    "      !cp $in_dicom_file_path $out_dcm_file_path\n",
    "else:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CBhAwt6Uvu73"
   },
   "source": [
    "Move papermill logs to archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aIsxwG6-vZ9Y"
   },
   "outputs": [],
   "source": [
    "for idx_model, model_name in enumerate(mhub_model_name_lst):\n",
    "  #get papermill logs for ai_seg combine,ai_sr,eval_quant tasks\n",
    "  papermill_combine_ai_seg_path = glob.glob(os.path.join(mhub_seg_out_unzip, model_name, \"**\", \"*.ipynb\"), recursive=True)[0]\n",
    "  papermil_ai_sr_path = glob.glob(os.path.join(mhub_sr_out_unzip, model_name, \"**\", \"*.ipynb\"), recursive=True)[0]\n",
    "  papermil_eval_path = glob.glob(os.path.join(eval_seg_out_unzip, model_name, \"**\", \"*.ipynb\"), recursive=True)[0]\n",
    "  ##copy MHUB outputs and eval to archive folder\n",
    "  out_papermil_ai_folder = os.path.join(papermill_logs_out_folder, model_name)\n",
    "  !mkdir -p $out_papermil_ai_folder\n",
    "  #copy to output folder\n",
    "  ##define out paths\n",
    "  out_papermil_ai_sr_file_path = os.path.join(out_papermil_ai_folder, papermil_ai_sr_path.split(\"/\")[-1])\n",
    "  out_papermil_ai_seg_file_path = os.path.join(out_papermil_ai_folder, papermill_combine_ai_seg_path.split(\"/\")[-1])\n",
    "  out_papermil_eval_file_path = os.path.join(out_papermil_ai_folder, papermil_eval_path.split(\"/\")[-1])\n",
    "  !cp $papermil_ai_sr_path $out_papermil_ai_sr_file_path\n",
    "  !cp $papermill_combine_ai_seg_path $out_papermil_ai_seg_file_path\n",
    "  !cp $papermil_eval_path $out_papermil_eval_file_path\n",
    "#get papermill logs for idc seg combine, idc_sr tasks\n",
    "papermil_idc_seg_path = glob.glob(os.path.join(idc_expert_seg_out_unzip, \"**\", \"*.ipynb\"), recursive=True)[0]\n",
    "papermil_idc_sr_path = glob.glob(os.path.join(idc_expert_sr_out_unzip, \"**\", \"*.ipynb\"), recursive=True)[0]\n",
    "out_papermill_idc_expert_path_folder =  os.path.join(papermill_logs_out_folder, \"idc_expert_seg\")\n",
    "!mkdir -p $out_papermill_idc_expert_path_folder\n",
    "out_papermil_idc_seg_path = os.path.join(out_papermill_idc_expert_path_folder,\n",
    "                                       papermil_idc_seg_path.split(\"/\")[-1])\n",
    "out_papermil_idc_sr_path = os.path.join(out_papermill_idc_expert_path_folder,\n",
    "                                      papermil_idc_sr_path.split(\"/\")[-1])\n",
    "#copy to output folder\n",
    "!cp $papermil_idc_seg_path $out_papermil_idc_seg_path\n",
    "!cp $papermil_idc_sr_path $out_papermil_idc_sr_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bONyOOfI5mvM"
   },
   "source": [
    "# Create LZ4 archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "f7E-f4us5qeL"
   },
   "outputs": [],
   "source": [
    "output_lz4_archive = os.path.join(OUTPUT_PATH, \"agg_archive.tar.lz4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "T-k08rDR5o88"
   },
   "outputs": [],
   "source": [
    "# # Compress output data and move it to Cromwell root directory\n",
    "!tar -C $OUTPUT_PATH -cvf - aggregated_results | lz4 > $output_lz4_archive"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Tags",
  "colab": {
   "gpuType": "T4",
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}