PROV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Provenance Demo\n",
    "\n",
    "* https://pypi.org/project/prov/\n",
    "* https://github.com/cehbrecht/provenance-demo\n",
    "\n",
    "\n",
    "Python PROV libray:\n",
    "* https://prov.readthedocs.io/en/latest/index.html\n",
    "* https://openprovenance.org/store/\n",
    "* https://trungdong.github.io/prov-python-short-tutorial.html\n",
    "* https://nbviewer.ipython.org/github/trungdong/notebooks/blob/master/PROV%20Tutorial.ipynb\n",
    "* https://github.com/trungdong/prov/blob/master/src/prov/tests/examples.py\n",
    "\n",
    "ESMValTool Provenance:\n",
    "* https://github.com/ESMValGroup/ESMValCore/blob/master/esmvalcore/_provenance.py\n",
    "* https://docs.esmvaltool.org/en/latest/community/diagnostic.html?highlight=provenance#recording-provenance\n",
    "* https://esmvaltool.cloud.dkrz.de/shared/esmvaltool/v2.0.0/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prov.model import ProvDocument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new provenance document\n",
    "doc = ProvDocument()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Namespaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Declaring namespaces for various prefixes used in the example\n",
    "ROOCS_URI_PREFIX = 'https://roocs.org/'\n",
    "doc.add_namespace('software', uri=ROOCS_URI_PREFIX + 'software')\n",
    "doc.add_namespace('project', ROOCS_URI_PREFIX + 'project')  # copernicus, roocs\n",
    "doc.add_namespace('workflow', ROOCS_URI_PREFIX + 'workflow')  # workflow description\n",
    "doc.add_namespace('operator', ROOCS_URI_PREFIX + 'operator')  # task, job, calculation, algorithm\n",
    "doc.add_namespace('parameter', ROOCS_URI_PREFIX + 'parameter') # operator parameter, option\n",
    "doc.add_namespace('file', ROOCS_URI_PREFIX + 'file')  # netcdf, plots\n",
    "doc.add_namespace('attribute', ROOCS_URI_PREFIX + 'attribute')  # netcdf attributes, headers variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Software"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "daops = doc.activity('software:daops==v0.2.0')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_cds = doc.agent('project:Copernicus Climate Data Store')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attributes = {'attribute:variable': 'tas'}\n",
    "ds_mpi = doc.entity('file:/data/cmip6/mpi_tas_2000-2010.nc', attributes)\n",
    "ds_ipsl = doc.entity('file:/data/cmip6/ipsl_tas_2000-2010.nc', attributes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Operators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "op_subset = doc.activity('operator:subset', other_attributes={'parameter:time': '2005'})\n",
    "op_diff = doc.activity('operator:diff')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create workflow\n",
    "wf_diff = doc.entity('workflow:diff.json')\n",
    "\n",
    "# Relate workflow to project\n",
    "doc.wasAttributedTo(wf_diff, project_cds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Subset Operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subset started by daops\n",
    "doc.start(op_subset, starter=daops, trigger=wf_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generated output file for mpi dataet\n",
    "output1 = doc.entity('file:mpi_tas_2005.nc', attributes)\n",
    "doc.wasDerivedFrom(output1, ds_mpi, activity=op_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generated output file for ipsl dataset\n",
    "output2 = doc.entity('file:ipsl_tas_2005.nc', attributes)\n",
    "doc.wasDerivedFrom(output2, ds_ipsl, activity=op_subset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Diff Operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# diff started by daops\n",
    "doc.start(op_diff, starter=daops, trigger=wf_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generated output\n",
    "output_diff = doc.entity('file:diff_tas_2005.nc', attributes)\n",
    "doc.wasDerivedFrom(output_diff, output1, activity=op_diff)\n",
    "doc.wasDerivedFrom(output_diff, output2, activity=op_diff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show Provenance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(doc.get_provn())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "summerschool_2022",
   "language": "python",
   "name": "summerschool_2022"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}