diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43ef6f845..4cf5e5788 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: docs/notes/ ) - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.5 + rev: v0.8.6 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes] diff --git a/docs/bio-registries.ipynb b/docs/bio-registries.ipynb index 5a3b64bd9..25479e4d5 100644 --- a/docs/bio-registries.ipynb +++ b/docs/bio-registries.ipynb @@ -601,7 +601,9 @@ "metadata": {}, "outputs": [], "source": [ - "bt.Gene.validate([\"ENSG00000000419\", \"ENSMUSG00002076988\"], field=bt.Gene.ensembl_gene_id)" + "bt.Gene.validate(\n", + " [\"ENSG00000000419\", \"ENSMUSG00002076988\"], field=bt.Gene.ensembl_gene_id\n", + ")" ] }, { @@ -721,7 +723,9 @@ "outputs": [], "source": [ "# validate against the NCBI Taxonomy\n", - "bt.Organism.validate([\"iris setosa\", \"iris versicolor\", \"iris virginica\"], source=source)" + "bt.Organism.validate(\n", + " [\"iris setosa\", \"iris versicolor\", \"iris virginica\"], source=source\n", + ")" ] }, { diff --git a/docs/curate-any.ipynb b/docs/curate-any.ipynb index f2a1966cb..25805c3ef 100644 --- a/docs/curate-any.ipynb +++ b/docs/curate-any.ipynb @@ -54,9 +54,24 @@ "import zarr\n", "import numpy as np\n", "\n", - "data = zarr.create((10,), dtype=[('value', 'f8'), (\"gene\", \"U15\"), ('disease', 'U16')], store='data.zarr')\n", - "data[\"gene\"] = [\"ENSG00000139618\", \"ENSG00000141510\", \"ENSG00000133703\", \"ENSG00000157764\", \"ENSG00000171862\", \"ENSG00000091831\", \"ENSG00000141736\", \"ENSG00000133056\", \"ENSG00000146648\", \"ENSG00000118523\"]\n", - "data[\"disease\"] = np.random.choice(['MONDO:0004975', 'MONDO:0004980'], 10)" + "data = zarr.create(\n", + " (10,),\n", + " dtype=[(\"value\", \"f8\"), (\"gene\", \"U15\"), (\"disease\", \"U16\")],\n", + " store=\"data.zarr\",\n", + ")\n", + "data[\"gene\"] = [\n", + " \"ENSG00000139618\",\n", + " \"ENSG00000141510\",\n", + " \"ENSG00000133703\",\n", + " \"ENSG00000157764\",\n", + " \"ENSG00000171862\",\n", + " \"ENSG00000091831\",\n", + " \"ENSG00000141736\",\n", + " \"ENSG00000133056\",\n", + " \"ENSG00000146648\",\n", + " \"ENSG00000118523\",\n", + "]\n", + "data[\"disease\"] = np.random.default_rng().choice([\"MONDO:0004975\", \"MONDO:0004980\"], 10)" ] }, { @@ -164,9 +179,9 @@ "outputs": [], "source": [ "projects = ln.ULabel.from_values(\n", - " [\"Project A\", \"Project B\"], \n", - " field=ln.ULabel.name, \n", - " create=True, # create non-existing labels rather than attempting to load them from the database\n", + " [\"Project A\", \"Project B\"],\n", + " field=ln.ULabel.name,\n", + " create=True, # create non-existing labels rather than attempting to load them from the database\n", ")\n", "ln.save(projects)" ] @@ -256,8 +271,8 @@ }, "outputs": [], "source": [ - "ln.Feature(name='project', dtype='cat[ULabel]').save()\n", - "ln.Feature(name='disease', dtype='cat[bionty.Disease]').save()\n", + "ln.Feature(name=\"project\", dtype=\"cat[ULabel]\").save()\n", + "ln.Feature(name=\"disease\", dtype=\"cat[bionty.Disease]\").save()\n", "artifact.features.add_values({\"project\": projects, \"disease\": diseases})\n", "artifact.features" ] diff --git a/docs/curate-df.ipynb b/docs/curate-df.ipynb index f1c05e8d9..92be191a6 100644 --- a/docs/curate-df.ipynb +++ b/docs/curate-df.ipynb @@ -129,11 +129,15 @@ "df = pd.DataFrame(\n", " {\n", " \"temperature\": [37.2, 36.3, 38.2],\n", - " \"cell_type\": [\"cerebral pyramidal neuron\", \"astrocytic glia\", \"oligodendrocyte\"],\n", + " \"cell_type\": [\n", + " \"cerebral pyramidal neuron\",\n", + " \"astrocytic glia\",\n", + " \"oligodendrocyte\",\n", + " ],\n", " \"assay_ontology_id\": [\"EFO:0008913\", \"EFO:0008913\", \"EFO:0008913\"],\n", - " \"donor\": [\"D0001\", \"D0002\", \"D0003\"]\n", + " \"donor\": [\"D0001\", \"D0002\", \"D0003\"],\n", " },\n", - " index = [\"obs1\", \"obs2\", \"obs3\"]\n", + " index=[\"obs1\", \"obs2\", \"obs3\"],\n", ")\n", "df" ] @@ -286,7 +290,9 @@ "outputs": [], "source": [ "# fix the cell type\n", - "df.cell_type = df.cell_type.replace({\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name})" + "df.cell_type = df.cell_type.replace(\n", + " {\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name}\n", + ")" ] }, { @@ -380,14 +386,14 @@ "\n", "X = pd.DataFrame(\n", " {\n", - " \"ENSG00000081059\": [1, 2, 3], \n", - " \"ENSG00000276977\": [4, 5, 6], \n", - " \"ENSG00000198851\": [7, 8, 9], \n", - " \"ENSG00000010610\": [10, 11, 12], \n", + " \"ENSG00000081059\": [1, 2, 3],\n", + " \"ENSG00000276977\": [4, 5, 6],\n", + " \"ENSG00000198851\": [7, 8, 9],\n", + " \"ENSG00000010610\": [10, 11, 12],\n", " \"ENSG00000153563\": [13, 14, 15],\n", - " \"ENSGcorrupted\": [16, 17, 18]\n", - " }, \n", - " index=df.index # because we already curated the dataframe above, it will validate \n", + " \"ENSGcorrupted\": [16, 17, 18],\n", + " },\n", + " index=df.index, # because we already curated the dataframe above, it will validate\n", ")\n", "adata = ad.AnnData(X=X, obs=df)\n", "adata" @@ -405,9 +411,9 @@ "outputs": [], "source": [ "curate = ln.Curator.from_anndata(\n", - " adata, \n", + " adata,\n", " var_index=bt.Gene.ensembl_gene_id, # validate var.index against Gene.ensembl_gene_id\n", - " categoricals=categoricals, \n", + " categoricals=categoricals,\n", " organism=\"human\",\n", ")\n", "curate.validate()" @@ -450,7 +456,9 @@ "metadata": {}, "outputs": [], "source": [ - "adata_validated = adata[:, ~adata.var.index.isin(curate.non_validated[\"var_index\"])].copy()" + "adata_validated = adata[\n", + " :, ~adata.var.index.isin(curate.non_validated[\"var_index\"])\n", + "].copy()" ] }, { @@ -473,9 +481,9 @@ "outputs": [], "source": [ "curate = ln.Curator.from_anndata(\n", - " adata_validated, \n", + " adata_validated,\n", " var_index=bt.Gene.ensembl_gene_id, # validate var.index against Gene.ensembl_gene_id\n", - " categoricals=categoricals, \n", + " categoricals=categoricals,\n", " organism=\"human\",\n", ")\n", "curate.validate()" diff --git a/docs/curate-subclass.ipynb b/docs/curate-subclass.ipynb index 84189c803..067965563 100644 --- a/docs/curate-subclass.ipynb +++ b/docs/curate-subclass.ipynb @@ -95,10 +95,22 @@ "source": [ "# create example DataFrame that has all mandatory columns but one ('patient_age') is wrongly named\n", "data = {\n", - " 'disease': ['Alzheimer disease', 'diabetes mellitus', 'breast cancer', 'Hypertension', 'asthma'],\n", - " 'phenotype': ['Mental deterioration', 'Hyperglycemia', 'Tumor growth', 'Increased blood pressure', 'Airway inflammation'],\n", - " 'developmental_stage': ['Adult', 'Adult', 'Adult', 'Adult', 'Child'],\n", - " 'patient_age': [70, 55, 60, 65, 12],\n", + " \"disease\": [\n", + " \"Alzheimer disease\",\n", + " \"diabetes mellitus\",\n", + " \"breast cancer\",\n", + " \"Hypertension\",\n", + " \"asthma\",\n", + " ],\n", + " \"phenotype\": [\n", + " \"Mental deterioration\",\n", + " \"Hyperglycemia\",\n", + " \"Tumor growth\",\n", + " \"Increased blood pressure\",\n", + " \"Airway inflammation\",\n", + " ],\n", + " \"developmental_stage\": [\"Adult\", \"Adult\", \"Adult\", \"Adult\", \"Child\"],\n", + " \"patient_age\": [70, 55, 60, 65, 12],\n", "}\n", "df = pd.DataFrame(data)\n", "df" @@ -148,15 +160,21 @@ "phenotype_lo = bt.Phenotype.public().lookup()\n", "developmental_stage_lo = bt.DevelopmentalStage.public().lookup()\n", "\n", - "df[\"disease\"] = df[\"disease\"].replace({\"Hypertension\": disease_lo.hypertensive_disorder.name})\n", - "df[\"phenotype\"] = df[\"phenotype\"].replace({\n", - " \"Tumor growth\": phenotype_lo.neoplasm.name,\n", - " \"Airway inflammation\": phenotype_lo.bronchitis.name}\n", + "df[\"disease\"] = df[\"disease\"].replace(\n", + " {\"Hypertension\": disease_lo.hypertensive_disorder.name}\n", + ")\n", + "df[\"phenotype\"] = df[\"phenotype\"].replace(\n", + " {\n", + " \"Tumor growth\": phenotype_lo.neoplasm.name,\n", + " \"Airway inflammation\": phenotype_lo.bronchitis.name,\n", + " }\n", + ")\n", + "df[\"developmental_stage\"] = df[\"developmental_stage\"].replace(\n", + " {\n", + " \"Adult\": developmental_stage_lo.adolescent_stage.name,\n", + " \"Child\": developmental_stage_lo.child_stage.name,\n", + " }\n", ")\n", - "df[\"developmental_stage\"] = df[\"developmental_stage\"].replace({\n", - " \"Adult\": developmental_stage_lo.adolescent_stage.name,\n", - " \"Child\": developmental_stage_lo.child_stage.name\n", - "})\n", "\n", "ehrcurator.validate()" ] diff --git a/docs/ehrcurator.py b/docs/ehrcurator.py index caaf6d24e..722e8dcf3 100644 --- a/docs/ehrcurator.py +++ b/docs/ehrcurator.py @@ -1,7 +1,7 @@ import bionty as bt import pandas as pd -from lamindb.core import DataFrameCurator, Record, logger -from lamindb.core.types import UPathStr, FieldAttr +from lamindb.core import DataFrameCurator, logger +from lamindb.core.types import UPathStr __version__ = "0.1.0" diff --git a/docs/faq/key.ipynb b/docs/faq/key.ipynb index 030845a49..e3da2539b 100644 --- a/docs/faq/key.ipynb +++ b/docs/faq/key.ipynb @@ -565,7 +565,7 @@ "try:\n", " artifact_key_3.save()\n", " artifact_key_4.save()\n", - "except Exception as e:\n", + "except Exception:\n", " print(\n", " \"It is not possible to save artifacts to the same key. This results in an\"\n", " \" Integrity Error!\"\n", @@ -615,8 +615,8 @@ "\n", "for root, _, artifacts in os.walk(\"complex_biological_project/raw\"):\n", " for artifactname in artifacts:\n", - " file_path = os.path.join(root, artifactname)\n", - " key_path = file_path.removeprefix(\"complex_biological_project\")\n", + " file_path = Path(root) / artifactname\n", + " key_path = str(file_path).removeprefix(\"complex_biological_project\")\n", " ln_artifact = ln.Artifact(file_path, key=key_path)\n", " ln_artifact.save()" ] @@ -653,7 +653,7 @@ "all_data_paths = []\n", "for root, _, artifacts in os.walk(\"complex_biological_project/raw\"):\n", " for artifactname in artifacts:\n", - " file_path = os.path.join(root, artifactname)\n", + " file_path = Path(root) / artifactname\n", " all_data_paths.append(file_path)\n", "\n", "all_data_artifacts = []\n", @@ -695,8 +695,8 @@ "source": [ "for root, _, artifacts in os.walk(\"complex_biological_project/raw\"):\n", " for artifactname in artifacts:\n", - " file_path = os.path.join(root, artifactname)\n", - " key_path = file_path.removeprefix(\"complex_biological_project\")\n", + " file_path = Path(root) / artifactname\n", + " key_path = str(file_path).removeprefix(\"complex_biological_project\")\n", " ln_artifact = ln.Artifact(file_path, key=key_path)\n", " ln_artifact.save()\n", "\n", @@ -786,8 +786,8 @@ "source": [ "for root, _, artifacts in os.walk(\"complex_biological_project/preprocessed\"):\n", " for artifactname in artifacts:\n", - " file_path = os.path.join(root, artifactname)\n", - " key_path = file_path.removeprefix(\"complex_biological_project\")\n", + " file_path = Path(root) / artifactname\n", + " key_path = str(file_path).removeprefix(\"complex_biological_project\")\n", "\n", " print(file_path)\n", " print()\n", diff --git a/docs/registries.ipynb b/docs/registries.ipynb index ca103ed7b..2daaad29e 100644 --- a/docs/registries.ipynb +++ b/docs/registries.ipynb @@ -95,7 +95,15 @@ "\n", "# Ingest dataset2\n", "adata2 = datasets.small_dataset2(format=\"anndata\")\n", - "curator = ln.Curator.from_anndata(adata2, var_index=bt.Gene.symbol, categoricals={\"cell_medium\": ln.ULabel.name, \"cell_type_by_model\": bt.CellType.name}, organism=\"human\")\n", + "curator = ln.Curator.from_anndata(\n", + " adata2,\n", + " var_index=bt.Gene.symbol,\n", + " categoricals={\n", + " \"cell_medium\": ln.ULabel.name,\n", + " \"cell_type_by_model\": bt.CellType.name,\n", + " },\n", + " organism=\"human\",\n", + ")\n", "artifact2 = curator.save_artifact(key=\"example_datasets/dataset2.h5ad\")\n", "artifact2.features.add_values(adata2.uns)" ] @@ -151,7 +159,15 @@ }, "outputs": [], "source": [ - "ln.Artifact.df(include=[\"created_by__name\", \"ulabels__name\", \"cell_types__name\", \"feature_sets__registry\", \"suffix\"])" + "ln.Artifact.df(\n", + " include=[\n", + " \"created_by__name\",\n", + " \"ulabels__name\",\n", + " \"cell_types__name\",\n", + " \"feature_sets__registry\",\n", + " \"suffix\",\n", + " ]\n", + ")" ] }, { @@ -449,7 +465,7 @@ }, "outputs": [], "source": [ - "ln.Artifact.filter(created_by__handle__startswith=\"testuse\").df() " + "ln.Artifact.filter(created_by__handle__startswith=\"testuse\").df()" ] }, { @@ -477,7 +493,7 @@ "cd8a = bt.Gene.get(symbol=\"CD8A\")\n", "# query for all feature sets that contain CD8A\n", "feature_sets_with_cd8a = ln.FeatureSet.filter(genes=cd8a).all()\n", - "# get all artifacts \n", + "# get all artifacts\n", "ln.Artifact.filter(feature_sets__in=feature_sets_with_cd8a).df()" ] }, diff --git a/docs/storage/prepare-transfer-local-to-cloud.ipynb b/docs/storage/prepare-transfer-local-to-cloud.ipynb index 343a132a2..2ea95a10d 100644 --- a/docs/storage/prepare-transfer-local-to-cloud.ipynb +++ b/docs/storage/prepare-transfer-local-to-cloud.ipynb @@ -44,10 +44,14 @@ "metadata": {}, "outputs": [], "source": [ - "artifact = ln.Artifact.from_df(pd.DataFrame({'a': [1, 2, 3]}), description='test-transfer-to-cloud')\n", + "artifact = ln.Artifact.from_df(\n", + " pd.DataFrame({\"a\": [1, 2, 3]}), description=\"test-transfer-to-cloud\"\n", + ")\n", "artifact.save()\n", "\n", - "features = bt.CellMarker.from_values([\"PD1\", \"CD21\"], field=bt.CellMarker.name, organism=\"human\")\n", + "features = bt.CellMarker.from_values(\n", + " [\"PD1\", \"CD21\"], field=bt.CellMarker.name, organism=\"human\"\n", + ")\n", "ln.save(features)\n", "artifact.features.add_feature_set(ln.FeatureSet(features), slot=\"var\")\n", "\n", diff --git a/docs/storage/transfer-local-to-cloud.ipynb b/docs/storage/transfer-local-to-cloud.ipynb index 1337ccc23..c71dff011 100644 --- a/docs/storage/transfer-local-to-cloud.ipynb +++ b/docs/storage/transfer-local-to-cloud.ipynb @@ -37,7 +37,8 @@ " features_sets.delete()\n", " experiments.delete()\n", "\n", - "artifacts = ln.Artifact.filter(description='test-transfer-to-cloud').all()\n", + "\n", + "artifacts = ln.Artifact.filter(description=\"test-transfer-to-cloud\").all()\n", "for artifact in artifacts:\n", " cleanup(artifact)" ] @@ -48,7 +49,9 @@ "metadata": {}, "outputs": [], "source": [ - "artifact = ln.Artifact.using(\"testuser1/test-transfer-to-cloud\").get(description='test-transfer-to-cloud')\n", + "artifact = ln.Artifact.using(\"testuser1/test-transfer-to-cloud\").get(\n", + " description=\"test-transfer-to-cloud\"\n", + ")\n", "artifact.describe()" ] }, diff --git a/docs/storage/upload.ipynb b/docs/storage/upload.ipynb index 27d3bb435..f80617d4b 100644 --- a/docs/storage/upload.ipynb +++ b/docs/storage/upload.ipynb @@ -99,7 +99,7 @@ "metadata": {}, "outputs": [], "source": [ - "pbmc68k_h5ad = ln.Artifact.from_anndata(pbmc68k, key=f\"test-upload/pbmc68k.h5ad\")" + "pbmc68k_h5ad = ln.Artifact.from_anndata(pbmc68k, key=\"test-upload/pbmc68k.h5ad\")" ] }, { @@ -236,7 +236,9 @@ "outputs": [], "source": [ "with pytest.raises(FileNotFoundError):\n", - " non_existent_h5ad = ln.Artifact(\"s3://lamindb-ci/test-upload/non_existent_file.h5ad\")" + " non_existent_h5ad = ln.Artifact(\n", + " \"s3://lamindb-ci/test-upload/non_existent_file.h5ad\"\n", + " )" ] }, { @@ -255,7 +257,9 @@ "outputs": [], "source": [ "with pytest.raises((FileNotFoundError, PermissionError)):\n", - " non_existent_h5ad = ln.Artifact(\"s3://non_existent_bucket_6612366/non_existent_file.h5ad\")" + " non_existent_h5ad = ln.Artifact(\n", + " \"s3://non_existent_bucket_6612366/non_existent_file.h5ad\"\n", + " )" ] }, { diff --git a/docs/storage/vitessce.ipynb b/docs/storage/vitessce.ipynb index 9561d5999..6ef6341a0 100644 --- a/docs/storage/vitessce.ipynb +++ b/docs/storage/vitessce.ipynb @@ -34,7 +34,6 @@ "import pytest\n", "from vitessce import (\n", " VitessceConfig,\n", - " Component as cm,\n", " AnnDataWrapper,\n", ")" ] @@ -95,7 +94,9 @@ "metadata": {}, "outputs": [], "source": [ - "vitessce_config_artifact = ln.integrations.save_vitessce_config(vc, description=\"View testdata in Vitessce\")" + "vitessce_config_artifact = ln.integrations.save_vitessce_config(\n", + " vc, description=\"View testdata in Vitessce\"\n", + ")" ] }, { @@ -144,7 +145,9 @@ " ),\n", " )\n", "print(error.exconly())\n", - "assert error.exconly().startswith(\"AttributeError: 'str' object has no attribute 'path'\")\n" + "assert error.exconly().startswith(\n", + " \"AttributeError: 'str' object has no attribute 'path'\"\n", + ")" ] }, { diff --git a/docs/track.ipynb b/docs/track.ipynb index 18ef068be..8a98f5882 100644 --- a/docs/track.ipynb +++ b/docs/track.ipynb @@ -276,7 +276,9 @@ }, "outputs": [], "source": [ - "ln.Run.params.filter(learning_rate=0.01, input_dir=\"./mydataset\", preprocess_params__downsample=True).df()" + "ln.Run.params.filter(\n", + " learning_rate=0.01, input_dir=\"./mydataset\", preprocess_params__downsample=True\n", + ").df()" ] }, { @@ -405,7 +407,11 @@ }, "outputs": [], "source": [ - "assert run.params.get_values() == {'input_dir': './mydataset', 'learning_rate': 0.01, 'preprocess_params': {'downsample': True, 'normalization': 'the_good_one'}}\n", + "assert run.params.get_values() == {\n", + " \"input_dir\": \"./mydataset\",\n", + " \"learning_rate\": 0.01,\n", + " \"preprocess_params\": {\"downsample\": True, \"normalization\": \"the_good_one\"},\n", + "}\n", "\n", "# clean up test instance\n", "!rm -r ./test-track\n", diff --git a/docs/transfer.ipynb b/docs/transfer.ipynb index a4d17ac6b..d3908191d 100644 --- a/docs/transfer.ipynb +++ b/docs/transfer.ipynb @@ -60,7 +60,7 @@ }, "outputs": [], "source": [ - "# query all latest artifact versions \n", + "# query all latest artifact versions\n", "artifacts = ln.Artifact.using(\"laminlabs/lamindata\").filter(is_latest=True)\n", "\n", "# convert the QuerySet to a DataFrame and show the latest 5 versions\n", diff --git a/lamindb/_artifact.py b/lamindb/_artifact.py index c143359f1..ee9b6b59e 100644 --- a/lamindb/_artifact.py +++ b/lamindb/_artifact.py @@ -2,7 +2,6 @@ import os import shutil -from collections.abc import Mapping from pathlib import Path, PurePath, PurePosixPath from typing import TYPE_CHECKING, Any diff --git a/lamindb/_feature.py b/lamindb/_feature.py index 007d1d025..1dba55712 100644 --- a/lamindb/_feature.py +++ b/lamindb/_feature.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Literal, get_args +from typing import TYPE_CHECKING, Any, get_args import lamindb_setup as ln_setup import pandas as pd @@ -14,7 +14,6 @@ from ._query_set import RecordList from ._utils import attach_func_to_class_method -from .core._settings import settings from .core.schema import dict_schema_name_to_model_name if TYPE_CHECKING: diff --git a/lamindb/_finish.py b/lamindb/_finish.py index 94edf6d94..28214ebc7 100644 --- a/lamindb/_finish.py +++ b/lamindb/_finish.py @@ -9,14 +9,11 @@ from lamindb_setup.core.hashing import hash_file from lamindb.core.exceptions import NotebookNotSaved +from lamindb.models import Artifact, Run, Transform if TYPE_CHECKING: from pathlib import Path - from lamindb import Run, Transform - - from ._query_set import QuerySet - def get_save_notebook_message() -> str: return f"Please save the notebook in your editor (shortcut `{get_shortcut()}`) within 2 sec before calling `finish()`" @@ -32,6 +29,23 @@ def get_seconds_since_modified(filepath) -> float: return datetime.now().timestamp() - filepath.stat().st_mtime +def save_run_logs(run: Run, save_run: bool = False) -> None: + logs_path = ln_setup.settings.cache_dir / f"run_logs_{run.uid}.txt" + if logs_path.exists(): + if run.report is not None: + logger.important("overwriting run.report") + artifact = Artifact( + logs_path, + description=f"log streams of run {run.uid}", + visibility=0, + run=False, + ) + artifact.save(upload=True, print_progress=False) + run.report = artifact + if save_run: # defaults to fast because is slow + run.save() + + # this is from the get_title function in nbproject # should be moved into lamindb sooner or later def prepare_notebook( @@ -139,7 +153,7 @@ def save_context_core( format_field_value, # needs to come after lamindb was imported because of CLI use ) - from .core._context import context, is_run_from_ipython + from .core._context import context ln.settings.verbosity = "success" @@ -152,7 +166,7 @@ def save_context_core( # for notebooks, we need more work if is_ipynb: try: - import jupytext + import jupytext # noqa: F401 from nbproject.dev import ( check_consecutiveness, read_notebook, @@ -225,7 +239,7 @@ def save_context_core( if env_path.exists(): overwrite_env = True if run.environment_id is not None and from_cli: - logger.important("run.environment is already saved") + logger.important("run.environment is already saved, ignoring") overwrite_env = False if overwrite_env: hash, _ = hash_file(env_path) @@ -247,6 +261,10 @@ def save_context_core( if finished_at and run is not None: run.finished_at = datetime.now(timezone.utc) + # track logs + if run is not None and not from_cli and not is_ipynb and not is_r_notebook: + save_run_logs(run) + # track report and set is_consecutive if run is not None: if report_path is not None: diff --git a/lamindb/_query_manager.py b/lamindb/_query_manager.py index d37bc7fe5..3e709e526 100644 --- a/lamindb/_query_manager.py +++ b/lamindb/_query_manager.py @@ -8,7 +8,6 @@ from lamindb.models import Record -from .core._feature_manager import get_feature_set_by_slot_ from .core._settings import settings if TYPE_CHECKING: diff --git a/lamindb/_query_set.py b/lamindb/_query_set.py index 3c59b621a..85968590c 100644 --- a/lamindb/_query_set.py +++ b/lamindb/_query_set.py @@ -20,7 +20,6 @@ Feature, IsVersioned, Record, - Registry, Run, Transform, VisibilityChoice, diff --git a/lamindb/_record.py b/lamindb/_record.py index 2991ce393..2863ec538 100644 --- a/lamindb/_record.py +++ b/lamindb/_record.py @@ -10,7 +10,6 @@ from django.core.exceptions import ValidationError as DjangoValidationError from django.db import connections, transaction from django.db.models import ( - F, IntegerField, Manager, Q, @@ -617,8 +616,6 @@ def transfer_fk_to_default_db_bulk( def get_transfer_run(record) -> Run: - from lamindb_setup import settings as setup_settings - from lamindb.core._context import context from lamindb.core._data import WARNING_RUN_TRANSFORM diff --git a/lamindb/_save.py b/lamindb/_save.py index d4ec42d8f..4691e2b5e 100644 --- a/lamindb/_save.py +++ b/lamindb/_save.py @@ -5,11 +5,9 @@ import traceback from collections import defaultdict from datetime import datetime -from functools import partial -from typing import TYPE_CHECKING, overload +from typing import TYPE_CHECKING -import lamindb_setup -from django.db import IntegrityError, transaction +from django.db import transaction from django.utils.functional import partition from lamin_utils import logger from lamindb_setup.core.upath import LocalPathClasses diff --git a/lamindb/_ulabel.py b/lamindb/_ulabel.py index fd397f640..509d6a11b 100644 --- a/lamindb/_ulabel.py +++ b/lamindb/_ulabel.py @@ -1,17 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING - import lamindb_setup as ln_setup -from lamindb_setup.core._docs import doc_args from lamindb.models import ULabel from ._utils import attach_func_to_class_method -if TYPE_CHECKING: - from lamindb.base.types import ListLike - def __init__(self, *args, **kwargs): if len(args) == len(self._meta.concrete_fields): diff --git a/lamindb/base/types.py b/lamindb/base/types.py index 149434cac..e47bdc562 100644 --- a/lamindb/base/types.py +++ b/lamindb/base/types.py @@ -1,6 +1,5 @@ from __future__ import annotations -from enum import Enum from typing import Literal, Union import numpy as np diff --git a/lamindb/core/_context.py b/lamindb/core/_context.py index 15b575505..336dc727c 100644 --- a/lamindb/core/_context.py +++ b/lamindb/core/_context.py @@ -2,12 +2,15 @@ import builtins import hashlib +import signal +import sys +import traceback from datetime import datetime, timezone -from pathlib import Path, PurePath +from pathlib import Path from typing import TYPE_CHECKING import lamindb_setup as ln_setup -from django.db.models import F, Func, IntegerField +from django.db.models import Func, IntegerField from lamin_utils import logger from lamindb_setup.core.hashing import hash_file @@ -20,9 +23,7 @@ from ._track_environment import track_environment from .exceptions import ( InconsistentKey, - MissingContextUID, NotebookNotSaved, - NoTitleError, TrackNotCalled, UpdateContext, ) @@ -91,6 +92,80 @@ def pretty_pypackages(dependencies: dict) -> str: return " ".join(deps_list) +class LogStreamHandler: + def __init__(self, log_stream, file): + self.log_stream = log_stream + self.file = file + + def write(self, data): + self.log_stream.write(data) + self.file.write(data) + self.file.flush() + + def flush(self): + self.log_stream.flush() + self.file.flush() + + +class LogStreamTracker: + def __init__(self): + self.original_stdout = None + self.original_stderr = None + self.log_file = None + self.original_excepthook = sys.excepthook + self.is_cleaning_up = False + + def start(self, run: Run): + self.original_stdout = sys.stdout + self.original_stderr = sys.stderr + self.run = run + self.log_file_path = ( + ln_setup.settings.cache_dir / f"run_logs_{self.run.uid}.txt" + ) + self.log_file = open(self.log_file_path, "w") + sys.stdout = LogStreamHandler(self.original_stdout, self.log_file) + sys.stderr = LogStreamHandler(self.original_stderr, self.log_file) + # handle signals + signal.signal(signal.SIGTERM, self.cleanup) + signal.signal(signal.SIGINT, self.cleanup) + # handle exceptions + sys.excepthook = self.handle_exception + + def finish(self): + if self.original_stdout: + sys.stdout = self.original_stdout + sys.stderr = self.original_stderr + self.log_file.close() + + def cleanup(self, signo=None, frame=None): + from lamindb._finish import save_run_logs + + if self.original_stdout and not self.is_cleaning_up: + self.is_cleaning_up = True + if signo is not None: + signal_msg = f"\nProcess terminated by signal {signo} ({signal.Signals(signo).name})\n" + if frame: + signal_msg += ( + f"Frame info:\n{''.join(traceback.format_stack(frame))}" + ) + self.log_file.write(signal_msg) + sys.stdout = self.original_stdout + sys.stderr = self.original_stderr + self.log_file.flush() + self.log_file.close() + save_run_logs(self.run, save_run=True) + + def handle_exception(self, exc_type, exc_value, exc_traceback): + if not self.is_cleaning_up: + error_msg = f"{''.join(traceback.format_exception(exc_type, exc_value, exc_traceback))}" + if self.log_file.closed: + self.log_file = open(self.log_file_path, "a") + self.log_file.write(error_msg) + self.log_file.flush() + self.cleanup() + self.original_excepthook(exc_type, exc_value, exc_traceback) + + class Context: """Run context. @@ -120,6 +195,7 @@ def __init__(self): """A local path to the script that's running.""" self._logging_message_track: str = "" self._logging_message_imports: str = "" + self._stream_tracker: LogStreamTracker = LogStreamTracker() @property def transform(self) -> Transform | None: @@ -273,6 +349,8 @@ def track( ) self._run = run track_environment(run) + if self.transform.type != "notebook": + self._stream_tracker.start(run) logger.important(self._logging_message_track) if self._logging_message_imports: logger.important(self._logging_message_imports) @@ -432,7 +510,7 @@ class SlashCount(Func): and self.version != transform.version # type: ignore ): raise SystemExit( - f"Please pass consistent version: ln.context.version = '{transform.version}'" # type: ignore + f"✗ please pass consistent version: ln.context.version = '{transform.version}'" # type: ignore ) # test whether version was already used for another member of the family suid, vuid = (self.uid[:-4], self.uid[-4:]) @@ -442,7 +520,7 @@ class SlashCount(Func): if transform is not None and vuid != transform.uid[-4:]: better_version = bump_version_function(self.version) raise SystemExit( - f"Version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'" + f"✗ version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'" ) # make a new transform record if transform is None: @@ -504,13 +582,12 @@ class SlashCount(Func): ) if bump_revision: change_type = ( - "Re-running saved notebook" + "re-running saved notebook" if is_run_from_ipython - else "Source code changed" + else "source code changed" ) raise UpdateContext( - f"{change_type}, bump revision by setting:\n\n" - f'ln.track("{uid[:-4]}{increment_base62(uid[-4:])}")' + f'✗ {change_type}, run: ln.track("{uid[:-4]}{increment_base62(uid[-4:])}")' ) else: self._logging_message_track += f"loaded Transform('{transform.uid}')" @@ -577,6 +654,8 @@ def finish(self, ignore_non_consecutive: None | bool = None) -> None: finished_at=True, ignore_non_consecutive=ignore_non_consecutive, ) + if self.transform.type != "notebook": + self._stream_tracker.finish() context = Context() diff --git a/lamindb/core/_feature_manager.py b/lamindb/core/_feature_manager.py index afb3a34ef..39a63da73 100644 --- a/lamindb/core/_feature_manager.py +++ b/lamindb/core/_feature_manager.py @@ -14,14 +14,13 @@ from django.contrib.postgres.aggregates import ArrayAgg from django.db import connections from django.db.models import Aggregate -from lamin_utils import colors, logger +from lamin_utils import logger from lamindb_setup.core.hashing import hash_set from lamindb_setup.core.upath import create_path from rich.table import Column, Table from rich.text import Text from lamindb._feature import ( - FEATURE_DTYPES, convert_pandas_dtype_to_lamin_dtype, suggest_categorical_for_str_iterable, ) diff --git a/lamindb/core/_mapped_collection.py b/lamindb/core/_mapped_collection.py index 26d967bc1..e0ab81188 100644 --- a/lamindb/core/_mapped_collection.py +++ b/lamindb/core/_mapped_collection.py @@ -2,12 +2,10 @@ from collections import Counter from functools import reduce -from pathlib import Path from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd -from lamin_utils import logger from lamindb_setup.core.upath import UPath from .storage._anndata_accessor import ( diff --git a/lamindb/core/_settings.py b/lamindb/core/_settings.py index 2ba90ff5c..7951176a6 100644 --- a/lamindb/core/_settings.py +++ b/lamindb/core/_settings.py @@ -1,7 +1,7 @@ from __future__ import annotations import os -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING import lamindb_setup as ln_setup from lamin_utils import logger diff --git a/lamindb/core/datasets/_core.py b/lamindb/core/datasets/_core.py index 09db3a862..4d5a338c9 100644 --- a/lamindb/core/datasets/_core.py +++ b/lamindb/core/datasets/_core.py @@ -5,7 +5,6 @@ from urllib.request import urlretrieve import anndata as ad -import numpy as np import pandas as pd from upath import UPath diff --git a/lamindb/core/exceptions.py b/lamindb/core/exceptions.py index 1c053f512..7057a30e9 100644 --- a/lamindb/core/exceptions.py +++ b/lamindb/core/exceptions.py @@ -7,7 +7,6 @@ DoesNotExist ValidationError NotebookNotSaved - NoTitleError MissingContextUID UpdateContext IntegrityError @@ -79,12 +78,6 @@ class IntegrityError(Exception): pass -class NoTitleError(SystemExit): - """Notebook has no title.""" - - pass - - class MissingContextUID(SystemExit): """User didn't define transform settings.""" diff --git a/lamindb/core/fields.py b/lamindb/core/fields.py index 040b66650..171d4f612 100644 --- a/lamindb/core/fields.py +++ b/lamindb/core/fields.py @@ -9,4 +9,4 @@ """ -from lamindb.base.types import FieldAttr +from lamindb.base.types import FieldAttr # noqa: F401 diff --git a/lamindb/core/loaders.py b/lamindb/core/loaders.py index 409b2b9f8..e12031ad3 100644 --- a/lamindb/core/loaders.py +++ b/lamindb/core/loaders.py @@ -33,7 +33,6 @@ from ._settings import settings if TYPE_CHECKING: - import mudata as md from lamindb_setup.core.types import UPathStr try: diff --git a/lamindb/core/schema.py b/lamindb/core/schema.py index 08d4ca12f..a4416beb4 100644 --- a/lamindb/core/schema.py +++ b/lamindb/core/schema.py @@ -8,7 +8,7 @@ ) from lamindb_setup.core._settings_store import instance_settings_file -from lamindb.models import Feature, FeatureSet, LinkORM, Record +from lamindb.models import FeatureSet, LinkORM, Record def get_schemas_modules(instance: str | None) -> set[str]: diff --git a/lamindb/core/storage/_anndata_accessor.py b/lamindb/core/storage/_anndata_accessor.py index 7b785ad1d..e3b0114bc 100644 --- a/lamindb/core/storage/_anndata_accessor.py +++ b/lamindb/core/storage/_anndata_accessor.py @@ -17,12 +17,11 @@ from anndata.compat import _read_attr from fsspec.implementations.local import LocalFileSystem from lamin_utils import logger -from lamindb_setup.core.upath import UPath, create_mapper, infer_filesystem +from lamindb_setup.core.upath import create_mapper, infer_filesystem from packaging import version if TYPE_CHECKING: from collections.abc import Mapping - from pathlib import Path from fsspec.core import OpenFile from lamindb_setup.core.types import UPathStr diff --git a/lamindb/core/storage/objects.py b/lamindb/core/storage/objects.py index b87e58248..54d38bf20 100644 --- a/lamindb/core/storage/objects.py +++ b/lamindb/core/storage/objects.py @@ -12,7 +12,7 @@ def _mudata_is_installed(): try: - import mudata + import mudata # noqa: F401c except ImportError: return False return True diff --git a/lamindb/core/storage/paths.py b/lamindb/core/storage/paths.py index 94b575fb0..3c0b2b1d8 100644 --- a/lamindb/core/storage/paths.py +++ b/lamindb/core/storage/paths.py @@ -3,16 +3,12 @@ import shutil from typing import TYPE_CHECKING -import anndata as ad import fsspec -import pandas as pd from lamin_utils import logger from lamindb_setup.core import StorageSettings from lamindb_setup.core.upath import ( LocalPathClasses, UPath, - create_path, - infer_filesystem, ) from lamindb.core._settings import settings diff --git a/lamindb/curators/_spatial.py b/lamindb/curators/_spatial.py index 5bdbc9a1e..284c5fc45 100644 --- a/lamindb/curators/_spatial.py +++ b/lamindb/curators/_spatial.py @@ -375,7 +375,7 @@ def save_artifact( # Write the SpatialData object to a random path in tmp directory # The Artifact constructor will move it to the cache - write_path = f"{settings.cache_dir}/{random.randint(10**7, 10**8 - 1)}.zarr" # noqa: S311 + write_path = f"{settings.cache_dir}/{random.randint(10**7, 10**8 - 1)}.zarr" self._sdata.write(write_path) # Create the Artifact and associate Artifact metadata diff --git a/lamindb/integrations/_vitessce.py b/lamindb/integrations/_vitessce.py index 13e78fa76..24ad4e65c 100644 --- a/lamindb/integrations/_vitessce.py +++ b/lamindb/integrations/_vitessce.py @@ -44,8 +44,6 @@ def save_vitessce_config( # can only import here because vitessce is not a dependency from vitessce import VitessceConfig - from lamindb.core.storage import VALID_SUFFIXES - assert isinstance(vitessce_config, VitessceConfig) # noqa: S101 vc_dict = vitessce_config.to_dict() try: diff --git a/lamindb/migrations/0063_populate_latest_field.py b/lamindb/migrations/0063_populate_latest_field.py index 9c49dc9f4..ab17b06da 100644 --- a/lamindb/migrations/0063_populate_latest_field.py +++ b/lamindb/migrations/0063_populate_latest_field.py @@ -1,6 +1,6 @@ # Generated by Django 5.2 on 2024-08-12 07:46 -from django.db import migrations, models +from django.db import migrations SQL_TEMPLATE = """\ -- First, set all is_latest to False diff --git a/pyproject.toml b/pyproject.toml index 4effb35cb..2c3fd74cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,8 +165,6 @@ lint.ignore = [ "D413", # Missing argument description in the docstring "D417", - # Imports unused - "F401", # camcelcase imported as lowercase "N813", # module import not at top level of file @@ -178,14 +176,16 @@ lint.ignore = [ # Starting a process with a partial executable path "S607", # Prefer absolute imports over relative imports from parent modules - "TID252" + "TID252", + # Standard pseudo-random generators are not suitable for cryptographic purposes + "S311" ] [tool.ruff.lint.pydocstyle] convention = "google" [tool.ruff.lint.per-file-ignores] -"docs/*" = ["I"] +"docs/*" = ["I", "S101"] "tests/**/*.py" = [ "D", # docstrings are allowed to look a bit off "S101", # asserts allowed in tests... @@ -194,7 +194,9 @@ convention = "google" "PLR2004", # Magic value used in comparison, ... "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes ] +"tests/**/*.ipynb" = ["S101"] "*/__init__.py" = ["F401"] +"lamindb/core/types.py" = ["F401"] [tool.pytest.ini_options] testpaths = [ diff --git a/sub/lamin-cli b/sub/lamin-cli index c5cbc15b7..d68f2101d 160000 --- a/sub/lamin-cli +++ b/sub/lamin-cli @@ -1 +1 @@ -Subproject commit c5cbc15b7184464bc135bb8deb645eb9d778bc0f +Subproject commit d68f2101d0bb917bce14a3721b4c4c0588138b59 diff --git a/sub/lamindb-setup b/sub/lamindb-setup index b5a612f1a..d9ee37293 160000 --- a/sub/lamindb-setup +++ b/sub/lamindb-setup @@ -1 +1 @@ -Subproject commit b5a612f1af95df150d415f2611ac4c28503233ec +Subproject commit d9ee372939f484f58e9161769057ff8b5d2d4d46 diff --git a/tests/core/notebooks/no-title.ipynb b/tests/core/notebooks/no-title.ipynb index 086d86a39..7eff837b8 100644 --- a/tests/core/notebooks/no-title.ipynb +++ b/tests/core/notebooks/no-title.ipynb @@ -15,9 +15,7 @@ "metadata": {}, "outputs": [], "source": [ - "import lamindb as ln\n", - "from lamindb.core._context import NoTitleError\n", - "import pytest" + "import lamindb as ln" ] }, { diff --git a/tests/core/notebooks/with-title-initialized-consecutive-finish-not-last-cell.ipynb b/tests/core/notebooks/with-title-initialized-consecutive-finish-not-last-cell.ipynb index 6f2d351ff..656b61733 100644 --- a/tests/core/notebooks/with-title-initialized-consecutive-finish-not-last-cell.ipynb +++ b/tests/core/notebooks/with-title-initialized-consecutive-finish-not-last-cell.ipynb @@ -13,8 +13,7 @@ "metadata": {}, "outputs": [], "source": [ - "import lamindb as ln\n", - "import pytest" + "import lamindb as ln" ] }, { diff --git a/tests/core/scripts/duplicate4/script-to-test-versioning.py b/tests/core/scripts/duplicate4/script-to-test-versioning.py index 6de733b8c..4ddd532f1 100644 --- a/tests/core/scripts/duplicate4/script-to-test-versioning.py +++ b/tests/core/scripts/duplicate4/script-to-test-versioning.py @@ -1,5 +1,3 @@ import lamindb as ln -import pytest -from lamindb.core.exceptions import MissingContextUID ln.track() diff --git a/tests/core/test_collection.py b/tests/core/test_collection.py index d74545f50..717e4c173 100644 --- a/tests/core/test_collection.py +++ b/tests/core/test_collection.py @@ -6,7 +6,6 @@ import numpy as np import pandas as pd import pytest -from django.db.models.deletion import ProtectedError from lamindb import _collection from scipy.sparse import csc_matrix, csr_matrix diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 0ad9eba28..8e28b605e 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -1,10 +1,13 @@ import subprocess +import sys +import time from pathlib import Path import lamindb as ln +import lamindb_setup as ln_setup import pytest from lamindb._finish import clean_r_notebook_html, get_shortcut -from lamindb.core._context import context, get_uid_ext +from lamindb.core._context import LogStreamTracker, context from lamindb.core.exceptions import TrackNotCalled, ValidationError SCRIPTS_DIR = Path(__file__).parent.resolve() / "scripts" @@ -150,7 +153,7 @@ def test_run_scripts(): ) assert result.returncode == 1 assert ( - "Version '1' is already taken by Transform('Ro1gl7n8YrdH0000'); please set another version, e.g., ln.context.version = '1.1'" + "✗ version '1' is already taken by Transform('Ro1gl7n8YrdH0000'); please set another version, e.g., ln.context.version = '1.1'" in result.stderr.decode() ) @@ -176,7 +179,7 @@ def test_run_scripts(): ) assert result.returncode == 1 assert ( - "Please pass consistent version: ln.context.version = '2'" + "✗ please pass consistent version: ln.context.version = '2'" in result.stderr.decode() ) @@ -277,3 +280,113 @@ def test_clean_r_notebook_html(): assert title_text == "My exemplary R analysis" assert compare == comparison_path.read_text() orig_notebook_path.write_text(content.replace(get_shortcut(), "SHORTCUT")) + + +class MockRun: + def __init__(self, uid): + self.uid = uid + self.report = None + self.saved = False + + def save(self): + self.saved = True + + +def test_logstream_tracker_multiple(): + tracker1 = LogStreamTracker() + tracker2 = LogStreamTracker() + tracker3 = LogStreamTracker() + + try: + # Start trackers one by one and print messages + print("Initial stdout") + + tracker1.start(MockRun("run1")) + print("After starting tracker1") + + tracker2.start(MockRun("run2")) + print("After starting tracker2") + + tracker3.start(MockRun("run3")) + print("After starting tracker3") + + print("Testing stderr", file=sys.stderr) + + time.sleep(0.1) + + # Clean up in reverse order + tracker3.finish() + tracker2.finish() + tracker1.finish() + + # Verify log contents - each log should only contain messages after its start + expected_contents = { + 1: [ + "After starting tracker1", + "After starting tracker2", + "After starting tracker3", + "Testing stderr", + ], + 2: ["After starting tracker2", "After starting tracker3", "Testing stderr"], + 3: ["After starting tracker3", "Testing stderr"], + } + + for i in range(1, 4): + log_path = Path(ln_setup.settings.cache_dir / f"run_logs_run{i}.txt") + with open(log_path) as f: + content = f.read() + print(f"\nContents of run{i} log:") + print(content) + # Check each expected line is in the content + for expected_line in expected_contents[i]: + assert ( + expected_line in content + ), f"Expected '{expected_line}' in log {i}" + + # Check earlier messages are NOT in the content + if i > 1: + assert "Initial stdout" not in content + assert "After starting tracker" + str(i - 1) not in content + + finally: + # Cleanup + for i in range(1, 4): + log_path = Path(ln_setup.settings.cache_dir / f"run_logs_run{i}.txt") + if log_path.exists(): + log_path.unlink() + + +def test_logstream_tracker_exception_handling(): + tracker = LogStreamTracker() + original_excepthook = sys.excepthook + run = MockRun("error") + + try: + tracker.start(run) + print("Before error") + + # Create and capture exception info + exc_type = ValueError + exc_value = ValueError("Test error") + exc_traceback = None + try: + raise exc_value + except ValueError: + exc_traceback = sys.exc_info()[2] + + # Handle the exception - this will trigger cleanup + tracker.handle_exception(exc_type, exc_value, exc_traceback) + + # Verify run status + assert run.saved + assert run.report is not None + + # Verify the content was written before cleanup + content = run.report.cache().read_text() + print("Log contents:", content) + assert "Before error" in content + assert "ValueError: Test error" in content + assert "Traceback" in content + + finally: + sys.excepthook = original_excepthook diff --git a/tests/core/test_describe_df.py b/tests/core/test_describe_df.py index 578292215..e4c46cc88 100644 --- a/tests/core/test_describe_df.py +++ b/tests/core/test_describe_df.py @@ -1,4 +1,3 @@ -import anndata as ad import bionty as bt import lamindb as ln import numpy as np diff --git a/tests/core/test_feature.py b/tests/core/test_feature.py index f19ea1594..65c00e20e 100644 --- a/tests/core/test_feature.py +++ b/tests/core/test_feature.py @@ -7,7 +7,6 @@ from lamindb import _feature from lamindb._feature import convert_pandas_dtype_to_lamin_dtype from lamindb.core.exceptions import ValidationError -from lamindb.models import ArtifactULabel from pandas.api.types import is_categorical_dtype, is_string_dtype diff --git a/tests/core/test_feature_label_manager.py b/tests/core/test_feature_label_manager.py index b759bc540..5a9c3d4d8 100644 --- a/tests/core/test_feature_label_manager.py +++ b/tests/core/test_feature_label_manager.py @@ -433,7 +433,6 @@ def test_labels_add(adata): adata2 = adata.copy() adata2.uns["mutated"] = True artifact2 = ln.Artifact(adata2, description="My new artifact").save() - from lamindb.core._label_manager import _get_labels artifact2.labels.add_from(artifact) experiments = artifact2.labels.get(experiment) diff --git a/tests/core/test_save.py b/tests/core/test_save.py index b7e5a649f..28b95d663 100644 --- a/tests/core/test_save.py +++ b/tests/core/test_save.py @@ -31,8 +31,6 @@ def test_prepare_error_message(): def test_save_data_object(): - import anndata as ad - ln.core.datasets.file_mini_csv() artifact = ln.Artifact("mini.csv", description="test") artifact.save() diff --git a/tests/core/test_transform.py b/tests/core/test_transform.py index 91a3608e1..db5ab1a6a 100644 --- a/tests/core/test_transform.py +++ b/tests/core/test_transform.py @@ -2,7 +2,6 @@ import lamindb as ln import pytest -from django.db.models.deletion import ProtectedError def test_revise_transforms(): diff --git a/tests/storage/conftest.py b/tests/storage/conftest.py index 98fadd7f8..fffa5555b 100644 --- a/tests/storage/conftest.py +++ b/tests/storage/conftest.py @@ -1,5 +1,4 @@ import shutil -from pathlib import Path from subprocess import DEVNULL, run from time import perf_counter