From a40fa268d7d40387016b067fe25e8152831dd294 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 3 Oct 2024 16:49:15 +0200 Subject: [PATCH 01/10] Use D: for Windows installations on Azure --- conda_smithy/templates/azure-pipelines-win.yml.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_smithy/templates/azure-pipelines-win.yml.tmpl b/conda_smithy/templates/azure-pipelines-win.yml.tmpl index 1e7c082ee..5e4af8467 100644 --- a/conda_smithy/templates/azure-pipelines-win.yml.tmpl +++ b/conda_smithy/templates/azure-pipelines-win.yml.tmpl @@ -27,10 +27,10 @@ jobs: urllib.request.urlretrieve(url, path) - script: | - start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\Miniforge + start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=D:\Miniforge displayName: Install Miniforge - - powershell: Write-Host "##vso[task.prependpath]C:\Miniforge\Scripts" + - powershell: Write-Host "##vso[task.prependpath]D:\Miniforge\Scripts" displayName: Add conda to PATH - script: | From aea32e88f396aa7622ff82b685661e7df21eaeb5 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 3 Oct 2024 17:08:56 +0200 Subject: [PATCH 02/10] add news --- news/2076-install-to-D.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 news/2076-install-to-D.rst diff --git a/news/2076-install-to-D.rst b/news/2076-install-to-D.rst new file mode 100644 index 000000000..21512e92b --- /dev/null +++ b/news/2076-install-to-D.rst @@ -0,0 +1,23 @@ +**Added:** + +* + +**Changed:** + +* Use the faster ``D:\`` drive for Miniforge installations on Windows. (#2076) + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* From b10d06be2057898b614592d8130ecfac6ab4fb9b Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 3 Oct 2024 18:07:47 +0200 Subject: [PATCH 03/10] parameterize MINIFORGE_HOME in windows --- conda_smithy/data/conda-forge.json | 18 +++--------------- conda_smithy/data/conda-forge.yml | 1 + conda_smithy/schema.py | 1 + .../templates/azure-pipelines-win.yml.tmpl | 4 ++-- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/conda_smithy/data/conda-forge.json b/conda_smithy/data/conda-forge.json index c79edbc8d..621bc1e55 100644 --- a/conda_smithy/data/conda-forge.json +++ b/conda_smithy/data/conda-forge.json @@ -110,27 +110,15 @@ "title": "Upload Packages" }, "settings_linux": { - "allOf": [ - { - "$ref": "#/$defs/AzureRunnerSettings" - } - ], + "$ref": "#/$defs/AzureRunnerSettings", "description": "Linux-specific settings for runners" }, "settings_osx": { - "allOf": [ - { - "$ref": "#/$defs/AzureRunnerSettings" - } - ], + "$ref": "#/$defs/AzureRunnerSettings", "description": "OSX-specific settings for runners" }, "settings_win": { - "allOf": [ - { - "$ref": "#/$defs/AzureRunnerSettings" - } - ], + "$ref": "#/$defs/AzureRunnerSettings", "description": "Windows-specific settings for runners" }, "user_or_org": { diff --git a/conda_smithy/data/conda-forge.yml b/conda_smithy/data/conda-forge.yml index e00e084f0..e9a21d110 100644 --- a/conda_smithy/data/conda-forge.yml +++ b/conda_smithy/data/conda-forge.yml @@ -26,6 +26,7 @@ azure: timeoutInMinutes: 360 variables: CONDA_BLD_PATH: D:\\bld\\ + MINIFORGE_HOME: D:\\Miniforge\\ UPLOAD_TEMP: D:\\tmp store_build_artifacts: false timeout_minutes: null diff --git a/conda_smithy/schema.py b/conda_smithy/schema.py index 2bd56947b..971c6c9cb 100644 --- a/conda_smithy/schema.py +++ b/conda_smithy/schema.py @@ -207,6 +207,7 @@ class AzureConfig(BaseModel): default_factory=lambda: AzureRunnerSettings( pool={"vmImage": "windows-2022"}, variables={ + "MINIFORGE_HOME": "D:\\\\Miniforge\\\\", "CONDA_BLD_PATH": "D:\\\\bld\\\\", "UPLOAD_TEMP": "D:\\\\tmp", }, diff --git a/conda_smithy/templates/azure-pipelines-win.yml.tmpl b/conda_smithy/templates/azure-pipelines-win.yml.tmpl index 5e4af8467..7f1525e59 100644 --- a/conda_smithy/templates/azure-pipelines-win.yml.tmpl +++ b/conda_smithy/templates/azure-pipelines-win.yml.tmpl @@ -27,10 +27,10 @@ jobs: urllib.request.urlretrieve(url, path) - script: | - start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=D:\Miniforge + start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=$(MINIFORGE_HOME) displayName: Install Miniforge - - powershell: Write-Host "##vso[task.prependpath]D:\Miniforge\Scripts" + - powershell: Write-Host "##vso[task.prependpath]$(MINIFORGE_HOME)\Scripts" displayName: Add conda to PATH - script: | From 96d432edf94169e059a8470a44e6ca5ec1830641 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 3 Oct 2024 18:14:53 +0200 Subject: [PATCH 04/10] fix path --- conda_smithy/data/conda-forge.yml | 2 +- conda_smithy/schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_smithy/data/conda-forge.yml b/conda_smithy/data/conda-forge.yml index e9a21d110..a9c0e1d4c 100644 --- a/conda_smithy/data/conda-forge.yml +++ b/conda_smithy/data/conda-forge.yml @@ -26,7 +26,7 @@ azure: timeoutInMinutes: 360 variables: CONDA_BLD_PATH: D:\\bld\\ - MINIFORGE_HOME: D:\\Miniforge\\ + MINIFORGE_HOME: D:\Miniforge UPLOAD_TEMP: D:\\tmp store_build_artifacts: false timeout_minutes: null diff --git a/conda_smithy/schema.py b/conda_smithy/schema.py index 971c6c9cb..618b706c3 100644 --- a/conda_smithy/schema.py +++ b/conda_smithy/schema.py @@ -207,7 +207,7 @@ class AzureConfig(BaseModel): default_factory=lambda: AzureRunnerSettings( pool={"vmImage": "windows-2022"}, variables={ - "MINIFORGE_HOME": "D:\\\\Miniforge\\\\", + "MINIFORGE_HOME": "D:\\Miniforge", "CONDA_BLD_PATH": "D:\\\\bld\\\\", "UPLOAD_TEMP": "D:\\\\tmp", }, From 4167d3f36f5db380a09a5d6af8994cee00815d91 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 3 Oct 2024 19:26:02 +0200 Subject: [PATCH 05/10] document variables --- conda_smithy/data/conda-forge.json | 2 +- conda_smithy/schema.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/conda_smithy/data/conda-forge.json b/conda_smithy/data/conda-forge.json index 621bc1e55..5de613d54 100644 --- a/conda_smithy/data/conda-forge.json +++ b/conda_smithy/data/conda-forge.json @@ -119,7 +119,7 @@ }, "settings_win": { "$ref": "#/$defs/AzureRunnerSettings", - "description": "Windows-specific settings for runners" + "description": "Windows-specific settings for runners. Some important variables you can set are:\n\n- `CONDA_BLD_PATH`: Location of the conda-build workspace. Defaults to `D:\\bld`\n- `MINIFORGE_HOME`: Location of the base environment installation. Defaults to\n `D:\\Miniforge`.\n- `SET_PAGEFILE`: `\"True\"` to increase the pagefile size via conda-forge-ci-setup.\n\nIf you are running out of space in `D:`, consider changing to `C:`.\nIt's a slower drive but has more space available. We recommend you keep\nboth `CONDA_BLD_PATH` and `MINIFORGE_HOME` in the same drive for performance." }, "user_or_org": { "anyOf": [ diff --git a/conda_smithy/schema.py b/conda_smithy/schema.py index 618b706c3..41b7ef54c 100644 --- a/conda_smithy/schema.py +++ b/conda_smithy/schema.py @@ -212,7 +212,20 @@ class AzureConfig(BaseModel): "UPLOAD_TEMP": "D:\\\\tmp", }, ), - description="Windows-specific settings for runners", + description=cleandoc( + """ + Windows-specific settings for runners. Some important variables you can set are: + + - `CONDA_BLD_PATH`: Location of the conda-build workspace. Defaults to `D:\\bld` + - `MINIFORGE_HOME`: Location of the base environment installation. Defaults to + `D:\\Miniforge`. + - `SET_PAGEFILE`: `"True"` to increase the pagefile size via conda-forge-ci-setup. + + If you are running out of space in `D:`, consider changing to `C:`. + It's a slower drive but has more space available. We recommend you keep + both `CONDA_BLD_PATH` and `MINIFORGE_HOME` in the same drive for performance. + """ + ) ) user_or_org: Optional[Union[str, Nullable]] = Field( From a957c3137532f37a024ae6dd33144f398ee0256c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 3 Oct 2024 19:28:58 +0200 Subject: [PATCH 06/10] pre-commit --- conda_smithy/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/schema.py b/conda_smithy/schema.py index 41b7ef54c..7c480f294 100644 --- a/conda_smithy/schema.py +++ b/conda_smithy/schema.py @@ -225,7 +225,7 @@ class AzureConfig(BaseModel): It's a slower drive but has more space available. We recommend you keep both `CONDA_BLD_PATH` and `MINIFORGE_HOME` in the same drive for performance. """ - ) + ), ) user_or_org: Optional[Union[str, Nullable]] = Field( From 72550c961712925d5923fa38ffb359ce2a458fdb Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 14 Oct 2024 21:35:33 +0200 Subject: [PATCH 07/10] rerender schema --- conda_smithy/data/conda-forge.json | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/conda_smithy/data/conda-forge.json b/conda_smithy/data/conda-forge.json index 5de613d54..b23fc45e8 100644 --- a/conda_smithy/data/conda-forge.json +++ b/conda_smithy/data/conda-forge.json @@ -110,15 +110,27 @@ "title": "Upload Packages" }, "settings_linux": { - "$ref": "#/$defs/AzureRunnerSettings", + "allOf": [ + { + "$ref": "#/$defs/AzureRunnerSettings" + } + ], "description": "Linux-specific settings for runners" }, "settings_osx": { - "$ref": "#/$defs/AzureRunnerSettings", + "allOf": [ + { + "$ref": "#/$defs/AzureRunnerSettings" + } + ], "description": "OSX-specific settings for runners" }, "settings_win": { - "$ref": "#/$defs/AzureRunnerSettings", + "allOf": [ + { + "$ref": "#/$defs/AzureRunnerSettings" + } + ], "description": "Windows-specific settings for runners. Some important variables you can set are:\n\n- `CONDA_BLD_PATH`: Location of the conda-build workspace. Defaults to `D:\\bld`\n- `MINIFORGE_HOME`: Location of the base environment installation. Defaults to\n `D:\\Miniforge`.\n- `SET_PAGEFILE`: `\"True\"` to increase the pagefile size via conda-forge-ci-setup.\n\nIf you are running out of space in `D:`, consider changing to `C:`.\nIt's a slower drive but has more space available. We recommend you keep\nboth `CONDA_BLD_PATH` and `MINIFORGE_HOME` in the same drive for performance." }, "user_or_org": { @@ -784,9 +796,6 @@ "Nullable": { "const": null, "description": "Created to avoid issue with schema validation of null values in lists or dicts.", - "enum": [ - null - ], "title": "Nullable" }, "Platforms": { From 8b675d690453061204edd5cd4e5987315ef0b109 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 15 Oct 2024 23:27:30 +0200 Subject: [PATCH 08/10] rerender schema --- conda_smithy/data/conda-forge.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_smithy/data/conda-forge.json b/conda_smithy/data/conda-forge.json index 4c0b3e769..ff0a9035e 100644 --- a/conda_smithy/data/conda-forge.json +++ b/conda_smithy/data/conda-forge.json @@ -131,7 +131,7 @@ "$ref": "#/$defs/AzureRunnerSettings" } ], - "description": "Windows-specific settings for runners" + "description": "Windows-specific settings for runners. Some important variables you can set are:\n\n- `CONDA_BLD_PATH`: Location of the conda-build workspace. Defaults to `D:\\bld`\n- `MINIFORGE_HOME`: Location of the base environment installation. Defaults to\n `D:\\Miniforge`.\n- `SET_PAGEFILE`: `\"True\"` to increase the pagefile size via conda-forge-ci-setup.\n\nIf you are running out of space in `D:`, consider changing to `C:`.\nIt's a slower drive but has more space available. We recommend you keep\nboth `CONDA_BLD_PATH` and `MINIFORGE_HOME` in the same drive for performance." }, "user_or_org": { "anyOf": [ From 662543ee7690ddb12bd5dff3d62436b42e958d0f Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 15 Oct 2024 23:45:43 +0200 Subject: [PATCH 09/10] Add a top-level example --- conda_smithy/data/conda-forge.json | 2 +- conda_smithy/schema.py | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/conda_smithy/data/conda-forge.json b/conda_smithy/data/conda-forge.json index ff0a9035e..b6b423218 100644 --- a/conda_smithy/data/conda-forge.json +++ b/conda_smithy/data/conda-forge.json @@ -2265,7 +2265,7 @@ "type": "null" } ], - "description": "Azure Pipelines CI settings. This is usually read-only and should not\nnormally be manually modified. Tools like conda-smithy may modify this, as needed.\nFor example:\n\n```yaml\nazure:\n # flag for forcing the building all supported providers\n force: False\n # toggle for storing the conda build_artifacts directory (including the\n # built packages) as an Azure pipeline artifact that can be downloaded\n store_build_artifacts: False\n # toggle for freeing up some extra space on the default Azure Pipelines\n # linux image before running the Docker container for building\n free_disk_space: False\n # limit the amount of CI jobs running concurrently at a given time\n # each OS will get its proportional share of the configured value\n max_parallel: 25\n```\n\nBelow is an example configuration for setting up a self-hosted Azure agent for Linux:\n\n```yaml\nazure:\n settings_linux:\n pool:\n name: your_local_pool_name\n demands:\n - some_key -equals some_value\n workspace:\n clean: all\n strategy:\n maxParallel: 1\n```\n\nBelow is an example configuration for adding a swapfile on an Azure agent for Linux:\n\n```yaml\nazure:\n settings_linux:\n swapfile_size: 10GiB\n```" + "description": "Azure Pipelines CI settings. This is usually read-only and should not\nnormally be manually modified. Tools like conda-smithy may modify this, as needed.\nFor example:\n\n```yaml\nazure:\n # flag for forcing the building all supported providers\n force: False\n # toggle for storing the conda build_artifacts directory (including the\n # built packages) as an Azure pipeline artifact that can be downloaded\n store_build_artifacts: False\n # toggle for freeing up some extra space on the default Azure Pipelines\n # linux image before running the Docker container for building\n free_disk_space: False\n # limit the amount of CI jobs running concurrently at a given time\n # each OS will get its proportional share of the configured value\n max_parallel: 25\n```\n\nBelow is an example configuration for setting up a self-hosted Azure agent for Linux:\n\n```yaml\nazure:\n settings_linux:\n pool:\n name: your_local_pool_name\n demands:\n - some_key -equals some_value\n workspace:\n clean: all\n strategy:\n maxParallel: 1\n```\n\nBelow is an example configuration for adding a swapfile on an Azure agent for Linux:\n\n```yaml\nazure:\n settings_linux:\n swapfile_size: 10GiB\n```\n\nIf you need more space on Windows, you can use `C:` at the cost of IO performance:\n\n```yaml\nazure:\n settings_win:\n variables:\n CONDA_BLD_PATH: \"C:\\bld\"\n MINIFORGE_HOME: \"C:\\Miniforge\"\n```" }, "drone": { "anyOf": [ diff --git a/conda_smithy/schema.py b/conda_smithy/schema.py index 7fddfbc80..0f93e9ed8 100644 --- a/conda_smithy/schema.py +++ b/conda_smithy/schema.py @@ -1251,14 +1251,14 @@ class ConfigModel(BaseModel): ```yaml azure: settings_linux: - pool: - name: your_local_pool_name - demands: - - some_key -equals some_value - workspace: - clean: all - strategy: - maxParallel: 1 + pool: + name: your_local_pool_name + demands: + - some_key -equals some_value + workspace: + clean: all + strategy: + maxParallel: 1 ``` Below is an example configuration for adding a swapfile on an Azure agent for Linux: @@ -1268,6 +1268,16 @@ class ConfigModel(BaseModel): settings_linux: swapfile_size: 10GiB ``` + + If you need more space on Windows, you can use `C:` at the cost of IO performance: + + ```yaml + azure: + settings_win: + variables: + CONDA_BLD_PATH: "C:\\bld" + MINIFORGE_HOME: "C:\\Miniforge" + ``` """ ), ) From 180ec15a6aab4cbdcc0e2daec89afc70b732c26d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 15 Oct 2024 23:46:26 +0200 Subject: [PATCH 10/10] amend news --- news/2076-install-to-D.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/2076-install-to-D.rst b/news/2076-install-to-D.rst index 21512e92b..42a29bf27 100644 --- a/news/2076-install-to-D.rst +++ b/news/2076-install-to-D.rst @@ -4,7 +4,7 @@ **Changed:** -* Use the faster ``D:\`` drive for Miniforge installations on Windows. (#2076) +* Use the faster ``D:\`` drive for Miniforge installations on Windows and expose this path as the ``MINIFORGE_HOME`` variable. (#2076) **Deprecated:**