From dd017f9698d033fc2d8ded6585af75cd56bcca20 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 28 Jun 2023 19:30:36 +0200 Subject: [PATCH 1/7] [CI] generate conda envs also for all possible OS --- .github/workflows/caches_cron_job.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 95759b956..8e307717e 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -15,6 +15,8 @@ jobs: runs-on: "ubuntu-latest" strategy: fail-fast: true + matrix: + os: ['ubuntu-latest', 'windows-latest', 'macos-latest'] defaults: # by default run in bash mode (required for conda usage) run: From b1aade8dbf751c2232e36ac07b9522e853c1f437 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 29 Jun 2023 10:31:30 +0200 Subject: [PATCH 2/7] [CI] fix conda envs generation also for all possible OS --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 8e307717e..4ecda3460 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -12,7 +12,7 @@ jobs: create-conda-env-cache-if-missing: name: Caching conda env - runs-on: "ubuntu-latest" + runs-on: ${{ matrix.os }} strategy: fail-fast: true matrix: From acff10075880b8c3a295632ff7eb54e61239b120 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 29 Jun 2023 14:46:02 +0200 Subject: [PATCH 3/7] [CI] add first version of os independant git-annex installation --- .github/workflows/caches_cron_job.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 4ecda3460..afa3602e7 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -40,7 +40,7 @@ jobs: # * when package dependencies change id: cache-conda-env with: - path: /usr/share/miniconda/envs/neo-test-env + path: ${{ CONDA }}/envs/neo-test-env key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }} - name: Cache found? @@ -54,6 +54,17 @@ jobs: environment-file: environment_testing.yml python-version: 3.9 + - name: Installing datalad and git-annex + if: steps.cache-conda-env.outputs.cache-hit != 'true' + run: | + git config --global user.email "neo_ci@fake_mail.com" + git config --global user.name "neo CI" + python -m pip install -U pip # Official recommended way + pip install datalad-installer + datalad-installer --sudo ok git-annex --method datalad/packages --bin-dir ${{CONDA}}/envs/neo-test-env/ + pip install datalad + git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency + - name: Create the conda environment to be cached if: steps.cache-conda-env.outputs.cache-hit != 'true' # create conda env, configure git and install pip, neo and test dependencies from master From 8bbecef91b914dc21914c1775de294166d95a4b9 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 29 Jun 2023 15:26:35 +0200 Subject: [PATCH 4/7] [CI] make conde env location independant of OS --- .github/workflows/caches_cron_job.yml | 6 +++--- .github/workflows/io-test.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index afa3602e7..6fb65afe5 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -40,7 +40,7 @@ jobs: # * when package dependencies change id: cache-conda-env with: - path: ${{ CONDA }}/envs/neo-test-env + path: envs/neo-test-env key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }} - name: Cache found? @@ -50,7 +50,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2.2.0 if: steps.cache-conda-env.outputs.cache-hit != 'true' with: - activate-environment: neo-test-env + activate-environment: ~/envs/neo-test-env environment-file: environment_testing.yml python-version: 3.9 @@ -61,7 +61,7 @@ jobs: git config --global user.name "neo CI" python -m pip install -U pip # Official recommended way pip install datalad-installer - datalad-installer --sudo ok git-annex --method datalad/packages --bin-dir ${{CONDA}}/envs/neo-test-env/ + datalad-installer --sudo ok git-annex --method datalad/packages --bin-dir ~/envs/neo-test-env/ pip install datalad git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index 51b29a98f..4957ae716 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -48,7 +48,7 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: neo-test-env + activate-environment: ~/envs/neo-test-env python-version: ${{ matrix.python-version }} - name: Get current dependencies hash @@ -62,7 +62,7 @@ jobs: # * when package dependencies change id: cache-conda-env with: - path: /usr/share/miniconda/envs/neo-test-env + path: ~/envs/neo-test-env key: ${{ runner.os }}-conda-env-${{ steps.dependencies.outputs.hash }}-${{ steps.date.outputs.date }} # restore-keys match any key that starts with the restore-key restore-keys: | @@ -74,7 +74,7 @@ jobs: # restore-key hits should result in `cache-hit` == 'false' if: steps.cache-conda-env.outputs.cache-hit != 'true' run: | - conda env update --name neo-test-env --file environment_testing.yml --prune + conda env update --prefix ~/envs --name neo-test-env --file environment_testing.yml --prune - name: Configure git run: | From 9b7297b591bb5c84e4b159b594b1f0e9901443c2 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 29 Jun 2023 15:27:02 +0200 Subject: [PATCH 5/7] [CI] install git-annex version 8 for compatibility with GIN --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index 6fb65afe5..cb589554f 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -102,7 +102,7 @@ jobs: git config --global user.name "neo CI" python -m pip install -U pip # Official recommended way pip install datalad-installer - datalad-installer --sudo ok git-annex --method datalad/packages + datalad-installer --sudo ok git-annex=8 --method datalad/packages pip install datalad git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency From edf9e0088f052ddc62b06ba09fe2f5558443d788 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 29 Jun 2023 15:48:53 +0200 Subject: [PATCH 6/7] [CI] fix datalad installer syntax --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index cb589554f..d820a1ee9 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -61,7 +61,7 @@ jobs: git config --global user.name "neo CI" python -m pip install -U pip # Official recommended way pip install datalad-installer - datalad-installer --sudo ok git-annex --method datalad/packages --bin-dir ~/envs/neo-test-env/ + datalad-installer --sudo ok --bin-dir ~/envs/neo-test-env/ git-annex --method datalad/packages pip install datalad git config --global filter.annex.process "git-annex filter-process" # recommended for efficiency From 8b6ea5f70b21379a04098d840011d92032c49c08 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 27 Jul 2023 11:09:27 +0200 Subject: [PATCH 7/7] Rename chron job --- .github/workflows/caches_cron_job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/caches_cron_job.yml b/.github/workflows/caches_cron_job.yml index d820a1ee9..dca221987 100644 --- a/.github/workflows/caches_cron_job.yml +++ b/.github/workflows/caches_cron_job.yml @@ -76,7 +76,7 @@ jobs: pip install --upgrade -e .[test] create-data-cache-if-missing: - name: Caching data env + name: Caching data runs-on: "ubuntu-latest" steps: