Try DevX shell for linux+macos

.github/workflows/devx.yml

@@ -0,0 +1,267 @@
+name: Haskell CI (DevX Shell)
+
+on:
+  pull_request: # Required for workflows to be able to be approved from forks
+  merge_group:
+
+  # DO NOT DELETE.
+  # This is required for nightly builds and is invoked by nightly-trigger.yml
+  # on a schedule trigger.
+  workflow_dispatch:
+    inputs:
+      reason:
+        description: 'Reason'
+        required: false
+        default: manual
+      tests:
+        description: 'Tests'
+        required: false
+        default: some
+
+jobs:
+  build:
+    defaults:
+      run:
+        shell: devx {0}
+    env:
+      CABAL_AWS_ACCESS_KEY_ID: ${{ secrets.CABAL_AWS_ACCESS_KEY_ID }}
+      CABAL_AWS_SECRET_ACCESS_KEY: ${{ secrets.CABAL_AWS_SECRET_ACCESS_KEY }}      
+      # Modify this value to "invalidate" the cabal cache.
+      CABAL_CACHE_VERSION: "2024-04-24"
+
+    runs-on: ${{ matrix.platform == 'x86_64-linux' && 'ubuntu-latest' || 'macos-latest' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ x86_64-linux, x86_64-darwin, aarch64-darwin ]
+        compiler-nix-name: [ ghc96, ghc98 ] # , ghc810 ]
+
+    concurrency:
+      group: >
+        a+${{ github.event_name }}
+        b+${{ github.workflow_ref }}
+        c+${{ github.job }}
+        d+${{ matrix.compiler-nix-name }}
+        e+${{ matrix.platform }}
+        f+${{ (startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.run_id) || github.event.pull_request.number || github.ref }}
+      cancel-in-progress: true
+
+    steps:
+    - name: Install GHC and Cabal
+      uses: input-output-hk/actions/devx@latest
+      with:
+        platform: ${{ matrix.platform }}
+        target-platform: ${{ matrix.target-platform }}
+        compiler-nix-name: ${{ matrix.compiler-nix-name }}
+        # for now we'll set minimal to false, as minimal-iog images don't exist.
+        minimal: false
+        iog: true
+
+    - name: Concurrency group
+      run: >
+        echo
+        a+${{ github.event_name }}
+        b+${{ github.workflow_ref }}
+        c+${{ github.job }}
+        d+${{ matrix.compiler-nix-name }}
+        e+${{ matrix.platform }}
+        f+${{ (startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.run_id) || github.event.pull_request.number || github.ref }}
+
+    - uses: actions/checkout@v4
+
+    - name: add patched cabal
+      run: |
+        mkdir ~/bin
+        case "$(uname -m)-$(uname -o)" in
+          arm64-Darwin) 
+            curl -L https://ci.zw3rk.com/job/input-output-hk-haskell-nix-example/pullrequest-2/aarch64-darwin.cabal-install-static/latest/download/1 | gunzip > ~/bin/cabal
+            ;;          
+          x86_64-Darwin)
+            curl -L https://ci.zw3rk.com/job/input-output-hk-haskell-nix-example/pullrequest-2/x86_64-darwin.cabal-install-static/latest/download/1 | gunzip > ~/bin/cabal
+            ;;          
+          x86_64-GNU/Linux)
+            curl -L https://ci.zw3rk.com/job/input-output-hk-haskell-nix-example/pullrequest-2/x86_64-linux.cabal-install-static/latest/download/1 | gunzip > ~/bin/cabal
+            ;;
+        esac
+        chmod +x ~/bin/cabal
+
+    - name: Cabal update
+      run: ~/bin/cabal update
+
+    - name: Configure build
+      run: |      
+        cp .github/workflows/cabal.project.local.ci cabal.project.local
+        echo "# cabal.project.local"
+        cat cabal.project.local
+
+    # A dry run `build all` operation does *NOT* downlaod anything, it just looks at the package
+    # indices to generate an install plan.
+    - name: Build dry run
+      run: ~/bin/cabal build all --enable-tests --dry-run --minimize-conflict-set
+
+    # From the install plan we generate a dependency list.
+    - name: Record dependencies
+      id: record-deps
+      run: |
+        # The tests call out to msys2 commands. We generally do not want to mix toolchains, so
+        # we are very deliberate about only adding msys64 to the path where absolutely necessary.
+        ${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }}
+        cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[] | select(.style != "local") | .id' | sort | uniq > dependencies.txt
+
+    # From the dependency list we restore the cached dependencies.
+    # We use the hash of `dependencies.txt` as part of the cache key because that will be stable
+    # until the `index-state` values in the `cabal.project` file changes.
+    - name: Restore cached dependencies
+      uses: actions/cache/restore@v4
+      id: cache
+      with:
+        path: |
+          ${{ steps.setup-haskell.outputs.cabal-store }}
+          dist-newstyle
+        key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
+
+    # Now we install the dependencies. If the cache was found and restored in the previous step,
+    # this should be a no-op, but if the cache key was not found we need to build stuff so we can
+    # cache it for the next step.
+    - name: Install dependencies
+      run: ~/bin/cabal build all --enable-tests --only-dependencies -j --ghc-option=-j4
+
+    # Always store the cabal cache.
+    # This can fail (benign failure) if there is already a hash at that key.
+    - name: Cache Cabal store
+      uses: actions/cache/save@v4
+      with:
+        path: |
+          ${{ steps.setup-haskell.outputs.cabal-store }}
+          dist-newstyle
+        key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
+
+    # Now we build.
+    - name: Build all
+      run: ~/bin/cabal build all --enable-tests
+
+    - name: Run tests
+      env:
+        KEEP_WORKSPACE: 1
+      run: |
+        export TMPDIR="${{ runner.temp }}"
+        export TMP="${{ runner.temp }}"
+        ~/bin/cabal test cardano-testnet cardano-node cardano-node-chairman cardano-submit-api
+
+    - name: Tar failed tests workspaces
+      if: ${{ failure() }}
+      env:
+        TMP: ${{ runner.temp }}
+      run: |
+        cd $TMP
+        find . -name 'module' -type f -exec dirname {} \; | xargs -L1 basename | sort -u | xargs tar -czvf workspaces.tgz
+
+    - name: Upload workspaces on tests failure
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: failed-test-workspaces-${{ matrix.os }}-ghc${{ matrix.ghc }}-cabal${{ matrix.cabal }}.tgz
+        path: ${{ runner.temp }}/workspaces.tgz
+
+    - name: "Tar artifacts"
+      run: |
+        mkdir -p artifacts
+
+        for exe in $(cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[] | select(.style == "local" and (."component-name" | startswith("exe:"))) | ."bin-file"'); do
+          if [ -f $exe ]; then
+            echo "Including artifact $exe"
+
+            ( cd artifacts
+              tar -C "$(dirname $exe)" -czf "$(basename $exe).tar.gz" "$(basename $exe)"
+            )
+          else
+            echo "Skipping artifact $exe"
+          fi
+        done
+
+    - name: Delete socket files in chairman tests in preparation for uploading artifacts
+      if: ${{ always() }}
+      run: |
+        if [ -d "${{ runner.temp }}/chairman" ]; then
+          find "${{ runner.temp }}/chairman" -type s -exec rm -f {} \;
+        fi
+
+    - name: Save Artifact
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}
+      continue-on-error: true
+      with:
+        name: chairman-test-artifacts-${{ matrix.os }}-${{ matrix.ghc }}
+        path: ${{ runner.temp }}/chairman/
+
+    # Uncomment the following back in for debugging. Remember to launch a `pwsh` from
+    # the tmux session to debug `pwsh` issues. And be reminded that the `/msys2` and
+    # `/msys2/mingw64` paths are not in PATH by default for the workflow, but tmate
+    # will put them in.
+    # You may also want to run
+    #
+    #   $env:PATH=("C:\Program Files\PowerShell\7;{0}" -f $env:ORIGINAL_PATH)
+    #
+    # to restore the original path.  Do note that some test might need msys2
+    # and will silently fail if msys2 is not in path. See the "Run tests" step.
+    #
+    # - name: Setup tmate session
+    #  if: ${{ failure() }}
+    #  uses: mxschmitt/action-tmate@v3
+    #  with:
+    #     limit-access-to-actor: true
+
+  build-complete:
+    needs: [build]
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check if any previous job failed
+      run: |
+        if [[ "${{ needs.build.result }}" == "failure" ]]; then
+          # this ignores skipped dependencies
+          echo 'Required jobs failed to build.'
+          exit 1
+        else
+          echo 'Build complete'
+        fi
+
+  release:
+    needs: [build]
+    if: ${{ startsWith(github.ref, 'refs/tags') }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Create Release Tag
+      id: create_release_tag
+      run: |
+        echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Release ${{ github.ref }}
+        draft: true
+        prerelease: false
+
+    - name: Download Artifact
+      uses: actions/download-artifact@v1
+      with:
+        name: artifacts-ubuntu-latest
+
+    - name: Upload Release Asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./artifacts-ubuntu-latest/cardano-submit-api.tar.gz
+        asset_name: cardano-submit-api_${{ steps.create_release_tag.outputs.TAG }}-linux.tar.gz
+        asset_content_type: application/gzip

.github/workflows/haskell.yml → .github/workflows/haskell.yaml

@@ -26,15 +26,10 @@ jobs:
       fail-fast: false
       matrix:
         # If you edit these versions, make sure the version in the lonely macos-latest job below is updated accordingly
+        # Linux / macOS are in devx.yaml, please ensure the GHC versions line up.
         ghc: ["9.6.4", "9.8.1"]
         cabal: ["3.10.2.1"]
-        os: [windows-latest, ubuntu-latest]
-        include:
-          # Using include, to make sure there will only be one macOS job, even if the matrix gets expanded later on.
-          # We want a single job, because macOS runners are scarce.
-          - os: macos-latest
-            cabal: "3.10.2.1"
-            ghc: "9.6.4"
+        os: [windows-latest]
 
     env:
       # Modify this value to "invalidate" the cabal cache.

.github/workflows/nightly-trigger.yml

@@ -23,10 +23,18 @@ jobs:
         git checkout -b nightly
         git push origin nightly --force
 
-    - name: Invoke workflow
+    - name: Invoke workflow (Windows)
       uses: input-output-hk/workflow-dispatch@v1
       with:
         workflow: .github/workflows/haskell.yml
         ref: nightly
         token: ${{ secrets.MACHINE_TOKEN }}
         inputs: '{ "reason": "nightly", "tests": "all" }'
+
+    - name: Invoke workflow (macOS & Linux)
+      uses: input-output-hk/workflow-dispatch@v1
+      with:
+        workflow: .github/workflows/devx.yml
+        ref: nightly
+        token: ${{ secrets.MACHINE_TOKEN }}
+        inputs: '{ "reason": "nightly", "tests": "all" }'

cabalHooks/postBuildHook

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+#./postBuildHook.local "$@"
+$(dirname "${BASH_SOURCE[0]}")/postBuildHook.s3 "$@"

cabalHooks/postBuildHook.local

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CACHE=$HOME/.cabal-build-cache
+
+mkdir -p $CACHE
+
+echo "Uploading $1 to $CACHE"
+tar -czf $CACHE/$1.tar.gz --exclude=$3/setup-config $3
+exit 0
+

cabalHooks/postBuildHook.s3

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1091,SC2046
+. "$(dirname "${BASH_SOURCE[0]}")/s3-credentials.bash"
+
+CACHE_KEY="$1.tar.gz"
+
+if ! test -f "$CACHE_KEY" ; then
+  echo "S3 upload    $1"
+  tar -czf "$CACHE_KEY" --exclude="$3/setup-config" "$3"
+  aws s3 cp "$CACHE_KEY" s3://"$CACHE_BUCKET"/devx-shell/"$CACHE_KEY" --endpoint-url "$AWS_ENDPOINT"
+  rm "$CACHE_KEY" # Optional: Cleanup local file
+fi

cabalHooks/preBuildHook

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+#./preBuildHook.local "$@"
+$(dirname "${BASH_SOURCE[0]}")/preBuildHook.s3 "$@"

cabalHooks/preBuildHook.local

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+CACHE=$HOME/.cabal-build-cache
+
+mkdir -p $CACHE
+
+if [ -f $CACHE/$1.tar.gz ]; then
+    echo "Cache hit"
+    tar -xzf $CACHE/$1.tar.gz
+    exit 0
+else
+    echo "Cache miss"
+    exit 1
+fi

cabalHooks/preBuildHook.s3

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC1091,SC2046
+. $(dirname "${BASH_SOURCE[0]}")/s3-credentials.bash
+
+CACHE_KEY="$1.tar.gz"
+
+# Check if artifact exists in S3 (with endpoint and credentials)
+aws s3 ls s3://"$CACHE_BUCKET"/devx-shell/"$CACHE_KEY" --endpoint-url "$AWS_ENDPOINT" > /dev/null 2>&1
+
+# shellcheck disable=SC2181
+if [ $? -eq 0 ]; then
+  echo "S3 hit       $1"
+  aws s3 cp s3://"$CACHE_BUCKET"/devx-shell/"$CACHE_KEY" - --endpoint-url "$AWS_ENDPOINT" | tar -xz
+  # leave an empty $CACHE_KEY file for the postBuildHook to see and not re-upload.
+  touch "$CACHE_KEY"
+else
+  echo "S3 miss      $1"
+  exit 1
+fi

cabalHooks/s3-credentials.bash

@@ -0,0 +1,5 @@
+CACHE_BUCKET="cabal-cache"  # Replace with your bucket name
+export AWS_ACCESS_KEY_ID=$CABAL_AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=$CABAL_AWS_SECRET_ACCESS_KEY
+export AWS_DEFAULT_REGION=auto
+AWS_ENDPOINT="https://c1fe99e5a421ac98af478e51dda5db4f.r2.cloudflarestorage.com"