.github/workflows/CompatHelper.yml

@@ -0,0 +1,35 @@
+name: CompatHelper
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get Julia compatibility
+        id: julia_compat
+        # NOTE: this requires a Julia compat lower-bound with minor version!
+        run : |
+          version=$(grep '^julia = ' Project.toml | grep -o '".*"' | cut -d '"' -f2)
+          echo "::set-output name=version::$version"
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ steps.julia_compat.outputs.version }}
+      - name: Install CompatHelper
+        run: |
+          import Pkg
+          name = "CompatHelper"
+          version = "2"
+          Pkg.add(; name, version)
+        shell: julia --color=yes {0}
+      - name: Run CompatHelper
+        run: |
+          using CompatHelper
+          CompatHelper.main()
+        shell: julia --color=yes {0}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/CustomCommands.yml

@@ -0,0 +1,43 @@
+name: Custom Commands
+
+on:
+  pull_request:
+
+jobs:
+  custom_actions:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for commands
+        id: read_commands
+        run: |
+          ci_commands=$(curl -sH "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X GET "${{ github.event.pull_request.comments_url }}" | jq -r ".[] | select(.body | startswith(\"/run-actions\")) | .body")
+          if [ ! -z "$ci_commands" ]; then
+            echo "Read commands $ci_commands"
+            echo "ci_commands=$ci_commands" >> $GITHUB_OUTPUT
+            break
+          fi
+
+      - uses: actions/checkout@v3
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1.9'
+          arch: 'x64'
+
+      - name: Run provided commands
+        run: |
+          echo "Got custom commands:"
+          ci_commands="${{ steps.read_commands.outputs.ci_commands }}"
+          echo $ci_commands
+          if echo "$ci_commands" | grep -q benchmark; then
+            echo "Running benchmarks..."
+            julia -e 'using Pkg; Pkg.add("BenchmarkTools"); Pkg.develop(;path=pwd())'
+            JULIA_PROJECT=$PWD julia --project benchmarks/benchmark.jl
+          fi
+          if echo "$ci_commands" | grep -q premerge; then
+            echo "Running pre-merge checks..."
+            julia -e 'using Pkg; Pkg.add("Aqua"); Pkg.develop(;path=pwd())'
+            JULIA_PROJECT=$PWD julia --project -e 'using Aqua, Dagger; Aqua.test_all(Dagger)'
+          fi
+        env:
+          BENCHMARK: "nmf:raw,dagger"
+          BENCHMARK_SCALE: "5:5:10"

Manifest.toml

@@ -36,9 +36,9 @@ version = "0.12.10"
 
 [[deps.Compat]]
 deps = ["Dates", "LinearAlgebra", "UUIDs"]
-git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d"
+git-tree-sha1 = "5ce999a19f4ca23ea484e92a1774a61b8ca4cf8e"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.7.0"
+version = "4.8.0"
 
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
@@ -211,9 +211,9 @@ version = "1.6.0"
 
 [[deps.StatsBase]]
 deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
-git-tree-sha1 = "d1bf48bfcc554a3761a133fe3a9bb01488e06916"
+git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4"
 uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.33.21"
+version = "0.34.0"
 
 [[deps.Test]]
 deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]

Project.toml

@@ -1,6 +1,6 @@
 name = "Dagger"
 uuid = "d58978e5-989f-55fb-8d15-ea34adc7bf54"
-version = "0.18.0"
+version = "0.18.1"
 
 [deps]
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
@@ -28,7 +28,7 @@ DataStructures = "0.18"
 MacroTools = "0.5"
 MemPool = "0.4.3"
 Requires = "1"
-StatsBase = "0.28, 0.29, 0.30, 0.31, 0.32, 0.33"
+StatsBase = "0.28, 0.29, 0.30, 0.31, 0.32, 0.33, 0.34"
 TimespanLogging = "0.1"
 julia = "1.7"
 

benchmarks/suites/nmf.jl

@@ -55,40 +55,43 @@ function nmf_suite(ctx; method, accels)
             nw = length(workers())
             nsuite = BenchmarkGroup()
             while nw > 0
-                opts = if accel == "cuda"
+                scope = if accel == "cuda"
+                    error("Not implemented")
                     Dagger.Sch.SchedulerOptions(;proctypes=[
                         DaggerGPU.CuArrayDeviceProc
                     ])
                 elseif accel == "amdgpu"
+                    error("Not implemented")
                     Dagger.Sch.SchedulerOptions(;proctypes=[
                         DaggerGPU.ROCArrayProc
                     ])
                 elseif accel == "cpu"
-                    Dagger.Sch.SchedulerOptions()
+                    scope = Dagger.scope(;workers=workers()[1:nw])
                 else
                     error("Unknown accelerator $accel")
                 end
                 #bsz = ncol ÷ length(workers())
                 bsz = ncol ÷ 64
                 nsuite["Workers: $nw"] = @benchmarkable begin
-                    _ctx = Context($ctx, workers()[1:$nw])
-                    compute(_ctx, nnmf($X[], $W[], $H[]); options=$opts)
+                    Dagger.with_options(;scope=$scope) do
+                        fetch(nnmf($X[], $W[], $H[]))
+                    end
                 end setup=begin
                     _nw, _scale = $nw, $scale
                     @info "Starting $_nw worker Dagger NNMF (scale by $_scale)"
                     if $accel == "cuda"
                         # FIXME: Allocate with CUDA.rand if possible
-                        $X[] = Dagger.mapchunks(CUDA.cu, compute(rand(Blocks($bsz, $bsz), Float32, $nrow, $ncol); options=$opts))
-                        $W[] = Dagger.mapchunks(CUDA.cu, compute(rand(Blocks($bsz, $bsz), Float32, $nrow, $nfeatures); options=$opts))
-                        $H[] = Dagger.mapchunks(CUDA.cu, compute(rand(Blocks($bsz, $bsz), Float32, $nfeatures, $ncol); options=$opts))
+                        $X[] = Dagger.mapchunks(CuArray, rand(Blocks($bsz, $bsz), Float32, $nrow, $ncol))
+                        $W[] = Dagger.mapchunks(CuArray, rand(Blocks($bsz, $bsz), Float32, $nrow, $nfeatures))
+                        $H[] = Dagger.mapchunks(CuArray, rand(Blocks($bsz, $bsz), Float32, $nfeatures, $ncol))
                     elseif $accel == "amdgpu"
-                        $X[] = Dagger.mapchunks(ROCArray, compute(rand(Blocks($bsz, $bsz), Float32, $nrow, $ncol); options=$opts))
-                        $W[] = Dagger.mapchunks(ROCArray, compute(rand(Blocks($bsz, $bsz), Float32, $nrow, $nfeatures); options=$opts))
-                        $H[] = Dagger.mapchunks(ROCArray, compute(rand(Blocks($bsz, $bsz), Float32, $nfeatures, $ncol); options=$opts))
+                        $X[] = Dagger.mapchunks(ROCArray, rand(Blocks($bsz, $bsz), Float32, $nrow, $ncol))
+                        $W[] = Dagger.mapchunks(ROCArray, rand(Blocks($bsz, $bsz), Float32, $nrow, $nfeatures))
+                        $H[] = Dagger.mapchunks(ROCArray, rand(Blocks($bsz, $bsz), Float32, $nfeatures, $ncol))
                     elseif $accel == "cpu"
-                        $X[] = compute(rand(Blocks($bsz, $bsz), Float32, $nrow, $ncol); options=$opts)
-                        $W[] = compute(rand(Blocks($bsz, $bsz), Float32, $nrow, $nfeatures); options=$opts)
-                        $H[] = compute(rand(Blocks($bsz, $bsz), Float32, $nfeatures, $ncol); options=$opts)
+                        $X[] = rand(Blocks($bsz, $bsz), Float32, $nrow, $ncol)
+                        $W[] = rand(Blocks($bsz, $bsz), Float32, $nrow, $nfeatures)
+                        $H[] = rand(Blocks($bsz, $bsz), Float32, $nfeatures, $ncol)
                     end
                 end teardown=begin
                     if render != "" && !live

src/utils/caching.jl

@@ -66,10 +66,10 @@ function enable_disk_caching!(
     processes::Vector{Int}=procs(),
 )
     results = [
-        remotecall(id) do
-            !isdefined(Main, :Dagger) && Main.eval(:(using Dagger))
-            Dagger.MemPool.setup_global_device!(
-                Dagger.MemPool.DiskCacheConfig(;
+        # N.B. We pass the Dagger module to help serialization find it remotely
+        remotecall(id, Dagger) do _Dagger
+            _Dagger.MemPool.setup_global_device!(
+                _Dagger.MemPool.DiskCacheConfig(;
                     toggle=true, membound=mem_limits[id], diskbound=disk_limit_mb * 2^20
                 ),
             )