Merge pull request #289 from JuliaParallel/jps/storage

Add memory and storage awareness

Author: jpsamaroo

Diff for: .buildkite/pipeline.yml

@@ -14,22 +14,22 @@
     arch: x86_64
     num_cpus: 16
 steps:
-  - label: Julia 1.6
+  - label: Julia 1.7
     timeout_in_minutes: 60
     <<: *test
     plugins:
       - JuliaCI/julia#v1:
-          version: "1.6"
+          version: "1.7"
       - JuliaCI/julia-test#v1:
           julia_args: "--threads=1"
       # - JuliaCI/julia-coverage#v1:
       #     codecov: true
-  - label: Julia 1.7
+  - label: Julia 1.8
     timeout_in_minutes: 60
     <<: *test
     plugins:
       - JuliaCI/julia#v1:
-          version: "1.7"
+          version: "1.8"
       - JuliaCI/julia-test#v1:
           julia_args: "--threads=1"
       # - JuliaCI/julia-coverage#v1:
@@ -39,12 +39,12 @@ steps:
     <<: *test
     plugins:
       - JuliaCI/julia#v1:
-          version: "1.8-nightly"
+          version: "1.9-nightly"
       - JuliaCI/julia-test#v1:
           julia_args: "--threads=1"
       # - JuliaCI/julia-coverage#v1:
       #     codecov: true
-  - label: Julia 1.6 (macOS)
+  - label: Julia 1.7 (macOS)
     timeout_in_minutes: 60
     <<: *test
     agents:
@@ -53,7 +53,7 @@ steps:
       arch: x86_64
     plugins:
       - JuliaCI/julia#v1:
-          version: "1.6"
+          version: "1.7"
       - JuliaCI/julia-test#v1:
           julia_args: "--threads=1"
       # - JuliaCI/julia-coverage#v1:

Diff for: .github/workflows/Documentation.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.6'
+          version: '1.7'
       - name: Install dependencies
         run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
       - name: Build and deploy

Diff for: Project.toml

@@ -7,6 +7,7 @@ Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 ContextVariablesX = "6add18c4-b38d-439d-96f6-d6bc489c04c5"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 MemPool = "f9f48841-c794-520a-933b-121f7ba6ed94"
 Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -21,10 +22,10 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 [compat]
 Colors = "0.10, 0.11, 0.12"
 ContextVariablesX = "0.1"
-MemPool = "0.3.5, 0.4"
+MemPool = "0.4"
 Requires = "1"
 StatsBase = "0.28, 0.29, 0.30, 0.31, 0.32, 0.33"
-julia = "1.6"
+julia = "1.7"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

Diff for: appveyor.yml

@@ -1,6 +1,6 @@
 environment:
   matrix:
-  - julia_version: 1.6
+  - julia_version: 1.7
   - julia_version: nightly
 
 platform:

Diff for: benchmarks/analysis.jl

@@ -0,0 +1,43 @@
+path = ARGS[1]
+start, finish = parse.(UInt64, ARGS[2:3])
+
+using DataFrames, BenchmarkTools, Dagger, Serialization
+
+logs = deserialize(path)["logs"]
+
+"""
+    spans(logs::DataFrame) -> DataFrame
+
+Combines start and finish event pairs, and computes their timespan.
+"""
+function spans(logs)
+    df = Any[]
+    evs = Dict{Symbol,Dict{Any,Any}}()
+    for log in eachrow(logs)
+        _evs = get!(evs, log.core.category) do
+            Dict{Any,Any}()
+        end
+        if log.core.kind == :finish
+            if haskey(_evs, log.id)
+                start = pop!(_evs, log.id)
+                ev = merge(log, (;span=(start.core.timestamp, log.core.timestamp)))
+                push!(df, ev)
+            else
+                @warn "Dropped :finish for $(log.core.category)"
+            end
+        elseif log.core.kind == :start
+            _evs[log.id] = log
+        end
+    end
+    DataFrame(df)
+end
+
+# Combine, select target range, and filter out `take`
+tgt = subset(spans(logs), :core=>ByRow(x->start <= x.timestamp <= finish),
+                          :core=>ByRow(x->x.category != :take))
+
+# Sort by largest time contribution
+tgt_sort = DataFrame(sort(eachrow(tgt), by=r->r.span[2] - r.span[1], rev=true))
+transform!(tgt_sort, :span=>ByRow(s->(s[2]-s[1]) / (1000^3))=>:span_s)
+println("Total: $((finish-start) / (1000^3))")
+foreach(println, zip(map(c->c.category, tgt_sort.core), tgt_sort.span_s))