Toward better docs

Ignore built docs
Add simple example to README

Author: ExpandingMan

.github/workflows/Documentation.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.4'
+          version: '1.6'
       - name: Install dependencies
         run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()'
       - name: Build and deploy

.gitignore

@@ -7,3 +7,4 @@ Manifest.toml
 *.jld
 *.jls
 *.dot
+docs/build

README.md

@@ -11,7 +11,7 @@
 [build-img]: https://badge.buildkite.com/d8f020afb67a5920709c2b0a29111cf596f3f052099b5b656f.svg?branch=master
 [build-url]: https://buildkite.com/julialang/dagger-dot-jl
 
-At the core of Dagger.jl is a scheduler heavily inspired by [Dask](https://docs.dask.org/en/latest/). It can run computations represented as [directed-acyclic-graphs](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAGs) efficiently on many Julia worker processes.
+At the core of Dagger.jl is a scheduler heavily inspired by [Dask](https://docs.dask.org/en/latest/). It can run computations represented as [directed-acyclic-graphs](https://en.wikipedia.org/wiki/Directed_acyclic_graph) (DAGs) efficiently on many Julia worker processes and threads, as well as GPUs via [DaggerGPU.jl](https://github.com/JuliaGPU/DaggerGPU.jl).
 
 
 ## Installation
@@ -24,10 +24,17 @@ julia> ] add Dagger
 
 ## Usage
 
-Once installed, the `Dagger` package can by used by typing
+Once installed, the `Dagger` package can by used like so
 
 ```julia
+using Distributed; addprocs() # get us some workers
 using Dagger
+
+# do some stuff in parallel!
+a = Dagger.@spawn 1+3
+b = Dagger.@spawn rand(a, 4)
+c = Dagger.@spawn sum(b)+a
+fetch(c) # some number!
 ```
 
 ## Acknowledgements

docs/make.jl

@@ -19,6 +19,10 @@ makedocs(;
         "Logging and Graphing" => "logging.md",
         "Benchmarking" => "benchmarking.md",
         "Scheduler Internals" => "scheduler-internals.md",
+        "API" => [
+            "Types" => "api/types.md",
+            "Functions and Macros" => "api/functions.md",
+        ]
     ]
 )
 

docs/src/api/functions.md

@@ -0,0 +1,90 @@
+```@meta
+CurrentModule = Dagger
+```
+
+# Functions
+```@index
+Pages = ["functions.md"]
+```
+
+## General
+```@docs
+delayed
+spawn
+tochunk
+domain
+compute
+dependents
+noffspring
+order
+treereduce
+```
+
+## Processors
+```@docs
+execute!
+iscompatible
+default_enabled
+get_processors
+get_parent
+move
+capacity
+get_tls
+set_tls!
+```
+
+## Context
+```@docs
+addprocs!
+rmprocs!
+```
+
+## Logging
+```@docs
+get_logs!
+```
+
+## Thunk Execution Environment
+
+These functions are used within the function called by a `Thunk`.
+
+```@docs
+in_thunk
+thunk_processor
+```
+
+### Dynamic Scheduler Control
+
+These functions query and control the scheduler remotely.
+
+```@docs
+Sch.sch_handle
+Sch.add_thunk!
+Base.fetch
+Base.wait
+Sch.exec!
+Sch.halt!
+Sch.get_dag_ids
+```
+
+## Arrays
+```@docs
+alignfirst
+view
+```
+
+## File IO
+
+!!! warning
+    These APIs are currently untested and may be removed or modified.
+
+```@docs
+save
+load
+```
+
+# Macros
+```@docs
+@par
+@spawn
+```

docs/src/api/types.md

@@ -0,0 +1,58 @@
+```@meta
+CurrentModule = Dagger
+```
+
+# Types
+```@index
+Pages = ["types.md"]
+```
+
+## General
+```@docs
+Thunk
+EagerThunk
+Chunk
+UnitDomain
+```
+
+## Processors
+```@docs
+Processor
+OSProc
+ThreadProc
+```
+
+## Context
+```@docs
+Context
+```
+
+## Logging
+```@docs
+NoOpLog
+LocalEventLog
+```
+
+## Scheduling
+```@docs
+Sch.SchedulerOptions
+Sch.ThunkOptions
+Sch.MaxUtilization
+Sch.DynamicThunkException
+```
+
+## Arrays
+```@docs
+DArray
+Blocks
+ArrayDomain
+```
+
+## File IO
+
+!!! warning
+    These APIs are currently untested and may be removed or modified.
+
+```@docs
+FileReader
+```

src/array/darray.jl

@@ -4,6 +4,11 @@ import Serialization: serialize, deserialize
 
 ###### Array Domains ######
 
+"""
+    ArrayDomain{N}
+
+An `N`-dimensional domain over an array.
+"""
 struct ArrayDomain{N}
     indexes::NTuple{N, Any}
 end
@@ -176,7 +181,9 @@ function Base.isequal(x::ArrayOp, y::ArrayOp)
 end
 
 """
-`view` of a `DArray` chunk returns a `DArray` of thunks
+    view(c::DArray, d)
+
+A `view` of a `DArray` chunk returns a `DArray` of `Thunk`s.
 """
 function Base.view(c::DArray, d)
     subchunks, subdomains = lookup_parts(chunks(c), domainchunks(c), d)
@@ -234,8 +241,10 @@ end
 
 
 """
-A DArray object may contain a thunk in it, in which case
-we first turn it into a Thunk object and then compute it.
+    compute(ctx::Context, x::DArray; persist=true, options=nothing)
+
+A `DArray` object may contain a thunk in it, in which case
+we first turn it into a `Thunk` and then compute it.
 """
 function compute(ctx::Context, x::DArray; persist=true, options=nothing)
     thunk = thunkize(ctx, x, persist=persist)
@@ -247,7 +256,9 @@ function compute(ctx::Context, x::DArray; persist=true, options=nothing)
 end
 
 """
-If a DArray tree has a Thunk in it, make the whole thing a big thunk
+    thunkize(ctx::Context, c::DArray; persist=true)
+
+If a `DArray` tree has a `Thunk` in it, make the whole thing a big thunk.
 """
 function thunkize(ctx::Context, c::DArray; persist=true)
     if any(istask, chunks(c))
@@ -271,14 +282,18 @@ end
 global _stage_cache = WeakKeyDict{Context, Dict}()
 
 """
+    cached_stage(ctx::Context, x)
+
 A memoized version of stage. It is important that the
-tasks generated for the same DArray have the same
+tasks generated for the same `DArray` have the same
 identity, for example:
 
-    A = rand(Blocks(100,100), Float64, 1000, 1000)
-    compute(A+A')
+```julia
+A = rand(Blocks(100,100), Float64, 1000, 1000)
+compute(A+A')
+```
 
-must not result in computation of A twice.
+must not result in computation of `A` twice.
 """
 function cached_stage(ctx::Context, x)
     cache = if !haskey(_stage_cache, ctx)
@@ -308,6 +323,12 @@ size(x::Distribute) = size(domain(x.data))
 
 export BlockPartition, Blocks
 
+"""
+    Blocks(xs...)
+
+Indicates the size of an array operation, specified as `xs`, whose length
+indicates the number of dimensions in the resulting array.
+"""
 struct Blocks{N}
     blocksize::NTuple{N, Int}
 end

src/chunks.jl

@@ -16,6 +16,8 @@ the result of evaluating a Thunk.
 function domain end
 
 """
+    UnitDomain
+
 Default domain -- has no information about the value
 """
 struct UnitDomain end
@@ -29,9 +31,22 @@ domain(x::Any) = UnitDomain()
 ###### Chunk ######
 
 """
-A reference to a piece of data located on a remote worker. `Chunk`s are typically created with `Dagger.tochunk(data)`, and the data can then be accessed from any worker with `collect(::Chunk)`. `Chunk`s are serialization-safe, and use distributed refcounting (provided by `MemPool.DRef`) to ensure that the data referenced by a `Chunk` won't be GC'd, as long as a reference exists on some worker.
-
-Each `Chunk` is associated with a given `Dagger.Processor`, which is (in a sense) the processor that "owns" or contains the data. Calling `collect(::Chunk)` will perform data movement and conversions defined by that processor to safely serialize the data to the calling worker.
+    Chunk
+
+A reference to a piece of data located on a remote worker. `Chunk`s are
+typically created with `Dagger.tochunk(data)`, and the data can then be
+accessed from any worker with `collect(::Chunk)`. `Chunk`s are
+serialization-safe, and use distributed refcounting (provided by
+`MemPool.DRef`) to ensure that the data referenced by a `Chunk` won't be GC'd,
+as long as a reference exists on some worker.
+
+Each `Chunk` is associated with a given `Dagger.Processor`, which is (in a
+sense) the processor that "owns" or contains the data. Calling
+`collect(::Chunk)` will perform data movement and conversions defined by that
+processor to safely serialize the data to the calling worker.
+
+## Constructors
+See [`tochunk`](@ref).
 """
 mutable struct Chunk{T, H, P<:Processor}
     chunktype::Type{T}

src/file-io.jl

@@ -1,6 +1,11 @@
 export save, load
 using SparseArrays
 
+"""
+    FileReader
+
+Used as a `Chunk` handle for reading a file, starting at a given offset.
+"""
 mutable struct FileReader{T}
     file::AbstractString
     chunktype::Type{T}
@@ -40,7 +45,9 @@ function save(ctx, chunk::Union{Chunk, Thunk}, file_path::AbstractString)
 end
 
 """
-special case distmem writing - write to disk on the process with the chunk.
+    save(ctx, chunk, file_path)
+
+Special case distmem writing - write to disk on the process with the chunk.
 """
 function save(ctx, chunk::Chunk{X,DRef}, file_path::AbstractString) where X
     pid = chunk.handle.where