diff --git a/NEWS.md b/NEWS.md index 6c3de55f01655..0deb5ebd87f65 100644 --- a/NEWS.md +++ b/NEWS.md @@ -20,6 +20,9 @@ Language changes to the `Core` module ([#29968]). * Using the same name for both a local variable and a static parameter is now an error instead of a warning ([#29429]). + * Precompilation cache files are now created atomically ([#30174]). Note that + invoking _n_ `julia` processes simultaneously may create _n_ temporary + copies of cache files. New library functions --------------------- diff --git a/base/loading.jl b/base/loading.jl index b68d005f53a8c..aa60adf8cb4e1 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1185,18 +1185,31 @@ function compilecache(pkg::PkgId, path::String) else @logmsg verbosity "Precompiling $pkg" end - p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid) - if success(p) - # append checksum to the end of the .ji file: - open(cachefile, "a+") do f - write(f, _crc32c(seekstart(f))) + # create a temporary file in `cachepath` directory, write the cache in it, + # write the checksum, _and then_ atomically move the file to `cachefile`. + tmppath, tmpio = mktemp(cachepath) + local p + try + close(tmpio) + p = create_expr_cache(path, tmppath, concrete_deps, pkg.uuid) + if success(p) + # append checksum to the end of the .ji file: + open(tmppath, "a+") do f + write(f, _crc32c(seekstart(f))) + end + # this is atomic according to POSIX: + rename(tmppath, cachefile) + return cachefile end - elseif p.exitcode == 125 + finally + # not using `mktemp() do ...` to pass `force=true` to `rm` + rm(tmppath, force=true) + end + if p.exitcode == 125 return PrecompilableError() else error("Failed to precompile $pkg to $cachefile.") end - return cachefile end module_build_id(m::Module) = ccall(:jl_module_build_id, UInt64, (Any,), m) diff --git a/doc/src/manual/faq.md b/doc/src/manual/faq.md index e6ae749fd1b5a..01cab373fe29f 100644 --- a/doc/src/manual/faq.md +++ b/doc/src/manual/faq.md @@ -834,6 +834,22 @@ a scalar can participate in linear algebra operations such as `2 * rand(2,2)`, but the analogous operation with a zero-dimensional array `fill(2) * rand(2,2)` is an error. +## Computing cluster + +### How do I manage precompilation cache files in distributed file systems? + +When using `julia` in high-performance computing (HPC) facilities, invoking +_n_ `julia` processes simultaneously creates at most _n_ temporary copies of +cache files. It may become a major issue in slow and/or small distributed +file systems. There are a few possible workarounds: + +1. Use `julia` with `--compilecache=no` flag to turn off precompilation. +2. Configure a private writable depot using `pushfirst!(DEPOT_PATH, private_path)` + where `private_path` is a path unique to this `julia` process. This + can also be done by setting environment variable `JULIA_DEPOT_PATH` to + `$private_path:$HOME/.julia`. +3. Create a symlink from `~/.julia/compiled` to a directory in a scratch space. + ## Julia Releases ### Do I want to use a release, beta, or nightly version of Julia?