__precompile__(isprecompilable) function for automated opt-in to module precompilation on import (closes #12462)

Author: stevengj

NEWS.md

@@ -28,13 +28,19 @@ New language features
   * The syntax `function foo end` can be used to introduce a generic function without
     yet adding any methods ([#8283]).
 
-  * Incremental compilation of modules: `Base.compile(module::Symbol)` imports the named module,
-    but instead of loading it into the current session saves the result of compiling it in
+  * Incremental precompilation of modules: call ``__precompile__()`` at the top of a
+    module file to automatically precompile it when it is imported ([#12491]), or manually
+    run `Base.compile(modulename)`. The resulting precompiled `.ji` file is saved in
     `~/.julia/lib/v0.4` ([#8745]).
 
-      * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.
+      * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata.  In particular, to be safely precompilable a module may need an `__init__` function to separate code that must be executed at runtime rather than precompile-time.  Modules that are *not* precompilable should call `__precompile__(false)`.
 
-      * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line.
+      * The precompiled `.ji` file includes a list of dependencies (modules and files that
+        were imported/included at precompile-time), and the module is automatically recompiled
+        upon `import` when any of its dependencies have changed.  Explicit dependencies
+        on other files can be declared with `include_dependency(path)` ([#12458]).
+
+      * New option `--output-incremental={yes|no}` added to invoke the equivalent of ``Base.compilecache`` from the command line.
 
   * The syntax `new{parameters...}(...)` can be used in constructors to specify parameters for
     the type to be constructed ([#8135]).
@@ -1557,3 +1563,5 @@ Too numerous to mention.
 [#12137]: https://github.com/JuliaLang/julia/issues/12137
 [#12162]: https://github.com/JuliaLang/julia/issues/12162
 [#12393]: https://github.com/JuliaLang/julia/issues/12393
+[#12458]: https://github.com/JuliaLang/julia/issues/12458
+[#12491]: https://github.com/JuliaLang/julia/issues/12491

base/docs/helpdb.jl

@@ -8472,12 +8472,12 @@ whos
 doc"""
 ```rst
 ::
-           compile(module::Symbol)
+           Base.compilecache(module::Symbol)
 
 Creates a precompiled cache file for module (see help for ``require``) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in LOAD_CACHE_PATH[1], which defaults to `~/.julia/lib/VERSION`. See the manual section `Module initialization and precompilation` (under `Modules`) for important notes.
 ```
 """
-compile
+compilecache
 
 doc"""
 ```rst
@@ -14579,6 +14579,26 @@ used via `include`.   It has no effect outside of compilation.
 """
 include_dependency
 
+doc"""
+```rst
+::
+           __precompile__(isprecompilable::Bool=true)
+
+Specify whether the file calling this function is precompilable.  If
+``isprecompilable`` is ``true``, then ``__precompile__`` throws an exception
+*unless* the file is being precompiled, and in a module file it causes
+the module to be automatically precompiled when it is imported.
+Typically, ``__precompile__()`` should occur before the ``module``
+declaration in the file, or better yet ``VERSION >= v"0.4" &&
+__precompile__()`` in order to be backward-compatible with Julia 0.3.
+
+If a module or file is *not* safely precompilable, it should call
+``__precompile__(false)`` in order to throw an error if Julia attempts
+to precompile it.
+```
+"""
+__precompile__
+
 doc"""
 ```rst
 ::

base/exports.jl

@@ -1101,6 +1101,7 @@ export
     workspace,
 
 # loading source files
+    __precompile__,
     evalfile,
     include,
     include_string,

base/loading.jl

@@ -131,6 +131,30 @@ function include_dependency(path::AbstractString)
     return nothing
 end
 
+# We throw PrecompilableError(true) when a module wants to be precompiled but isn't,
+# and PrecompilableError(false) when a module doesn't want to be precompiled but is
+immutable PrecompilableError <: Exception
+    isprecompilable::Bool
+end
+function show(io::IO, ex::PrecompilableError)
+    if ex.isprecompilable
+        print(io, "__precompile__(true) is only allowed in module files being imported")
+    else
+        print(io, "__precompile__(false) is not allowed in files that are being precompiled")
+    end
+end
+precompilableerror(ex::PrecompilableError, c) = ex.isprecompilable == c
+precompilableerror(ex::LoadError, c) = precompilableerror(ex.error, c)
+precompilableerror(ex, c) = false
+
+# put at the top of a file to force it to be precompiled (true), or
+# to be prevent it from being precompiled (false).
+function __precompile__(isprecompilable::Bool=true)
+    if myid() == 1 && isprecompilable != (0 != ccall(:jl_generating_output, Cint, ()))
+        throw(PrecompilableError(isprecompilable))
+    end
+end
+
 # require always works in Main scope and loads files from node 1
 toplevel_load = true
 function require(mod::Symbol)
@@ -155,8 +179,8 @@ function require(mod::Symbol)
             return
         end
         if JLOptions().incremental != 0
-            # spawn off a new incremental compile task from node 1 for recursive `require` calls
-            cachefile = compile(mod)
+            # spawn off a new incremental precompile task from node 1 for recursive `require` calls
+            cachefile = compilecache(mod)
             if nothing === _require_from_serialized(1, mod, cachefile, last)
                 warn("require failed to create a precompiled cache file")
             end
@@ -166,13 +190,26 @@ function require(mod::Symbol)
         name = string(mod)
         path = find_in_node_path(name, source_dir(), 1)
         path === nothing && throw(ArgumentError("$name not found in path"))
-        if last && myid() == 1 && nprocs() > 1
-            # broadcast top-level import/using from node 1 (only)
-            content = open(readall, path)
-            refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ]
-            for r in refs; wait(r); end
-        else
-            eval(Main, :(Base.include_from_node1($path)))
+        try
+            if last && myid() == 1 && nprocs() > 1
+                # include on node 1 first to check for PrecompilableErrors
+                eval(Main, :(Base.include_from_node1($path)))
+
+                # broadcast top-level import/using from node 1 (only)
+                refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ]
+                for r in refs; wait(r); end
+            else
+                eval(Main, :(Base.include_from_node1($path)))
+            end
+        catch ex
+            if !precompilableerror(ex, true)
+                rethrow() # rethrow non-precompilable=true errors
+            end
+            isinteractive() && info("Precompiling module $mod...")
+            cachefile = compilecache(mod)
+            if nothing === _require_from_serialized(1, mod, cachefile, last)
+                error("__precompile__(true) but require failed to create a precompiled cache file")
+            end
         end
     finally
         toplevel_load = last
@@ -284,17 +321,19 @@ function create_expr_cache(input::AbstractString, output::AbstractString)
     return pobj
 end
 
-compile(mod::Symbol) = compile(string(mod))
-function compile(name::ByteString)
-    myid() == 1 || error("can only compile from node 1")
+compilecache(mod::Symbol) = compilecache(string(mod))
+function compilecache(name::ByteString)
+    myid() == 1 || error("can only precompile from node 1")
     path = find_in_path(name)
     path === nothing && throw(ArgumentError("$name not found in path"))
     cachepath = LOAD_CACHE_PATH[1]
     if !isdir(cachepath)
         mkpath(cachepath)
     end
     cachefile = abspath(cachepath, name*".ji")
-    create_expr_cache(path, cachefile)
+    if !success(create_expr_cache(path, cachefile))
+        error("Failed to precompile $name to $cachefile")
+    end
     return cachefile
 end
 

doc/manual/modules.rst

@@ -263,9 +263,22 @@ incremental compile and custom system image.
 To create a custom system image that can be used to start julia with the -J option,
 recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules.
 
-To create an incremental precompiled module file,
-call ``Base.compile(modulename::Symbol)``.
-The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``.
+To create an incremental precompiled module file, add
+``__precompile__()`` at the top of your module file (before the
+``module`` starts).  This will cause it to be automatically compiled
+the first time it is imported.  Alternatively, you can manually call
+``Base.compilecache(modulename)``.  The resulting cache files will be
+stored in ``Base.LOAD_CACHE_PATH[1]``.  Subsequently, the module is
+automatically recompiled upon ``import`` whenever any of its
+dependencies change; dependencies are modules it imports, the Julia
+build, files it includes, or explicit dependencies declared by
+``include_dependency(path)`` in the module file(s).  Precompiling a
+module also recursively precompiles any modules that are imported
+therein.  If you know that it is *not* safe to precompile your module
+(for the reasons described below), you should put
+``__precompile__(false)`` in the module file to cause ``Base.compilecache`` to
+throw an error (and thereby prevent the module from being imported by
+any other precompiled module).
 
 In order to make your module work with precompilation,
 however, you may need to change your module to explicitly separate any
@@ -379,15 +392,15 @@ Other known potential failure scenarios include:
        # or move the assignment into the runtime:
        __init__() = global mystdout = Base.STDOUT #= also works =#
 
-Several additional restrictions are placed on the operations that can be done while compiling code
+Several additional restrictions are placed on the operations that can be done while precompiling code
 to help the user avoid other wrong-behavior situations:
 
 1. Calling ``eval`` to cause a side-effect in another module.
-   This will also cause a warning to be emitted when the incremental compile flag is set.
+   This will also cause a warning to be emitted when the incremental precompile flag is set.
 
 2. ``global const`` statements from local scope after ``__init__()`` has been started (see issue #12010 for plans to add an error for this)
 
-3. Replacing a module (or calling ``workspace()``) is a runtime error while doing an incremental compile.
+3. Replacing a module (or calling ``workspace()``) is a runtime error while doing an incremental precompile.
 
 A few other points to be aware of:
 

src/dump.c

@@ -1657,7 +1657,7 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) {
                         jl_errorf("invalid redefinition of constant %s", mod->name->name); // this also throws
                     }
                     if (jl_generating_output() && jl_options.incremental) {
-                        jl_errorf("cannot replace module %s during incremental compile", mod->name->name);
+                        jl_errorf("cannot replace module %s during incremental precompile", mod->name->name);
                     }
                     jl_printf(JL_STDERR, "WARNING: replacing module %s\n", mod->name->name);
                 }

test/compile.jl

@@ -7,10 +7,11 @@ insert!(LOAD_PATH, 1, dir)
 insert!(Base.LOAD_CACHE_PATH, 1, dir)
 Foo_module = :Foo4b3a94a1a081a8cb
 try
-    file = joinpath(dir, "$Foo_module.jl")
+    Foo_file = joinpath(dir, "$Foo_module.jl")
 
-    open(file, "w") do f
+    open(Foo_file, "w") do f
         print(f, """
+              __precompile__(true)
               module $Foo_module
               @doc "foo function" foo(x) = x + 1
               include_dependency("foo.jl")
@@ -22,10 +23,17 @@ try
               """)
     end
 
-    cachefile = Base.compile(Foo_module)
+    if myid() == 1
+        @test_throws Base.PrecompilableError __precompile__(true)
+        @test_throws LoadError include(Foo_file) # from __precompile__(true)
+    end
+
+    Base.require(Foo_module)
+    cachefile = joinpath(dir, "$Foo_module.ji")
 
     # use _require_from_serialized to ensure that the test fails if
     # the module doesn't load from the image:
+    println(STDERR, "\nNOTE: The following 'replacing module' warning indicates normal operation:")
     @test nothing !== Base._require_from_serialized(myid(), Foo_module, true)
 
     let Foo = eval(Main, Foo_module)
@@ -39,9 +47,19 @@ try
         deps = Base.cache_dependencies(cachefile)
         @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))),
                                    [:Base,:Core,:Main])
-        @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
+        @test sort(deps[2]) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
     end
 
+    Baz_file = joinpath(dir, "Baz.jl")
+    open(Baz_file, "w") do f
+        print(f, """
+              __precompile__(false)
+              module Baz
+              end
+              """)
+    end
+    println(STDERR, "\nNOTE: The following 'LoadError: __precompile__(false)' indicates normal operation")
+    @test_throws ErrorException Base.compilecache("Baz") # from __precompile__(false)
 finally
     splice!(Base.LOAD_CACHE_PATH, 1)
     splice!(LOAD_PATH, 1)