store the dependency list in the serialized image header

Author: stevengj

base/docs/helpdb.jl

@@ -14564,6 +14564,21 @@ Evaluate the contents of a source file in the current context. During including,
 """
 include
 
+doc"""
+```rst
+::
+           include_dependency(path::AbstractString)
+
+In a module, declare that the file specified by `path` (relative or
+absolute) is a dependency for precompilation; that is, the
+module will need to be recompiled if this file changes.
+
+This is only needed if your module depends on a file that is not
+used via `include`.   It has no effect outside of compilation.
+```
+"""
+include_dependency
+
 doc"""
 ```rst
 ::

base/loading.jl

@@ -122,9 +122,11 @@ end
 
 # require always works in Main scope and loads files from node 1
 toplevel_load = true
-function _require(mod::Symbol, track_dependencies::Bool)
+function require(mod::Symbol)
+    # dependency-tracking is only used for one top-level include(path),
+    # and is not applied recursively to imported modules:
     old_track_dependencies = _track_dependencies[1]
-    _track_dependencies[1] = track_dependencies
+    _track_dependencies[1] = false
 
     global toplevel_load
     loading = get(package_locks, mod, false)
@@ -176,8 +178,6 @@ function _require(mod::Symbol, track_dependencies::Bool)
     nothing
 end
 
-require(mod::Symbol) = _require(mod, false)
-
 # remote/parallel load
 
 include_string(txt::ByteString, fname::ByteString) =
@@ -267,6 +267,7 @@ function create_expr_cache(input::AbstractString, output::AbstractString)
             task_local_storage()[:SOURCE_PATH] = $(source)
         end)
     end
+    serialize(io, :(Base._track_dependencies[1] = true))
     serialize(io, :(Base.include($(abspath(input)))))
     if source !== nothing
         serialize(io, quote
@@ -291,3 +292,26 @@ function compile(name::ByteString)
     create_expr_cache(path, cachefile)
     return cachefile
 end
+
+module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m)
+
+function cache_dependencies(cachefile::AbstractString)
+    modules = Tuple{ByteString,UInt64}[]
+    files = ByteString[]
+    open(cachefile, "r") do f
+        while true
+            n = ntoh(read(f, Int32))
+            n == 0 && break
+            push!(modules,
+                  (bytestring(readbytes(f, n)), # module name
+                   ntoh(read(f, UInt64)))) # module UUID (timestamp)
+        end
+        read(f, Int64) # total bytes for file dependencies
+        while true
+            n = ntoh(read(f, Int32))
+            n == 0 && break
+            push!(files, bytestring(readbytes(f, n)))
+        end
+    end
+    return modules, files
+end

src/dump.c

@@ -132,46 +132,48 @@ static jl_array_t *datatype_list=NULL; // (only used in MODE_SYSTEM_IMAGE)
 #define write_int8(s, n) write_uint8(s, n)
 #define read_int8(s) read_uint8(s)
 
+/* read and write in network (bigendian) order: */
+
 static void write_int32(ios_t *s, int32_t i)
 {
-    write_uint8(s, i       & 0xff);
-    write_uint8(s, (i>> 8) & 0xff);
-    write_uint8(s, (i>>16) & 0xff);
     write_uint8(s, (i>>24) & 0xff);
+    write_uint8(s, (i>>16) & 0xff);
+    write_uint8(s, (i>> 8) & 0xff);
+    write_uint8(s, i       & 0xff);
 }
 
 static int32_t read_int32(ios_t *s)
 {
-    int b0 = read_uint8(s);
-    int b1 = read_uint8(s);
-    int b2 = read_uint8(s);
     int b3 = read_uint8(s);
+    int b2 = read_uint8(s);
+    int b1 = read_uint8(s);
+    int b0 = read_uint8(s);
     return b0 | (b1<<8) | (b2<<16) | (b3<<24);
 }
 
 static void write_uint64(ios_t *s, uint64_t i)
 {
-    write_int32(s, i       & 0xffffffff);
     write_int32(s, (i>>32) & 0xffffffff);
+    write_int32(s, i       & 0xffffffff);
 }
 
 static uint64_t read_uint64(ios_t *s)
 {
-    uint64_t b0 = (uint32_t)read_int32(s);
     uint64_t b1 = (uint32_t)read_int32(s);
+    uint64_t b0 = (uint32_t)read_int32(s);
     return b0 | (b1<<32);
 }
 
 static void write_uint16(ios_t *s, uint16_t i)
 {
-    write_uint8(s, i       & 0xff);
     write_uint8(s, (i>> 8) & 0xff);
+    write_uint8(s, i       & 0xff);
 }
 
 static uint16_t read_uint16(ios_t *s)
 {
-    int b0 = read_uint8(s);
     int b1 = read_uint8(s);
+    int b0 = read_uint8(s);
     return b0 | (b1<<8);
 }
 
@@ -949,6 +951,54 @@ void jl_serialize_mod_list(ios_t *s)
     write_int32(s, 0);
 }
 
+// serialize the global _require_dependencies array of pathnames that
+// are include depenencies
+void jl_serialize_dependency_list(ios_t *s)
+{
+     size_t total_size = 0;
+     static jl_array_t *deps = NULL;
+     if (!deps)
+          deps = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("_require_dependencies"));
+     if (deps) {
+          // sort!(deps) so that we can easily eliminate duplicates
+          static jl_value_t *sort_func = NULL;
+          if (!sort_func)
+               sort_func = jl_get_global(jl_base_module, jl_symbol("sort!"));
+          jl_apply((jl_function_t*)sort_func, (jl_value_t**)&deps, 1);
+
+          size_t l = jl_array_len(deps);
+          jl_value_t *prev = NULL;
+          for (size_t i=0; i < l; i++) {
+               jl_value_t *dep = jl_cellref(deps, i);
+               size_t slen = jl_string_len(dep);
+               if (!prev ||
+                   memcmp(jl_string_data(dep), jl_string_data(prev), slen)) {
+                    total_size += 4 + slen;
+               }
+               prev = dep;
+          }
+          total_size += 4;
+     }
+     // write the total size so that we can quickly seek past all of the
+     // dependencies if we don't need them
+     write_uint64(s, total_size);
+     if (deps) {
+          size_t l = jl_array_len(deps);
+          jl_value_t *prev = NULL;
+          for (size_t i=0; i < l; i++) {
+               jl_value_t *dep = jl_cellref(deps, i);
+               size_t slen = jl_string_len(dep);
+               if (!prev ||
+                   memcmp(jl_string_data(dep), jl_string_data(prev), slen)) {
+                    write_int32(s, slen);
+                    ios_write(s, jl_string_data(dep), slen);
+               }
+               prev = dep;
+          }
+          write_int32(s, 0); // terminator, for ease of reading
+     }
+}
+
 // --- deserialize ---
 
 static jl_fptr_t jl_deserialize_fptr(ios_t *s)
@@ -1893,6 +1943,7 @@ DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
     }
     serializer_worklist = worklist;
     jl_serialize_mod_list(&f); // this can throw, keep it early (before any actual initialization)
+    jl_serialize_dependency_list(&f);
 
     JL_SIGATOMIC_BEGIN();
     arraylist_new(&reinit_list, 0);
@@ -1937,6 +1988,8 @@ static jl_array_t *_jl_restore_incremental(ios_t *f)
         ios_close(f);
         return NULL;
     }
+    size_t deplen = read_uint64(f);
+    ios_skip(f, deplen); // skip past the dependency list
     JL_SIGATOMIC_BEGIN();
     arraylist_new(&backref_list, 4000);
     arraylist_push(&backref_list, jl_main_module);

test/compile.jl

@@ -13,26 +13,34 @@ try
         print(f, """
               module $Foo_module
               @doc "foo function" foo(x) = x + 1
+              include_dependency("foo.jl")
               module Bar
               @doc "bar function" bar(x) = x + 2
+              include_dependency("bar.jl")
               end
               end
               """)
     end
 
-    Base.compile(Foo_module)
+    cachefile = Base.compile(Foo_module)
     eval(Main, :(import $Foo_module))
+
+    let Foo = eval(Main, Foo_module)
+        @test Foo.foo(17) == 18
+        @test Foo.Bar.bar(17) == 19
+
+        # issue #12284:
+        @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n"
+        @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n"
+
+        deps = Base.cache_dependencies(cachefile)
+        @test sort(deps[1]) == map(n -> (n, Base.module_uuid(eval(symbol(n)))),
+                                   ["Base","Core","Main"])
+        @test sort(deps[2]) == [file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
+    end
+
 finally
     splice!(Base.LOAD_CACHE_PATH, 1)
     splice!(LOAD_PATH, 1)
     rm(dir, recursive=true)
 end
-
-let Foo = eval(Main, Foo_module)
-    @test Foo.foo(17) == 18
-    @test Foo.Bar.bar(17) == 19
-
-    # issue #12284:
-    @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n"
-    @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n"
-end