fix #12444 (add magic and format version to beginning of .ji files)

Author: stevengj

base/loading.jl

@@ -84,6 +84,14 @@ function _require_from_serialized(node::Int, path_to_try::ByteString, toplevel_l
         restored = _include_from_serialized(content)
     end
     # otherwise, continue search
+
+    if restored !== nothing
+        for M in restored
+            if isdefined(M, :__META__)
+                push!(Base.Docs.modules, M)
+            end
+        end
+    end
     return restored
 end
 
@@ -119,14 +127,8 @@ function require(mod::Symbol)
     last = toplevel_load::Bool
     try
         toplevel_load = false
-        restored = _require_from_serialized(1, mod, last)
-        if restored !== nothing
-            for M in restored
-                if isdefined(M, :__META__)
-                    push!(Base.Docs.modules, M)
-                end
-            end
-            return true
+        if nothing !== _require_from_serialized(1, mod, last)
+            return
         end
         if JLOptions().incremental != 0
             # spawn off a new incremental compile task from node 1 for recursive `require` calls

src/dump.c

@@ -949,6 +949,25 @@ void jl_serialize_mod_list(ios_t *s)
     write_int32(s, 0);
 }
 
+// "magic" string and version header of .ji file
+static const int JI_FORMAT_VERSION = 0;
+static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature
+static const uint16_t BOM = 0xFEFF; // byte-order marker
+static void jl_serialize_header(ios_t *s)
+{
+    ios_write(s, JI_MAGIC, strlen(JI_MAGIC));
+    write_uint16(s, JI_FORMAT_VERSION);
+    ios_write(s, (char *) &BOM, 2);
+    write_uint8(s, sizeof(void*));
+    const char *OS_NAME = jl_get_OS_NAME()->name, *ARCH = jl_get_ARCH()->name;
+    ios_write(s, OS_NAME, strlen(OS_NAME)+1);
+    ios_write(s, ARCH, strlen(ARCH)+1);
+    ios_write(s, JULIA_VERSION_STRING, strlen(JULIA_VERSION_STRING)+1);
+    const char *branch = jl_git_branch(), *commit = jl_git_commit();
+    ios_write(s, branch, strlen(branch)+1);
+    ios_write(s, commit, strlen(commit)+1);
+}
+
 // --- deserialize ---
 
 static jl_fptr_t jl_deserialize_fptr(ios_t *s)
@@ -1525,6 +1544,29 @@ int jl_deserialize_verify_mod_list(ios_t *s)
     }
 }
 
+static int readstr_verify(ios_t *s, const char *str)
+{
+    size_t len = strlen(str);
+    for (size_t i=0; i < len; ++i)
+        if ((char) read_uint8(s) != str[i])
+            return 0;
+    return 1;
+}
+
+static int jl_deserialize_header(ios_t *s)
+{
+    uint16_t bom;
+    return (readstr_verify(s, JI_MAGIC) &&
+            read_uint16(s) == JI_FORMAT_VERSION &&
+            ios_read(s, (char *) &bom, 2) == 2 && bom == BOM &&
+            read_uint8(s) == sizeof(void*) &&
+            readstr_verify(s, jl_get_OS_NAME()->name) && !read_uint8(s) &&
+            readstr_verify(s, jl_get_ARCH()->name) && !read_uint8(s) &&
+            readstr_verify(s, JULIA_VERSION_STRING) && !read_uint8(s) &&
+            readstr_verify(s, jl_git_branch()) && !read_uint8(s) &&
+            readstr_verify(s, jl_git_commit()) && !read_uint8(s));
+}
+
 jl_array_t *jl_module_init_order;
 
 static void jl_finalize_serializer(ios_t *f) {
@@ -1892,6 +1934,7 @@ DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
         return 1;
     }
     serializer_worklist = worklist;
+    jl_serialize_header(&f);
     jl_serialize_mod_list(&f); // this can throw, keep it early (before any actual initialization)
 
     JL_SIGATOMIC_BEGIN();
@@ -1933,7 +1976,8 @@ static jl_array_t *_jl_restore_incremental(ios_t *f)
         ios_close(f);
         return NULL;
     }
-    if (!jl_deserialize_verify_mod_list(f)) {
+    if (!jl_deserialize_header(f) ||
+        !jl_deserialize_verify_mod_list(f)) {
         ios_close(f);
         return NULL;
     }

src/jlapi.c

@@ -203,7 +203,7 @@ DLLEXPORT void jl_yield()
         jl_call0(yieldfunc);
 }
 
-DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, char *fld)
+DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, const char *fld)
 {
     jl_value_t *v;
     JL_TRY {
@@ -279,6 +279,30 @@ DLLEXPORT const char* jl_ver_string(void)
    return JULIA_VERSION_STRING;
 }
 
+// return char* from ByteString field in Base.GIT_VERSION_INFO
+static const char *git_info_string(const char *fld) {
+    static jl_value_t *GIT_VERSION_INFO = NULL;
+    if (!GIT_VERSION_INFO)
+        GIT_VERSION_INFO = jl_get_global(jl_base_module, jl_symbol("GIT_VERSION_INFO"));
+    jl_value_t *f = jl_get_field(GIT_VERSION_INFO, fld);
+    assert(jl_is_byte_string(f));
+    return jl_string_data(f);
+}
+
+DLLEXPORT const char *jl_git_branch()
+{
+    const char *branch = NULL;
+    if (!branch) branch = git_info_string("branch");
+    return branch;
+}
+
+DLLEXPORT const char *jl_git_commit()
+{
+    const char *commit = NULL;
+    if (!commit) commit = git_info_string("commit");
+    return commit;
+}
+
 // Create function versions of some useful macros
 #undef jl_astaggedvalue
 DLLEXPORT jl_taggedvalue_t *jl_astaggedvalue(jl_value_t *v)

src/julia.h

@@ -977,7 +977,7 @@ DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i);
 DLLEXPORT jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i);
 DLLEXPORT void        jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs);
 DLLEXPORT int         jl_field_isdefined(jl_value_t *v, size_t i);
-DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, char *fld);
+DLLEXPORT jl_value_t *jl_get_field(jl_value_t *o, const char *fld);
 DLLEXPORT jl_value_t *jl_value_ptr(jl_value_t *a);
 
 // arrays
@@ -1073,6 +1073,8 @@ DLLEXPORT int jl_cpu_cores(void);
 DLLEXPORT long jl_getpagesize(void);
 DLLEXPORT long jl_getallocationgranularity(void);
 DLLEXPORT int jl_is_debugbuild(void);
+DLLEXPORT jl_sym_t* jl_get_OS_NAME();
+DLLEXPORT jl_sym_t* jl_get_ARCH();
 
 // environment entries
 DLLEXPORT jl_value_t *jl_environ(int i);
@@ -1569,6 +1571,8 @@ DLLEXPORT extern int jl_ver_minor(void);
 DLLEXPORT extern int jl_ver_patch(void);
 DLLEXPORT extern int jl_ver_is_release(void);
 DLLEXPORT extern const char* jl_ver_string(void);
+DLLEXPORT const char *jl_git_branch();
+DLLEXPORT const char *jl_git_commit();
 
 // nullable struct representations
 typedef struct {

src/sys.c

@@ -731,6 +731,14 @@ DLLEXPORT jl_sym_t* jl_get_OS_NAME()
 #endif
 }
 
+DLLEXPORT jl_sym_t* jl_get_ARCH()
+{
+    static jl_sym_t* ARCH = NULL;
+    if (!ARCH)
+        ARCH = (jl_sym_t*) jl_get_global(jl_base_module, jl_symbol("ARCH"));
+    return ARCH;
+}
+
 #ifdef __cplusplus
 }
 #endif

test/compile.jl

@@ -21,7 +21,10 @@ try
     end
 
     Base.compile(Foo_module)
-    eval(Main, :(import $Foo_module))
+
+    # use _require_from_serialized to ensure that the test fails if
+    # the module doesn't load from the image:
+    @test nothing !== Base._require_from_serialized(myid(), Foo_module, true)
 finally
     splice!(Base.LOAD_CACHE_PATH, 1)
     splice!(LOAD_PATH, 1)