From b4e4b4b00215504dbd12c57a8f0d33d15098271c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 6 Jul 2015 16:06:43 -0400 Subject: [PATCH 01/28] pull julia_save out of the external API and into the atexit hook --- src/init.c | 4 +++- src/julia.h | 5 +++-- ui/repl.c | 3 --- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/init.c b/src/init.c index 1b6040e98aa1f..3f69a26180b34 100644 --- a/src/init.c +++ b/src/init.c @@ -541,6 +541,7 @@ static void jl_uv_exitcleanup_walk(uv_handle_t *handle, void *arg) void jl_write_coverage_data(void); void jl_write_malloc_log(void); +static void julia_save(void); static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdown_queue_item *item) { @@ -551,6 +552,7 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow DLLEXPORT void jl_atexit_hook() { + julia_save(); #if defined(GC_FINAL_STATS) jl_print_gc_stats(JL_STDERR); #endif @@ -1282,7 +1284,7 @@ DLLEXPORT int jl_generating_output() void jl_compile_all(void); -DLLEXPORT void julia_save() +static void julia_save() { if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL) jl_compile_all(); diff --git a/src/julia.h b/src/julia.h index 7b0281f768170..e24684956b775 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1141,8 +1141,9 @@ DLLEXPORT ios_t *jl_create_system_image(); DLLEXPORT void jl_save_system_image(const char *fname); DLLEXPORT void jl_restore_system_image(const char *fname); DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); -DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod); -DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname); +DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t* worklist); +DLLEXPORT jl_array_t *jl_restore_incremental(const char *fname); +DLLEXPORT jl_array_t *jl_restore_incremental_from_buf(const char *buf, size_t sz); void jl_init_restored_modules(); // front end interface diff --git a/ui/repl.c b/ui/repl.c index 40dc6d584fbf5..f4d90b7c807cd 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -513,8 +513,6 @@ static int true_main(int argc, char *argv[]) return 0; } -DLLEXPORT extern void julia_save(); - #ifndef _OS_WINDOWS_ int main(int argc, char *argv[]) { @@ -571,7 +569,6 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) } julia_init(imagepathspecified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); int ret = true_main(argc, (char**)argv); - julia_save(); jl_atexit_hook(); return ret; } From 0ab5f86dc6da861472138f8c1664d6f5605aefce Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 8 Jul 2015 15:35:51 -0400 Subject: [PATCH 02/28] forward the exit status to the atexit hook, so it can avoid recursing as much (or call ing julia_save) during the error case --- src/init.c | 4 ++-- src/jl_uv.c | 2 +- src/julia.h | 10 +++++----- src/julia_internal.h | 2 -- ui/repl.c | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/init.c b/src/init.c index 3f69a26180b34..09de9d24e25e9 100644 --- a/src/init.c +++ b/src/init.c @@ -550,9 +550,9 @@ static struct uv_shutdown_queue_item *next_shutdown_queue_item(struct uv_shutdow return rv; } -DLLEXPORT void jl_atexit_hook() +DLLEXPORT void jl_atexit_hook(int exitcode) { - julia_save(); + if (exitcode == 0) julia_save(); #if defined(GC_FINAL_STATS) jl_print_gc_stats(JL_STDERR); #endif diff --git a/src/jl_uv.c b/src/jl_uv.c index 0241169ea3741..e8f93685dcdb0 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -449,7 +449,7 @@ DLLEXPORT void jl_safe_printf(const char *fmt, ...) DLLEXPORT void jl_exit(int exitcode) { uv_tty_reset_mode(); - jl_atexit_hook(); + jl_atexit_hook(exitcode); exit(exitcode); } diff --git a/src/julia.h b/src/julia.h index e24684956b775..7865134908567 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1133,18 +1133,18 @@ DLLEXPORT void jl_init(const char *julia_home_dir); DLLEXPORT void jl_init_with_image(const char *julia_home_dir, const char *image_relative_path); DLLEXPORT int jl_is_initialized(void); DLLEXPORT int julia_trampoline(int argc, const char *argv[], int (*pmain)(int ac,char *av[])); -DLLEXPORT void jl_atexit_hook(void); +DLLEXPORT void jl_atexit_hook(int status); DLLEXPORT void NORETURN jl_exit(int status); DLLEXPORT void jl_preload_sysimg_so(const char *fname); -DLLEXPORT ios_t *jl_create_system_image(); +DLLEXPORT ios_t *jl_create_system_image(void); DLLEXPORT void jl_save_system_image(const char *fname); DLLEXPORT void jl_restore_system_image(const char *fname); DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t* worklist); DLLEXPORT jl_array_t *jl_restore_incremental(const char *fname); DLLEXPORT jl_array_t *jl_restore_incremental_from_buf(const char *buf, size_t sz); -void jl_init_restored_modules(); +void jl_init_restored_modules(void); // front end interface DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len); @@ -1276,7 +1276,7 @@ DLLEXPORT jl_value_t *jl_call2(jl_function_t *f, jl_value_t *a, jl_value_t *b); DLLEXPORT jl_value_t *jl_call3(jl_function_t *f, jl_value_t *a, jl_value_t *b, jl_value_t *c); // interfacing with Task runtime -DLLEXPORT void jl_yield(); +DLLEXPORT void jl_yield(void); // async signal handling ------------------------------------------------------ @@ -1530,7 +1530,7 @@ typedef struct { extern DLLEXPORT jl_options_t jl_options; -DLLEXPORT int jl_generating_output(); +DLLEXPORT int jl_generating_output(void); // Settings for code_coverage and malloc_log // NOTE: if these numbers change, test/cmdlineargs.jl will have to be updated diff --git a/src/julia_internal.h b/src/julia_internal.h index 7175f37b44e21..ac861689fe7b6 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -182,8 +182,6 @@ extern uv_lib_t *jl_crtdll_handle; extern uv_lib_t *jl_winsock_handle; #endif -DLLEXPORT void jl_atexit_hook(); - #if defined(_CPU_X86_) || defined(_CPU_X86_64_) #define HAVE_CPUID #endif diff --git a/ui/repl.c b/ui/repl.c index f4d90b7c807cd..aabe69a47acf3 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -569,7 +569,7 @@ int wmain(int argc, wchar_t *argv[], wchar_t *envp[]) } julia_init(imagepathspecified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); int ret = true_main(argc, (char**)argv); - jl_atexit_hook(); + jl_atexit_hook(ret); return ret; } From 3c3f46da6d76e1a9b0294924eb1fad6a594045fa Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 8 Jul 2015 22:42:21 -0400 Subject: [PATCH 03/28] consider exit-on-segv to be an exceptional exit status, per common unix convention --- src/init.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/init.c b/src/init.c index 09de9d24e25e9..b18edf0a32083 100644 --- a/src/init.c +++ b/src/init.c @@ -169,7 +169,7 @@ static void jl_find_stack_bottom(void) // what to do on SIGINT DLLEXPORT void jl_sigint_action(void) { - if (exit_on_sigint) jl_exit(0); + if (exit_on_sigint) jl_exit(130); // 128+SIGINT jl_throw(jl_interrupt_exception); } @@ -363,7 +363,7 @@ static BOOL WINAPI sigint_handler(DWORD wsig) //This needs winapi types to guara } else { jl_signal_pending = 0; - if (exit_on_sigint) jl_exit(0); + if (exit_on_sigint) jl_exit(130); if ((DWORD)-1 == SuspendThread(hMainThread)) { //error jl_safe_printf("error: SuspendThread failed\n"); @@ -770,7 +770,7 @@ void *mach_segv_listener(void *arg) while (1) { int ret = mach_msg_server(exc_server,2048,segv_port,MACH_MSG_TIMEOUT_NONE); jl_safe_printf("mach_msg_server: %s\n", mach_error_string(ret)); - jl_exit(1); + jl_exit(128+SIGSEGV); } } From aacf7d3839b2dadb436f4672abe581afddf979ec Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 11 Dec 2014 02:31:37 -0500 Subject: [PATCH 04/28] automatically search cache modules when calling require(::Symbol) or using/import --- base/client.jl | 3 +++ base/loading.jl | 29 ++++++++++++++++++++++ src/dump.c | 66 ++++++++++++++++++++++++++++++++++--------------- src/toplevel.c | 5 +--- 4 files changed, 79 insertions(+), 24 deletions(-) diff --git a/base/client.jl b/base/client.jl index 33701242590b4..38b486783a06e 100644 --- a/base/client.jl +++ b/base/client.jl @@ -322,6 +322,7 @@ is_interactive = false isinteractive() = (is_interactive::Bool) const LOAD_PATH = ByteString[] +const LOAD_CACHE_PATH = ByteString[] function init_load_path() vers = "v$(VERSION.major).$(VERSION.minor)" if haskey(ENV,"JULIA_LOAD_PATH") @@ -329,6 +330,8 @@ function init_load_path() end push!(LOAD_PATH,abspath(JULIA_HOME,"..","local","share","julia","site",vers)) push!(LOAD_PATH,abspath(JULIA_HOME,"..","share","julia","site",vers)) + push!(LOAD_CACHE_PATH,abspath(homedir(),".julia",".libs")) + push!(LOAD_CACHE_PATH,abspath(JULIA_HOME,"..","usr","lib","julia")) #TODO: fixme end function load_juliarc() diff --git a/base/loading.jl b/base/loading.jl index b6eb34c9e05e3..cf605605f5a70 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -43,6 +43,35 @@ require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs req # only broadcast top-level (not nested) requires and reloads toplevel_load = true +function _require_from_serialized(content) + m = ccall(:jl_restore_new_module_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content)) + #package_list[path] = time() + return m +end + +function require(sname::Symbol) + name = string(sname) + for prefix in LOAD_CACHE_PATH + path = joinpath(prefix, name*".ji") + if isfile(path) + if nprocs() == 1 + if ccall(:jl_restore_new_module, UInt, (Ptr{Uint8},), path) != 0 + #package_list[path] = time() + return + end + else + content = open(readbytes, path) + if _require_from_serialized(content) != 0 + refs = Any[ @spawnat p _require_from_serialized(content) for p in filter(x->x!=1, procs()) ] + for r in refs; wait(r); end + return + end + end + end + end + require(name) +end + function require(name::AbstractString) path = find_in_node1_path(name) path == nothing && throw(ArgumentError("$name not found in path")) diff --git a/src/dump.c b/src/dump.c index 1f08520b28daf..66800cb870c54 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1423,17 +1423,31 @@ int jl_deserialize_verify_mod_list(ios_t *s) ios_read(s, name, len); name[len] = '\0'; uint64_t uuid = read_uint64(s); - jl_module_t *m = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol(name)); + jl_sym_t *sym = jl_symbol(name); + jl_module_t *m = (jl_module_t*)jl_get_global(jl_main_module, sym); if (!m) { - jl_printf(JL_STDERR, "error: Module %s must be loaded first\n", name); + static jl_value_t *require_func = NULL; + if (!require_func) + require_func = jl_get_global(jl_base_module, jl_symbol("require")); + JL_TRY { + jl_apply((jl_function_t*)require_func, (jl_value_t**)&sym, 1); + } + JL_CATCH { + ios_close(s); + jl_rethrow(); + } + m = (jl_module_t*)jl_get_global(jl_main_module, sym); + } + if (!m) { + jl_printf(JL_STDERR, "error: requiring \"%s\" did not define a corresponding module\n", name); return 0; } if (!jl_is_module(m)) { ios_close(s); - jl_errorf("typeassert: expected %s::Module", name); + jl_errorf("invalid module path (%s does not name a module)", name); } if (m->uuid != uuid) { - jl_printf(JL_STDERR, "error: Module %s uuid did not match cache file\n", name); + jl_printf(JL_STDERR, "warning: Module %s uuid did not match cache file\n", name); return 0; } } @@ -1790,19 +1804,14 @@ DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) jl_function_t *jl_method_cache_insert(jl_methtable_t *mt, jl_tupletype_t *type, jl_function_t *method); -DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) +static jl_module_t *_jl_restore_new_module(ios_t *f) { - ios_t f; - if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { - jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); + if (ios_eof(f)) { + ios_close(f); return NULL; } - if (ios_eof(&f)) { - ios_close(&f); - return NULL; - } - if (!jl_deserialize_verify_mod_list(&f)) { - ios_close(&f); + if (!jl_deserialize_verify_mod_list(f)) { + ios_close(f); return NULL; } arraylist_new(&backref_list, 4000); @@ -1813,14 +1822,14 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; - jl_module_t *parent = (jl_module_t*)jl_deserialize_value(&f, NULL); - jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(&f, NULL); + jl_module_t *parent = (jl_module_t*)jl_deserialize_value(f, NULL); + jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(f, NULL); jl_binding_t *b = jl_get_binding_wr(parent, name); jl_declare_constant(b); if (b->value != NULL) { jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); } - b->value = jl_deserialize_value(&f, &b->value); + b->value = jl_deserialize_value(f, &b->value); size_t i = 0; while (i < flagref_list.len) { @@ -1882,9 +1891,9 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) } mode = MODE_MODULE_LAMBDAS; - jl_deserialize_lambdas_from_mod(&f); + jl_deserialize_lambdas_from_mod(f); - jl_module_init_order = (jl_array_t*)jl_deserialize_value(&f, NULL); + jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); for (i = 0; i < methtable_list.len; i++) { jl_methtable_t *mt = (jl_methtable_t*)methtable_list.items[i]; @@ -1921,13 +1930,30 @@ DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) arraylist_free(&flagref_list); arraylist_free(&methtable_list); arraylist_free(&backref_list); - ios_close(&f); + ios_close(f); jl_init_restored_modules(); return (jl_module_t*)b->value; } +DLLEXPORT jl_module_t *jl_restore_new_module_from_buf(const char *buf, size_t sz) +{ + ios_t f; + ios_static_buffer(&f, (char*)buf, sz); + return _jl_restore_new_module(&f); +} + +DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) +{ + ios_t f; + if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { + jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); + return NULL; + } + return _jl_restore_new_module(&f); +} + // --- init --- void jl_init_serializer(void) diff --git a/src/toplevel.c b/src/toplevel.c index 9d0afe6d059da..8fedd64f163ad 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -348,10 +348,7 @@ static jl_module_t *eval_import_path_(jl_array_t *args, int retrying) if (require_func == NULL && jl_base_module != NULL) require_func = jl_get_global(jl_base_module, jl_symbol("require")); if (require_func != NULL) { - jl_value_t *str = jl_cstr_to_string(var->name); - JL_GC_PUSH1(&str); - jl_apply((jl_function_t*)require_func, &str, 1); - JL_GC_POP(); + jl_apply((jl_function_t*)require_func, (jl_value_t**)&var, 1); return eval_import_path_(args, 1); } } From 53460176c86ddc2df39acc57cd748633d19314ec Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 29 Dec 2014 01:14:38 -0500 Subject: [PATCH 05/28] first cut at automatically caching @Base.cacheable modules --- base/boot.jl | 4 ++-- base/loading.jl | 62 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index b95d541127e42..f96475c39810d 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -71,8 +71,8 @@ # module::Module #end -#type Box{T} -# contents::T +#type Box +# contents::Any #end #abstract Ref{T} diff --git a/base/loading.jl b/base/loading.jl index cf605605f5a70..7b6d7b62a7b23 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -49,27 +49,33 @@ function _require_from_serialized(content) return m end -function require(sname::Symbol) - name = string(sname) +function _require_from_cache(name::AbstractString) for prefix in LOAD_CACHE_PATH path = joinpath(prefix, name*".ji") if isfile(path) if nprocs() == 1 if ccall(:jl_restore_new_module, UInt, (Ptr{Uint8},), path) != 0 #package_list[path] = time() - return + return true end else content = open(readbytes, path) if _require_from_serialized(content) != 0 refs = Any[ @spawnat p _require_from_serialized(content) for p in filter(x->x!=1, procs()) ] for r in refs; wait(r); end - return + return true end end end end - require(name) + return false +end + +function require(sname::Symbol) + name = string(sname) + if !_require_from_cache(name) + require(name) + end end function require(name::AbstractString) @@ -188,7 +194,7 @@ function reload_path(path::AbstractString) had || delete!(package_list, path) rethrow(e) finally - if prev != nothing + if prev !== nothing tls[:SOURCE_PATH] = prev end end @@ -209,3 +215,47 @@ function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[]) :(include($path)))) end evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...]) + +function create_expr_cache(m::Expr) + assert(m.head === :module) + typeassert(m.args[2], Symbol) + cachepath = LOAD_CACHE_PATH[1] + if !isdir(cachepath) + mkdir(cachepath) + end + code_object = """ + while !eof(STDIN) + eval(Main, deserialize(STDIN)) + end + """ + io, pobj = open(detach(setenv(`$(julia_cmd()) --build $cachepath --startup-file=no --history-file=no -E $code_object`,["JULIA_HOME=$JULIA_HOME","HOME=$(homedir())"])), "w") + serialize(io, quote + empty!(Base.LOAD_PATH) + append!(Base.LOAD_PATH, $LOAD_PATH) + empty!(Base.LOAD_CACHE_PATH) + append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH) + empty!(Base.DL_LOAD_PATH) + append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH) + end) + source = source_path(nothing) + if source !== nothing + serialize(io, quote + task_local_storage()[:SOURCE_PATH] = $(source) + end) + end + serialize(io, m) + if source !== nothing + serialize(io, quote + delete!(task_local_storage(), :SOURCE_PATH) + end) + end + close(io) + wait(pobj) + if !_require_from_cache(string(m.args[2])) + error("Warning: @cacheable module failed to define a module") + end +end + +macro cacheable(m) + :(create_expr_cache($(QuoteNode(m)))) +end From e78535cfb76e128b6e64aa5c216f3b1193b45a5f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 5 Jul 2015 16:54:02 -0400 Subject: [PATCH 06/28] make dump/restore sigatomic --- src/dump.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/dump.c b/src/dump.c index 66800cb870c54..7e21f13143481 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1459,8 +1459,9 @@ extern jl_array_t *jl_module_init_order; void jl_save_system_image_to_stream(ios_t *f) { - jl_gc_collect(1); - jl_gc_collect(0); + jl_gc_collect(1); // full + jl_gc_collect(0); // incremental (sweep finalizers) + JL_SIGATOMIC_BEGIN(); int en = jl_gc_enable(0); htable_reset(&backref_table, 250000); arraylist_new(&reinit_list, 0); @@ -1508,6 +1509,7 @@ void jl_save_system_image_to_stream(ios_t *f) arraylist_free(&reinit_list); jl_gc_enable(en); + JL_SIGATOMIC_END(); } DLLEXPORT void jl_save_system_image(const char *fname) @@ -1516,8 +1518,10 @@ DLLEXPORT void jl_save_system_image(const char *fname) if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) { jl_errorf("Cannot open system image file \"%s\" for writing.\n", fname); } + JL_SIGATOMIC_BEGIN(); jl_save_system_image_to_stream(&f); ios_close(&f); + JL_SIGATOMIC_END(); } DLLEXPORT ios_t *jl_create_system_image() @@ -1561,6 +1565,7 @@ DLLEXPORT void jl_preload_sysimg_so(const char *fname) void jl_restore_system_image_from_stream(ios_t *f) { + JL_SIGATOMIC_BEGIN(); int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_SYSTEM_IMAGE; @@ -1630,6 +1635,7 @@ void jl_restore_system_image_from_stream(ios_t *f) jl_gc_enable(en); mode = last_mode; jl_update_all_fptrs(); + JL_SIGATOMIC_END(); } DLLEXPORT void jl_restore_system_image(const char *fname) @@ -1649,17 +1655,21 @@ DLLEXPORT void jl_restore_system_image(const char *fname) ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) jl_errorf("System image file \"%s\" not found\n", fname); + JL_SIGATOMIC_BEGIN(); jl_restore_system_image_from_stream(&f); ios_close(&f); + JL_SIGATOMIC_END(); } } DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) { ios_t f; + JL_SIGATOMIC_BEGIN(); ios_static_buffer(&f, (char*)buf, len); jl_restore_system_image_from_stream(&f); ios_close(&f); + JL_SIGATOMIC_END(); } void jl_init_restored_modules() @@ -1681,6 +1691,7 @@ DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) { if (jl_is_expr(ast)) return jl_lam_body((jl_expr_t*)ast)->etype; + JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; if (li->module->constant_table == NULL) @@ -1696,11 +1707,13 @@ DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) jl_gc_enable(en); tree_literal_values = NULL; mode = last_mode; + JL_SIGATOMIC_END(); return rt; } DLLEXPORT jl_value_t *jl_compress_ast(jl_lambda_info_t *li, jl_value_t *ast) { + JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; ios_t dest; @@ -1732,11 +1745,13 @@ DLLEXPORT jl_value_t *jl_compress_ast(jl_lambda_info_t *li, jl_value_t *ast) tree_enclosing_module = last_tem; jl_gc_enable(en); mode = last_mode; + JL_SIGATOMIC_END(); return v; } DLLEXPORT jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data) { + JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; jl_array_t *bytes = (jl_array_t*)data; @@ -1753,6 +1768,7 @@ DLLEXPORT jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data) tree_literal_values = NULL; tree_enclosing_module = NULL; mode = last_mode; + JL_SIGATOMIC_END(); return v; } @@ -1763,6 +1779,7 @@ DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname); return 1; } + JL_SIGATOMIC_BEGIN(); jl_module_t *lastmod = jl_current_module; jl_current_module = mod; jl_serialize_mod_list(&f); @@ -1797,6 +1814,7 @@ DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) htable_reset(&backref_table, 0); ios_close(&f); + JL_SIGATOMIC_END(); return 0; } @@ -1814,6 +1832,7 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) ios_close(f); return NULL; } + JL_SIGATOMIC_BEGIN(); arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); arraylist_new(&flagref_list, 0); @@ -1824,10 +1843,17 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) mode = MODE_MODULE; jl_module_t *parent = (jl_module_t*)jl_deserialize_value(f, NULL); jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(f, NULL); - jl_binding_t *b = jl_get_binding_wr(parent, name); - jl_declare_constant(b); - if (b->value != NULL) { - jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); + jl_exception_in_transit = NULL; + jl_binding_t *b; + JL_TRY { + b = jl_get_binding_wr(parent, name); + jl_declare_constant(b); // this can throw + if (b->value != NULL) { + jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); + } + } + JL_CATCH { + goto cleanup; } b->value = jl_deserialize_value(f, &b->value); @@ -1925,12 +1951,15 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) } } +cleanup: mode = last_mode; jl_gc_enable(en); arraylist_free(&flagref_list); arraylist_free(&methtable_list); arraylist_free(&backref_list); ios_close(f); + JL_SIGATOMIC_END(); + if (jl_exception_in_transit) jl_rethrow(); jl_init_restored_modules(); From c8119f8140b8cea7e30ac9626e87e4d72885dabc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 6 Jul 2015 15:55:40 -0400 Subject: [PATCH 07/28] give the dump serializer the ability to handle a list of modules to serialize, rather than just a single module --- src/dump.c | 301 ++++++++++++++++++++++++++++--------------------- src/toplevel.c | 2 +- 2 files changed, 176 insertions(+), 127 deletions(-) diff --git a/src/dump.c b/src/dump.c index 7e21f13143481..8fab923b9fc97 100644 --- a/src/dump.c +++ b/src/dump.c @@ -43,19 +43,24 @@ int backref_table_numel; static arraylist_t backref_list; // list of (jl_value_t **loc, size_t pos) entries -// for anything that was flagged by the serializer for later +// for anything that was flagged by the deserializer for later // type-rewriting of some sort static arraylist_t flagref_list; // list of (size_t pos, (void *f)(jl_value_t*)) entries // for the serializer to mark values in need of rework by function f +// during deserialization later static arraylist_t reinit_list; // list of any methtable objects that were deserialized in MODE_MODULE // and need to be rehashed after assigning the uid fields to types -// (only used in MODE_MODULE and MODE_MODULE_LAMBDAS) +// (only used in MODE_MODULE and MODE_MODULE_POSTWORK) static arraylist_t methtable_list; +// list of stuff that is being serialized +// (only used by the incremental serializer in MODE_MODULE) +static jl_array_t *serializer_worklist; + // hash of definitions for predefined function pointers static htable_t fptr_to_id; // array of definitions for the predefined function pointers @@ -110,7 +115,7 @@ typedef enum _DUMP_MODES { // restoring a single module from disk for integration // into the currently running system image / environment MODE_MODULE, // first-stage (pre type-uid assignment) - MODE_MODULE_LAMBDAS, // second-stage (post type-uid assignment) + MODE_MODULE_POSTWORK, // second-stage (post type-uid assignment) } DUMP_MODES; static DUMP_MODES mode = (DUMP_MODES) 0; @@ -392,15 +397,33 @@ static void jl_serialize_fptr(ios_t *s, void *fptr) write_uint16(s, *(ptrint_t*)pbp); } +static int module_in_worklist(jl_module_t *mod) { + int i, l = jl_array_len(serializer_worklist); + for (i = 0; i < l; i++) { + jl_module_t *workmod = (jl_module_t*)jl_cellref(serializer_worklist, i); + if (jl_is_module(workmod) && jl_is_submodule(mod, workmod)) + return 1; + } + return 0; +} + static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) { int tag = 0; - if (mode == MODE_MODULE_LAMBDAS) { + if (mode == MODE_MODULE_POSTWORK) { if (dt->uid != 0) tag = 6; // must use apply_type } else if (mode == MODE_MODULE) { - int internal = jl_is_submodule(dt->name->module, jl_current_module); + int internal = module_in_worklist(dt->name->module); + int i, l = jl_array_len(serializer_worklist); + for (i = 0; i < l; i++) { + jl_module_t *mod = (jl_module_t*)jl_cellref(serializer_worklist, i); + if (jl_is_module(mod) && jl_is_submodule(dt->name->module, mod)) { + internal = 1; + break; + } + } if (!internal && dt->name->primary == (jl_value_t*)dt) { tag = 6; // external primary type } @@ -443,7 +466,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) write_uint8(s, dt->abstract | (dt->mutabl<<1) | (dt->pointerfree<<2) | (has_instance<<3)); if (!dt->abstract) { write_uint16(s, dt->ninitialized); - if (mode != MODE_MODULE && mode != MODE_MODULE_LAMBDAS) { + if (mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK) { write_int32(s, dt->uid); } } @@ -466,12 +489,12 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) writetag(s, jl_module_type); jl_serialize_value(s, m->name); int ref_only = 0; - if (mode == MODE_MODULE_LAMBDAS) { - assert(!jl_is_submodule(m, jl_current_module)); + if (mode == MODE_MODULE_POSTWORK) { + assert(!module_in_worklist(m)); ref_only = 1; } if (mode == MODE_MODULE) { - if (!jl_is_submodule(m, jl_current_module)) + if (!module_in_worklist(m)) ref_only = 1; write_int8(s, ref_only); } @@ -581,13 +604,22 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) return; } ptrint_t pos = backref_table_numel++; - if (mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) pos <<= 1; ptrhash_put(&backref_table, v, (void*)pos); if (jl_typeof(v) == jl_idtable_type) { + // will need to rehash this, later (after type is fully constructed) arraylist_push(&reinit_list, (void*)pos); arraylist_push(&reinit_list, (void*)1); } + if (mode == MODE_MODULE && jl_is_module(v)) { + jl_module_t *m = (jl_module_t*)v; + if (module_in_worklist(m) && !module_in_worklist(m->parent)) { + // will need to reinsert this into parent bindings, later (in case of any errors during reinsert) + arraylist_push(&reinit_list, (void*)pos); + arraylist_push(&reinit_list, (void*)2); + } + } } size_t i; @@ -756,14 +788,14 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else { if (v == t->instance) { - assert(mode != MODE_MODULE_LAMBDAS); + assert(mode != MODE_MODULE_POSTWORK); writetag(s, (jl_value_t*)Singleton_tag); return; } writetag(s, (jl_value_t*)jl_datatype_type); jl_serialize_value(s, t); - if ((mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) && t == jl_typename_type) { - if (jl_is_submodule(((jl_typename_t*)v)->module, jl_current_module)) { + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && t == jl_typename_type) { + if (module_in_worklist(((jl_typename_t*)v)->module)) { write_uint8(s, 0); } else { @@ -814,7 +846,7 @@ static void jl_serialize_methtable_from_mod(ios_t *s, jl_module_t *m, jl_sym_t * } *chain = NULL; jl_methlist_t *ml = mt->defs; while (ml != (void*)jl_nothing) { - if (jl_is_submodule(ml->func->linfo->module, jl_current_module)) { + if (module_in_worklist(ml->func->linfo->module)) { struct _chain *link = (struct _chain*)alloca(sizeof(struct _chain)); link->ml = ml; link->next = chain; @@ -840,16 +872,17 @@ static void jl_serialize_methtable_from_mod(ios_t *s, jl_module_t *m, jl_sym_t * static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) { - if (m == jl_current_module) return; + if (module_in_worklist(m)) return; size_t i; void **table = m->bindings.table; for(i=1; i < m->bindings.size; i+=2) { if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; - if (b->owner == m && b->value) { + if (b->owner == m && b->value && b->constp) { if (jl_is_function(b->value)) { jl_function_t *gf = (jl_function_t*)b->value; if (jl_is_gf(gf)) { + // TODO: verify this is not an alternative name for the gf jl_methtable_t *mt = jl_gf_mtable(gf); jl_serialize_methtable_from_mod(s, m, b->name, mt, 0); jl_serialize_methtable_from_mod(s, m, b->name, mt, 1); @@ -868,6 +901,7 @@ static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) } } +// serialize information about all of the modules accessible directly from Main void jl_serialize_mod_list(ios_t *s) { jl_module_t *m = jl_main_module; @@ -877,9 +911,9 @@ void jl_serialize_mod_list(ios_t *s) if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; if (b->owner == m && - b->value && - b->value != (jl_value_t*)jl_current_module && - jl_is_module(b->value)) { + b->value && b->constp && + jl_is_module(b->value) && + module_in_worklist((jl_module_t*)b->value)) { jl_module_t *child = (jl_module_t*)b->value; if (child->name == b->name) { // this is the original/primary binding for the submodule @@ -941,7 +975,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt->pointerfree = (flags>>2)&1; if (!dt->abstract) { dt->ninitialized = read_uint16(s); - dt->uid = mode != MODE_MODULE && mode != MODE_MODULE_LAMBDAS ? read_int32(s) : 0; + dt->uid = mode != MODE_MODULE && mode != MODE_MODULE_POSTWORK ? read_int32(s) : 0; } else { dt->ninitialized = 0; @@ -1019,7 +1053,7 @@ static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) isdatatype = !!(offs & 1); offs >>= 1; } - else if (mode == MODE_MODULE_LAMBDAS) { + else if (mode == MODE_MODULE_POSTWORK) { offs >>= 1; } assert(offs >= 0 && offs < backref_list.len); @@ -1223,7 +1257,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t arraylist_push(&backref_list, NULL); jl_sym_t *mname = (jl_sym_t*)jl_deserialize_value(s, NULL); int ref_only = 0; - if (mode == MODE_MODULE_LAMBDAS) { + if (mode == MODE_MODULE_POSTWORK) { ref_only = 1; } else if (mode == MODE_MODULE) { @@ -1301,7 +1335,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); if (dt == jl_datatype_type) return jl_deserialize_datatype(s, pos, loc); - if ((mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) && dt == jl_typename_type) { + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && dt == jl_typename_type) { int ref_only = read_uint8(s); if (ref_only) { jl_module_t *m = (jl_module_t*)jl_deserialize_value(s, NULL); @@ -1356,7 +1390,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_set_nth_field(v, i, jl_deserialize_value(s, (dt->fields[i].isptr) ? (jl_value_t**)(data+jl_field_offset(dt, i)) : NULL)); } - if ((mode == MODE_MODULE || mode == MODE_MODULE_LAMBDAS) && jl_is_mtable(v)) + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && jl_is_mtable(v)) arraylist_push(&methtable_list, v); } // TODO: put WeakRefs on the weak_refs list @@ -1367,7 +1401,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return v; } else if (vtag == (jl_value_t*)Singleton_tag) { - assert(mode != MODE_MODULE_LAMBDAS); + assert(mode != MODE_MODULE_POSTWORK); jl_value_t *v = (jl_value_t*)jl_gc_alloc_0w(); if (usetable) { uptrint_t pos = backref_list.len; @@ -1453,9 +1487,90 @@ int jl_deserialize_verify_mod_list(ios_t *s) } } -// --- entry points --- +jl_array_t *jl_module_init_order; + +static void jl_finalize_serializer(ios_t *f) { + size_t i, l; + // save module initialization order + if (jl_module_init_order != NULL) { + l = jl_array_len(jl_module_init_order); + for(i=0; i < l; i++) { + // verify that all these modules were saved + assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); + } + } + jl_serialize_value(f, jl_module_init_order); + + // record list of reinitialization functions + l = reinit_list.len; + for (i = 0; i < l; i += 2) { + write_int32(f, (int)((uintptr_t) reinit_list.items[i])); + write_int32(f, (int)((uintptr_t) reinit_list.items[i+1])); + } + write_int32(f, -1); +} + +static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) { + JL_TRY { + switch (how) { + case 1: { // rehash ObjectIdDict + jl_array_t **a = (jl_array_t**)&v->fieldptr[0]; + jl_idtable_rehash(a, jl_array_len(*a)); + jl_gc_wb(v, *a); + break; + } + case 2: { // reinsert v (const) into parent as sym + jl_module_t *parent = (jl_module_t*)jl_deserialize_value(f, NULL); + jl_sym_t *sym = (jl_sym_t*)jl_deserialize_value(f, NULL); + jl_binding_t *b = jl_get_binding_wr(parent, sym); + jl_declare_constant(b); // this can throw + if (b->value != NULL) { + jl_printf(JL_STDERR, "Warning: replacing module %s\n", sym->name); + } + b->value = v; + jl_gc_wb_binding(b, v); + break; + } + default: + assert(0); + } + } + JL_CATCH { + jl_printf(JL_STDERR, "Warning: error reinitializing value "); + jl_static_show(JL_STDERR, v); + jl_printf(JL_STDERR, ":\n"); + jl_static_show(JL_STDERR, jl_exception_in_transit); + jl_printf(JL_STDERR, "\n"); + } +} +static void jl_finalize_deserializer(ios_t *f) { + jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); -extern jl_array_t *jl_module_init_order; + // run reinitialization functions + int pos = read_int32(f); + while (pos != -1) { + jl_reinit_item(f, (jl_value_t*)backref_list.items[pos], read_int32(f)); + pos = read_int32(f); + } +} + +void jl_init_restored_modules() +{ + if (jl_module_init_order != NULL) { + jl_array_t *temp = jl_module_init_order; + jl_module_init_order = NULL; + JL_GC_PUSH1(&temp); + int i; + for(i=0; i < jl_array_len(temp); i++) { + jl_value_t *mod = jl_cellref(temp, i); + jl_module_run_initializer((jl_module_t*)mod); + } + JL_GC_POP(); + } +} + + +// --- entry points --- void jl_save_system_image_to_stream(ios_t *f) { @@ -1485,25 +1600,9 @@ void jl_save_system_image_to_stream(ios_t *f) jl_serialize_gv_syms(f, jl_get_root_symbol()); // serialize symbols with GlobalValue references jl_serialize_value(f, NULL); // signal the end of the symbols list - // save module initialization order - if (jl_module_init_order != NULL) { - size_t i; - for(i=0; i < jl_array_len(jl_module_init_order); i++) { - // verify that all these modules were saved - assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); - } - } - jl_serialize_value(f, jl_module_init_order); - write_int32(f, jl_get_t_uid_ctr()); write_int32(f, jl_get_gs_ctr()); - - // record reinitialization functions - for (i = 0; i < reinit_list.len; i += 2) { - write_int32(f, (int)((uintptr_t) reinit_list.items[i])); - write_int32(f, (int)((uintptr_t) reinit_list.items[i+1])); - } - write_int32(f, -1); + jl_finalize_serializer(f); htable_reset(&backref_table, 0); arraylist_free(&reinit_list); @@ -1591,7 +1690,9 @@ void jl_restore_system_image_from_stream(ios_t *f) jl_deserialize_globalvals(f); jl_deserialize_gv_syms(f); - jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); + int uid_ctr = read_int32(f); + int gs_ctr = read_int32(f); + jl_finalize_deserializer(f); // done with f // cache builtin parametric types for(int i=0; i < jl_array_len(datatype_list); i++) { @@ -1609,25 +1710,8 @@ void jl_restore_system_image_from_stream(ios_t *f) jl_boot_file_loaded = 1; jl_init_box_caches(); - jl_set_t_uid_ctr(read_int32(f)); - jl_set_gs_ctr(read_int32(f)); - - // run reinitialization functions - int pos = read_int32(f); - while (pos != -1) { - jl_value_t *v = (jl_value_t*)backref_list.items[pos]; - switch (read_int32(f)) { - case 1: { - jl_array_t **a = (jl_array_t**)&v->fieldptr[0]; - jl_idtable_rehash(a, jl_array_len(*a)); - jl_gc_wb(v, *a); - break; - } - default: - assert(0); - } - pos = read_int32(f); - } + jl_set_t_uid_ctr(uid_ctr); + jl_set_gs_ctr(gs_ctr); //jl_printf(JL_STDERR, "backref_list.len = %d\n", backref_list.len); arraylist_free(&backref_list); @@ -1672,21 +1756,6 @@ DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) JL_SIGATOMIC_END(); } -void jl_init_restored_modules() -{ - if (jl_module_init_order != NULL) { - jl_array_t *temp = jl_module_init_order; - jl_module_init_order = NULL; - JL_GC_PUSH1(&temp); - int i; - for(i=0; i < jl_array_len(temp); i++) { - jl_value_t *mod = jl_cellref(temp, i); - jl_module_run_initializer((jl_module_t*)mod); - } - JL_GC_POP(); - } -} - DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) { if (jl_is_expr(ast)) @@ -1772,17 +1841,18 @@ DLLEXPORT jl_value_t *jl_uncompress_ast(jl_lambda_info_t *li, jl_value_t *data) return v; } -DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) +DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) { ios_t f; if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) { jl_printf(JL_STDERR, "Cannot open cache file \"%s\" for writing.\n", fname); return 1; } + serializer_worklist = worklist; + jl_serialize_mod_list(&f); // this can throw, keep it early (before any actual initialization) + JL_SIGATOMIC_BEGIN(); - jl_module_t *lastmod = jl_current_module; - jl_current_module = mod; - jl_serialize_mod_list(&f); + arraylist_new(&reinit_list, 0); htable_new(&backref_table, 5000); ptrhash_put(&backref_table, jl_main_module, (void*)(uintptr_t)0); backref_table_numel = 1; @@ -1790,29 +1860,20 @@ DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; - jl_serialize_value(&f, mod->parent); - jl_serialize_value(&f, mod->name); - jl_serialize_value(&f, mod); + jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; - mode = MODE_MODULE_LAMBDAS; - jl_serialize_lambdas_from_mod(&f, jl_main_module); - jl_serialize_value(&f, NULL); + jl_serialize_value(&f, worklist); - // save module initialization order - if (jl_module_init_order != NULL) { - size_t i; - for(i=0; i < jl_array_len(jl_module_init_order); i++) { - // verify that all these modules were saved - assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); - } - } - jl_serialize_value(&f, jl_module_init_order); + mode = MODE_MODULE_POSTWORK; + jl_serialize_lambdas_from_mod(&f, jl_main_module); + jl_serialize_value(&f, NULL); // signal end of lambdas + jl_finalize_serializer(&f); - jl_current_module = lastmod; mode = last_mode; jl_gc_enable(en); htable_reset(&backref_table, 0); + arraylist_free(&reinit_list); ios_close(&f); JL_SIGATOMIC_END(); @@ -1822,7 +1883,7 @@ DLLEXPORT int jl_save_new_module(const char *fname, jl_module_t *mod) jl_function_t *jl_method_cache_insert(jl_methtable_t *mt, jl_tupletype_t *type, jl_function_t *method); -static jl_module_t *_jl_restore_new_module(ios_t *f) +static jl_array_t *_jl_restore_incremental(ios_t *f) { if (ios_eof(f)) { ios_close(f); @@ -1841,21 +1902,8 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; - jl_module_t *parent = (jl_module_t*)jl_deserialize_value(f, NULL); - jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(f, NULL); - jl_exception_in_transit = NULL; - jl_binding_t *b; - JL_TRY { - b = jl_get_binding_wr(parent, name); - jl_declare_constant(b); // this can throw - if (b->value != NULL) { - jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); - } - } - JL_CATCH { - goto cleanup; - } - b->value = jl_deserialize_value(f, &b->value); + jl_array_t *restored = NULL; + restored = (jl_array_t*)jl_deserialize_value(f, (jl_value_t**)&restored); size_t i = 0; while (i < flagref_list.len) { @@ -1916,11 +1964,13 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) } } - mode = MODE_MODULE_LAMBDAS; - jl_deserialize_lambdas_from_mod(f); - - jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); + // at this point, the AST is fully reconstructed, but still completely disconnected + // in postwork mode, all of the interconnects will be created + mode = MODE_MODULE_POSTWORK; + jl_deserialize_lambdas_from_mod(f); // hook up methods of external generic functions + jl_finalize_deserializer(f); // done with f + // Resort the internal method tables for (i = 0; i < methtable_list.len; i++) { jl_methtable_t *mt = (jl_methtable_t*)methtable_list.items[i]; jl_array_t *cache_targ = mt->cache_targ; @@ -1951,7 +2001,6 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) } } -cleanup: mode = last_mode; jl_gc_enable(en); arraylist_free(&flagref_list); @@ -1959,28 +2008,28 @@ static jl_module_t *_jl_restore_new_module(ios_t *f) arraylist_free(&backref_list); ios_close(f); JL_SIGATOMIC_END(); - if (jl_exception_in_transit) jl_rethrow(); jl_init_restored_modules(); + jl_module_init_order = NULL; - return (jl_module_t*)b->value; + return restored; } -DLLEXPORT jl_module_t *jl_restore_new_module_from_buf(const char *buf, size_t sz) +DLLEXPORT jl_array_t *jl_restore_incremental_from_buf(const char *buf, size_t sz) { ios_t f; ios_static_buffer(&f, (char*)buf, sz); - return _jl_restore_new_module(&f); + return _jl_restore_incremental(&f); } -DLLEXPORT jl_module_t *jl_restore_new_module(const char *fname) +DLLEXPORT jl_array_t *jl_restore_incremental(const char *fname) { ios_t f; if (ios_file(&f, fname, 1, 0, 0, 0) == NULL) { jl_printf(JL_STDERR, "Cache file \"%s\" not found\n", fname); return NULL; } - return _jl_restore_new_module(&f); + return _jl_restore_incremental(&f); } // --- init --- diff --git a/src/toplevel.c b/src/toplevel.c index 8fedd64f163ad..f09c103f0c4b3 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -70,7 +70,7 @@ jl_module_t *jl_new_main_module(void) return old_main; } -jl_array_t *jl_module_init_order = NULL; +extern jl_array_t *jl_module_init_order; // load time init procedure: in build mode, only record order void jl_module_load_time_initialize(jl_module_t *m) From ef3ec3d31a378566481a05d6e0f707fdf78312ae Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 8 Jul 2015 15:33:58 -0400 Subject: [PATCH 08/28] reimplement require/include to better support precompilation the codepaths through these methods were not particularly amenable to adding incremental precompilation. the problem is that the cache-load code paths need to be mirrors of their non-cache counterparts. this minimizes the code paths to running "include" to one, and similarly for "reload"/"require" those are now limited to only handling module names (symbols), rather than arbitrary paths this also fixes a number of issues with the incremental serializer that have snuck in over time. --- base/client.jl | 4 +- base/docs/basedocs.jl | 2 +- base/exports.jl | 1 - base/interactiveutil.jl | 1 - base/loading.jl | 291 ++++++++++++++++++++++------------------ base/options.jl | 1 + base/precompile.jl | 7 +- base/sysimg.jl | 3 +- src/dump.c | 238 ++++++++++++++++---------------- src/init.c | 53 +++++--- src/julia.h | 4 +- src/julia_internal.h | 2 + src/toplevel.c | 10 +- test/choosetests.jl | 2 +- test/core.jl | 2 - test/loading.jl | 29 ++++ test/parallel.jl | 2 +- test/runtests.jl | 24 ++-- test/test_sourcepath.jl | 11 +- test/testdefs.jl | 4 +- ui/repl.c | 17 ++- 21 files changed, 392 insertions(+), 316 deletions(-) create mode 100644 test/loading.jl diff --git a/base/client.jl b/base/client.jl index 38b486783a06e..90ea29a9aa633 100644 --- a/base/client.jl +++ b/base/client.jl @@ -274,7 +274,9 @@ let reqarg = Set(UTF8String["--home", "-H", end # load file immediately on all processors if opts.load != C_NULL - require(bytestring(opts.load)) + let file = bytestring(opts.load), cwd = pwd() + @everywhere include($file, (1, $cwd)) + end end # eval expression if opts.eval != C_NULL diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 999c106205664..59aaccd42729f 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -368,7 +368,7 @@ function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. """ -include_from_node1 +base_include """ 0 (zero; BrE: `/ˈzɪərəʊ/` or AmE: `/ˈziːroʊ/`) is both a number and the numerical digit used to represent that number in numerals. It fulfills a central role in mathematics as the additive identity of the integers, real numbers, and many other algebraic structures. As a digit, 0 is used as a placeholder in place value systems. Names for the number 0 in English include zero, nought or (US) naught (`/ˈnɔːt/`), nil, or — in contexts where at least one adjacent digit distinguishes it from the letter "O" — oh or o (`/ˈoʊ/`). Informal or slang terms for zero include zilch and zip. Ought and aught (/ˈɔːt/), as well as cipher, have also been used historically. diff --git a/base/exports.jl b/base/exports.jl index 1bee26066a743..dff4317e1548b 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1097,7 +1097,6 @@ export include, include_string, reload, - require, # RTS internals finalizer, diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 3f4e70aeeed54..af7282e07187d 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -385,7 +385,6 @@ function workspace() Expr(:toplevel, :(const Base = $(Expr(:quote, b))), :(const LastMain = $(Expr(:quote, last))))) - empty!(package_list) empty!(package_locks) nothing end diff --git a/base/loading.jl b/base/loading.jl index 7b6d7b62a7b23..3f3fa4de84706 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -# require +# Base.require is the implementation for the `import` statement function find_in_path(name::AbstractString) isabspath(name) && return name @@ -23,8 +23,8 @@ function find_in_path(name::AbstractString) return nothing end -find_in_node1_path(name) = myid()==1 ? - find_in_path(name) : remotecall_fetch(1, find_in_path, name) +find_in_node_path(name, node::Int=1) = myid() == node ? + find_in_path(name) : remotecall_fetch(node, find_in_path, name) function find_source_file(file) (isabspath(file) || isfile(file)) && return file @@ -34,112 +34,118 @@ function find_source_file(file) isfile(file2) ? file2 : nothing end -# Store list of files and their load time -package_list = Dict{ByteString,Float64}() -# to synchronize multiple tasks trying to require something -package_locks = Dict{ByteString,Any}() -require(f::AbstractString, fs::AbstractString...) = (require(f); for x in fs require(x); end) - -# only broadcast top-level (not nested) requires and reloads -toplevel_load = true - -function _require_from_serialized(content) - m = ccall(:jl_restore_new_module_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content)) - #package_list[path] = time() - return m -end - -function _require_from_cache(name::AbstractString) +function find_in_cache_path(mod::Symbol) + name = string(mod) for prefix in LOAD_CACHE_PATH path = joinpath(prefix, name*".ji") if isfile(path) - if nprocs() == 1 - if ccall(:jl_restore_new_module, UInt, (Ptr{Uint8},), path) != 0 - #package_list[path] = time() - return true - end - else - content = open(readbytes, path) - if _require_from_serialized(content) != 0 - refs = Any[ @spawnat p _require_from_serialized(content) for p in filter(x->x!=1, procs()) ] - for r in refs; wait(r); end - return true - end - end + produce(path) end end - return false + nothing end -function require(sname::Symbol) - name = string(sname) - if !_require_from_cache(name) - require(name) - end +function _include_from_serialized(content::Vector{UInt8}) + m = ccall(:jl_restore_incremental_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content)) + return m end -function require(name::AbstractString) - path = find_in_node1_path(name) - path == nothing && throw(ArgumentError("$name not found in path")) - - if myid() == 1 && toplevel_load - refs = Any[ @spawnat p _require(path) for p in filter(x->x!=1, procs()) ] - _require(path) - for r in refs; wait(r); end +function _require_from_serialized(node::Int, path_to_try::ByteString) + if myid() == 1 && current_module() === Main && nprocs() > 1 + # broadcast top-level import/using from node 1 (only) + if node == myid() + content = open(readbytes, path_to_try) + else + content = remotecall_fetch(node, open, readbytes, path_to_try) + end + if _include_from_serialized(content) + others = filter(x -> x != myid(), procs()) + refs = Any[ @spawnat p _include_from_serialized(content) for p in others] + successes = Any[ fetch(ref) for ref in refs ] + for (id, ref) in zip(others, refs) + if fetch(ref) != 0 + warn("node state is inconsistent: node $i failed to load serialized cache from :node $id:$path_to_try.") + end + end + return true + end + elseif node == myid() + if ccall(:jl_restore_incremental, UInt, (Ptr{Uint8},), path_to_try) != 0 + return true + end else - _require(path) + content = remotecall_fetch(node, open, readbytes, path_to_try) + if _include_from_serialized(content) + return true + end end - nothing + # otherwise, continue search + return false end -function _require(path) - global toplevel_load - if haskey(package_list,path) - loaded, c = package_locks[path] - !loaded && wait(c) - else - last = toplevel_load - toplevel_load = false - try - reload_path(path) - finally - toplevel_load = last +function _require_from_serialized(node::Int, mod::Symbol) + name = string(mod) + finder = @spawnat node @task find_in_cache_path(mod) # TODO: switch this to an explicit Channel + while true + path_to_try = remotecall_fetch(node, finder->consume(fetch(finder)), finder) + path_to_try === nothing && return false + if _require_from_serialized(node, path_to_try) + return true end end end -function reload(name::AbstractString) - global toplevel_load - path = find_in_node1_path(name) - path == nothing && throw(ArgumentError("$name not found in path")) - refs = nothing - if myid() == 1 && toplevel_load - refs = Any[ @spawnat p reload_path(path) for p in filter(x->x!=1, procs()) ] +# to synchronize multiple tasks trying to import/using something +package_locks = Dict{Symbol,Condition}() + +# setting cachecompile=true will force a compile of a cache file +# require always works in Main scope and loads files from node 1 +function require(mod::Symbol, cachecompile::Bool=false) + loading = get(package_locks, mod, false) + if loading !== false + # load already in progress for this module + wait(loading) + return end - last = toplevel_load - toplevel_load = false + package_locks[mod] = Condition() + try - reload_path(path) + if !cachecompile && _require_from_serialized(1, mod) + return + end + + name = string(mod) + path = find_in_node_path(name, 1) + path === nothing && throw(ArgumentError("$name not found in path")) + + if cachecompile || JLOptions().incremental != 0 + # spawn off a new incremental compile task from node 1 for recursive `require` calls + cachefile = remotecall_fetch(1, create_expr_cache, path, name) + if !_require_from_serialized(1, cachefile) + error("Warning: require failed to create a precompiled cache file") + end + return + end + + if myid() == 1 && current_module() === Main && nprocs() > 1 + # broadcast top-level import/using from node 1 (only) + content = open(readall, path) + refs = Any[ @spawnat p eval(Main, :(Base.base_include($content, $path, (1, "")))) for p in procs() ] + for r in refs; wait(r); end + else + eval(Main, :(Base.base_include($path, (1, "")))) + end finally - toplevel_load = last - end - if refs !== nothing - for r in refs; wait(r); end + loading = pop!(package_locks, mod) + notify(loading, all=true) end nothing end +const reload = require # remote/parallel load -include_string(txt::ByteString, fname::ByteString) = - ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t), - txt, sizeof(txt), fname, sizeof(fname)) - -include_string(txt::AbstractString, fname::AbstractString) = include_string(bytestring(txt), bytestring(fname)) - -include_string(txt::AbstractString) = include_string(txt, "string") - -function source_path(default::Union{AbstractString,Void}="") +function source_path(default::Union{AbstractString,Void}=nothing) t = current_task() while true s = t.storage @@ -153,82 +159,105 @@ function source_path(default::Union{AbstractString,Void}="") end end -macro __FILE__() source_path() end +macro __FILE__() + sp = source_path() + if !isa(sp, Tuple{Int,Any}) + return ":unknown:\0$sp" + elseif sp[1] === 0 + return ":string:\0$(sp[2])" + elseif sp[1] !== myid() + return ":node $(sp[1]):\0$(sp[2])" + else + return sp[2] + end +end -function include_from_node1(path::AbstractString) +# base_include is the implementation for Base.include +# if relative_to::Void, it uses the working directory and node of the previous (recursive) include to find the file +# relative_to is a tuple of (node, cwd), where node can be 0 iff the content is from a string (sans a real path) +function base_include(content::Nullable{ByteString}, path::AbstractString, relative_to=nothing) prev = source_path(nothing) - path = (prev == nothing) ? abspath(path) : joinpath(dirname(prev),path) - tls = task_local_storage() - tls[:SOURCE_PATH] = path - local result - try - if myid()==1 - # sleep a bit to process file requests from other nodes - nprocs()>1 && sleep(0.005) - result = Core.include(path) - nprocs()>1 && sleep(0.005) + local bytepath::ByteString, node::Int + if !isnull(content) + # already have content, so no need for file-system operations + if relative_to === nothing + node = 0 + bytepath = bytestring(path) else - result = include_string(remotecall_fetch(1, readall, path), path) + node = relative_to[1] + bytepath = bytestring(joinpath(relative_to[2],path)) end - finally - if prev == nothing - delete!(tls, :SOURCE_PATH) + else + if prev !== nothing && relative_to === nothing + # if this is a recursive include without explicit relative_to location + # pick up previous location for last include + relative_to = (prev[1], dirname(prev[2])) + end + if relative_to === nothing || relative_to[1] === 0 + # no previous path, pick up location from host + node = myid() + bytepath = bytestring(abspath(path)) else - tls[:SOURCE_PATH] = prev + node = relative_to[1] + bytepath = bytestring(joinpath(relative_to[2],path)) end end - result -end -function reload_path(path::AbstractString) - had = haskey(package_list, path) - if !had - package_locks[path] = (false, Condition()) - end - package_list[path] = time() tls = task_local_storage() - prev = pop!(tls, :SOURCE_PATH, nothing) + tls[:SOURCE_PATH] = (node, bytepath) try - eval(Main, :(Base.include_from_node1($path))) - catch e - had || delete!(package_list, path) - rethrow(e) + if isnull(content) + if myid() === node + # sleep a bit to process requests from other nodes + nprocs()>1 && yield() + result = Core.include(bytepath) + nprocs()>1 && yield() + return result + else + content = Nullable{ByteString}(remotecall_fetch(node, readall, bytepath)) + # fall-through now that content is assigned + end + end + txt = get(content) + return ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t), + txt, sizeof(txt), bytepath, sizeof(bytepath)) finally - if prev !== nothing + if prev === nothing + delete!(tls, :SOURCE_PATH) + else tls[:SOURCE_PATH] = prev end end - reloaded, c = package_locks[path] - if !reloaded - package_locks[path] = (true, c) - notify(c, all=true) - end - nothing end +base_include(path::AbstractString, relative_to=nothing) = base_include(Nullable{ByteString}(), path, relative_to) +include_string(txt::AbstractString, fname::AbstractString="none") = base_include(Nullable{ByteString}(bytestring(txt)), bytestring(fname)) function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[]) return eval(Module(:__anon__), Expr(:toplevel, :(const ARGS = $args), - :(eval(x) = Core.eval(__anon__,x)), - :(eval(m,x) = Core.eval(m,x)), - :(include($path)))) + :(eval(x) = Main.Core.eval(__anon__,x)), + :(eval(m,x) = Main.Core.eval(m,x)), + :(Main.Base.include($path)))) end evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...]) -function create_expr_cache(m::Expr) - assert(m.head === :module) - typeassert(m.args[2], Symbol) +function create_expr_cache(m::Expr, name) cachepath = LOAD_CACHE_PATH[1] if !isdir(cachepath) mkdir(cachepath) end + cachefile = abspath(cachepath, name*".ji") code_object = """ while !eof(STDIN) eval(Main, deserialize(STDIN)) end """ - io, pobj = open(detach(setenv(`$(julia_cmd()) --build $cachepath --startup-file=no --history-file=no -E $code_object`,["JULIA_HOME=$JULIA_HOME","HOME=$(homedir())"])), "w") + io, pobj = open(detach(setenv(`$(julia_cmd()) + --output-ji $cachefile --output-incremental=yes + --startup-file=no --history-file=no + --eval $code_object`, + ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w") serialize(io, quote empty!(Base.LOAD_PATH) append!(Base.LOAD_PATH, $LOAD_PATH) @@ -251,11 +280,9 @@ function create_expr_cache(m::Expr) end close(io) wait(pobj) - if !_require_from_cache(string(m.args[2])) - error("Warning: @cacheable module failed to define a module") - end + return cachefile end -macro cacheable(m) - :(create_expr_cache($(QuoteNode(m)))) +function create_expr_cache(file::AbstractString, name) + return create_expr_cache(:(Base.include($(abspath(file)))), name) end diff --git a/base/options.jl b/base/options.jl index ca1810f4c813b..fd4f7c3211b38 100644 --- a/base/options.jl +++ b/base/options.jl @@ -32,6 +32,7 @@ immutable JLOptions outputbc::Ptr{UInt8} outputo::Ptr{UInt8} outputji::Ptr{UInt8} + incremental::Int8 end JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions)) diff --git a/base/precompile.jl b/base/precompile.jl index f0b913e297daf..acb84f1808248 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -162,7 +162,6 @@ precompile(Base._deleteat_beg!, (Array{UInt8, 1}, Int, Int)) precompile(Base._deleteat_end!, (Array{UInt8, 1}, Int, Int)) precompile(Base._growat_beg!, (Array{UInt8, 1}, Int, Int)) precompile(Base._growat_end!, (Array{UInt8, 1}, Int, Int)) -precompile(Base._require, (ASCIIString,)) precompile(Base._setindex!, (Base.Dict{Symbol, Any}, Base.LineEdit.Prompt, Symbol, Int)) precompile(Base._setindex!, (Dict{Any, Any}, Base.LineEdit.PromptState, Base.LineEdit.Prompt, Int)) precompile(Base._setindex!, (Dict{Any, Any}, Bool, WeakRef, Int)) @@ -252,7 +251,8 @@ precompile(Base.ht_keyindex2, (Dict{UInt8, Any}, UInt8)) precompile(Base.in, (Char, ASCIIString)) precompile(Base.in, (Int, Base.UnitRange{Int})) precompile(Base.in, (UInt8, Base.KeyIterator{Dict{UInt8, Any}})) -precompile(Base.include_from_node1, (ASCIIString,)) +precompile(Base.base_include, (ASCIIString,)) +precompile(Base.base_include, (UTF8String,)) precompile(Base.init_stdio, (Ptr{Void},)) precompile(Base.input_color, ()) precompile(Base.insert!, (Array{Any,1}, Int, Base.GlobalRef)) @@ -351,10 +351,9 @@ precompile(Base.readuntil, (IOBuffer, UInt8)) precompile(Base.rehash!, (Dict{Any,Any}, Int)) precompile(Base.rehash!, (Dict{UInt8, Any}, Int)) precompile(Base.reinit_stdio, ()) -precompile(Base.reload_path, (ASCIIString,)) precompile(Base.repeat, (ASCIIString, Int)) precompile(Base.repl_cmd, (Cmd,)) -precompile(Base.require, (ASCIIString,)) +precompile(Base.require, (Symbol,)) precompile(Base.rr2id, (RemoteRef,)) precompile(Base.rsearch, (ASCIIString, Char)) precompile(Base.rstrip, (ASCIIString,)) diff --git a/base/sysimg.jl b/base/sysimg.jl index 71fea54c4a0de..9fd1bc1e15860 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -314,10 +314,9 @@ function __init__() init_parallel() end +include = base_include include("precompile.jl") -include = include_from_node1 - end # baremodule Base using Base diff --git a/src/dump.c b/src/dump.c index 8fab923b9fc97..9cb3d657b1cfe 100644 --- a/src/dump.c +++ b/src/dump.c @@ -25,6 +25,9 @@ extern "C" { #endif +// TODO: put WeakRefs on the weak_refs list during deserialization +// TODO: handle finalizers + // hash of definitions for predefined tagged object static htable_t ser_tag; // array of definitions for the predefined tagged object types @@ -86,7 +89,7 @@ static const ptrint_t LongSvec_tag = 24; static const ptrint_t LongExpr_tag = 25; static const ptrint_t LiteralVal_tag = 26; static const ptrint_t SmallInt64_tag = 27; -static const ptrint_t UNUSED_tag = 28; +static const ptrint_t SmallDataType_tag= 28; static const ptrint_t Int32_tag = 29; static const ptrint_t Array1d_tag = 30; static const ptrint_t Singleton_tag = 31; @@ -217,7 +220,7 @@ static int jl_load_sysimg_so() if (jl_sysimg_handle == 0) return -1; - int imaging_mode = jl_generating_output(); + int imaging_mode = jl_generating_output() && !jl_options.incremental; // in --build mode only use sysimg data, not precompiled native code if (!imaging_mode && jl_options.use_precompiled==JL_OPTIONS_USE_PRECOMPILED_YES) { sysimg_gvars = (jl_value_t***)jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars"); @@ -411,8 +414,12 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) { int tag = 0; if (mode == MODE_MODULE_POSTWORK) { - if (dt->uid != 0) - tag = 6; // must use apply_type + if (dt->uid != 0) { + if (dt->name->primary == (jl_value_t*)dt) + tag = 6; // primary type + else + tag = 7; // must use apply_type + } } else if (mode == MODE_MODULE) { int internal = module_in_worklist(dt->name->module); @@ -447,7 +454,8 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) tag = 3; else if (dt == jl_int64_type) tag = 4; - writetag(s, (jl_value_t*)jl_datatype_type); + writetag(s, (jl_value_t*)SmallDataType_tag); + write_uint8(s, 0); // virtual size jl_serialize_value(s, (jl_value_t*)jl_datatype_type); write_uint8(s, tag); if (tag == 6) { @@ -604,11 +612,8 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) return; } ptrint_t pos = backref_table_numel++; - if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) - pos <<= 1; - ptrhash_put(&backref_table, v, (void*)pos); if (jl_typeof(v) == jl_idtable_type) { - // will need to rehash this, later (after type is fully constructed) + // will need to rehash this, later (after types are fully constructed) arraylist_push(&reinit_list, (void*)pos); arraylist_push(&reinit_list, (void*)1); } @@ -620,6 +625,9 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) arraylist_push(&reinit_list, (void*)2); } } + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) + pos <<= 1; + ptrhash_put(&backref_table, v, (void*)pos); } size_t i; @@ -671,9 +679,9 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) write_uint16(s, ar->ndims); write_uint16(s, (ar->ptrarray<<15) | (ar->elsize & 0x7fff)); } - jl_serialize_value(s, jl_typeof(ar)); for (i=0; i < ar->ndims; i++) jl_serialize_value(s, jl_box_long(jl_array_dim(ar,i))); + jl_serialize_value(s, jl_typeof(ar)); if (!ar->ptrarray) { size_t tot = jl_array_len(ar) * ar->elsize; ios_write(s, (char*)jl_array_data(ar), tot); @@ -683,9 +691,6 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, jl_cellref(v, i)); } } - if (mode == MODE_MODULE) { - jl_serialize_value(s, jl_typeof(ar)); - } } else if (jl_is_expr(v)) { jl_expr_t *e = (jl_expr_t*)v; @@ -792,7 +797,14 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) writetag(s, (jl_value_t*)Singleton_tag); return; } - writetag(s, (jl_value_t*)jl_datatype_type); + if (t->size <= 255) { + writetag(s, (jl_value_t*)SmallDataType_tag); + write_uint8(s, t->size); + } + else { + writetag(s, (jl_value_t*)jl_datatype_type); + write_int32(s, t->size); + } jl_serialize_value(s, t); if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && t == jl_typename_type) { if (module_in_worklist(((jl_typename_t*)v)->module)) { @@ -823,9 +835,6 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) jl_serialize_value(s, jl_get_nth_field(v, i)); } } - if (mode == MODE_MODULE) { - jl_serialize_value(s, t); - } } } } @@ -839,35 +848,22 @@ static void jl_serialize_methtable_from_mod(ios_t *s, jl_module_t *m, jl_sym_t * mt = jl_gf_mtable(mt->kwsorter); assert(!mt->kwsorter); } - //XXX: we are reversing the list of methods due to #8652 - struct _chain { - jl_methlist_t *ml; - struct _chain *next; - } *chain = NULL; jl_methlist_t *ml = mt->defs; while (ml != (void*)jl_nothing) { if (module_in_worklist(ml->func->linfo->module)) { - struct _chain *link = (struct _chain*)alloca(sizeof(struct _chain)); - link->ml = ml; - link->next = chain; - chain = link; + jl_serialize_value(s, m); + jl_serialize_value(s, name); + write_int8(s, iskw); + jl_serialize_value(s, ml->sig); + jl_serialize_value(s, ml->func); + if (jl_is_svec(ml->tvars)) + jl_serialize_value(s, ml->tvars); + else + jl_serialize_value(s, jl_svec1(ml->tvars)); + write_int8(s, ml->isstaged); } ml = ml->next; } - while (chain) { - ml = chain->ml; - jl_serialize_value(s, m); - jl_serialize_value(s, name); - write_int8(s, iskw); - jl_serialize_value(s, ml->sig); - jl_serialize_value(s, ml->func); - if (jl_is_svec(ml->tvars)) - jl_serialize_value(s, ml->tvars); - else - jl_serialize_value(s, jl_svec1(ml->tvars)); - write_int8(s, ml->isstaged); - chain = chain->next; - } } static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) @@ -913,7 +909,7 @@ void jl_serialize_mod_list(ios_t *s) if (b->owner == m && b->value && b->constp && jl_is_module(b->value) && - module_in_worklist((jl_module_t*)b->value)) { + !module_in_worklist((jl_module_t*)b->value)) { jl_module_t *child = (jl_module_t*)b->value; if (child->name == b->name) { // this is the original/primary binding for the submodule @@ -966,6 +962,8 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) dt = jl_int64_type; else dt = jl_new_uninitialized_datatype(nf); + assert(tree_literal_values==NULL && mode != MODE_AST); + backref_list.items[pos] = dt; dt->size = size; dt->struct_decl = NULL; dt->instance = NULL; @@ -983,18 +981,16 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) } int has_instance = (flags>>3)&1; if (has_instance) { + assert(mode != MODE_MODULE_POSTWORK); // there shouldn't be an instance on a type with uid = 0 dt->instance = jl_deserialize_value(s, &dt->instance); jl_set_typeof(dt->instance, dt); + jl_gc_wb(dt, dt->instance); } - assert(tree_literal_values==NULL && mode != MODE_AST); - backref_list.items[pos] = dt; if (tag == 5) { + assert(pos > 0); + assert(mode != MODE_MODULE_POSTWORK); arraylist_push(&flagref_list, loc); arraylist_push(&flagref_list, (void*)(uptrint_t)pos); - if (has_instance) { - arraylist_push(&flagref_list, &jl_astaggedvalue(dt->instance)->type); - arraylist_push(&flagref_list, (void*)(uptrint_t)-1); - } } if (nf > 0) { @@ -1136,13 +1132,14 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t int pos = backref_list.len; if (usetable) arraylist_push(&backref_list, NULL); - jl_value_t *aty = jl_deserialize_value(s, NULL); size_t *dims = (size_t*)alloca(ndims*sizeof(size_t)); for(i=0; i < ndims; i++) dims[i] = jl_unbox_long(jl_deserialize_value(s, NULL)); - jl_array_t *a = jl_new_array_for_deserialization((jl_value_t*)aty, ndims, dims, isunboxed, elsize); + jl_array_t *a = jl_new_array_for_deserialization((jl_value_t*)NULL, ndims, dims, isunboxed, elsize); if (usetable) backref_list.items[pos] = a; + jl_value_t *aty = jl_deserialize_value(s, &jl_astaggedvalue(a)->type); + jl_set_typeof(a, aty); if (!a->ptrarray) { size_t tot = jl_array_len(a) * a->elsize; ios_read(s, (char*)jl_array_data(a), tot); @@ -1154,10 +1151,6 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t if (data[i]) jl_gc_wb(a, data[i]); } } - if (mode == MODE_MODULE) { - aty = jl_deserialize_value(s, &jl_astaggedvalue(a)->type); - assert(aty == jl_typeof(a)); - } return (jl_value_t*)a; } else if (vtag == (jl_value_t*)jl_expr_type || @@ -1243,6 +1236,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t li->inInference = 0; li->inCompile = 0; li->unspecialized = (jl_function_t*)jl_deserialize_value(s, (jl_value_t**)&li->unspecialized); + if (li->unspecialized) jl_gc_wb(li, li->unspecialized); li->functionID = 0; li->specFunctionID = 0; int32_t cfunc_llvm, func_llvm; @@ -1324,15 +1318,23 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_value_t *sym = jl_deserialize_value(s, NULL); return jl_module_globalref(tree_enclosing_module, (jl_sym_t*)sym); } - else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)jl_globalref_type) { + else if (vtag == (jl_value_t*)jl_globalref_type) { + jl_value_t *v = jl_new_struct_uninit(jl_globalref_type); + if (usetable) + arraylist_push(&backref_list, v); + jl_value_t* *data = jl_data_ptr(v); + data[0] = jl_deserialize_value(s, &data[0]); + data[1] = jl_deserialize_value(s, &data[1]); + return v; + } + else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { + int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s) : read_int32(s)); + jl_value_t *v = jl_gc_allocobj(sz); int pos = backref_list.len; if (usetable) - arraylist_push(&backref_list, NULL); - jl_datatype_t *dt; - if (vtag == (jl_value_t*)jl_globalref_type) - dt = jl_globalref_type; - else - dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); + arraylist_push(&backref_list, v); + jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type); + jl_set_typeof(v, dt); if (dt == jl_datatype_type) return jl_deserialize_datatype(s, pos, loc); if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && dt == jl_typename_type) { @@ -1347,34 +1349,12 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t backref_list.items[pos] = v; return v; } + assert(mode != MODE_MODULE_POSTWORK); } size_t nf = jl_datatype_nfields(dt); - jl_value_t *v; if (nf == 0 && jl_datatype_size(dt)>0) { int nby = jl_datatype_size(dt); - char *data = (char*)alloca(nby); - ios_read(s, data, nby); - v = NULL; - if (dt == jl_int32_type) - v = jl_box_int32(*(int32_t*)data); - else if (dt == jl_int64_type) - v = jl_box_int64(*(int64_t*)data); - else if (dt == jl_bool_type) - v = jl_box_bool(*(int8_t*)data); - else { - switch (nby) { - case 1: v = jl_box8 (dt, *(int8_t *)data); break; - case 2: v = jl_box16(dt, *(int16_t*)data); break; - case 4: v = jl_box32(dt, *(int32_t*)data); break; - case 8: v = jl_box64(dt, *(int64_t*)data); break; - default: - v = (jl_value_t*)jl_gc_allocobj(nby); - jl_set_typeof(v, dt); - memcpy(jl_data_ptr(v), data, nby); - } - } - if (usetable) - backref_list.items[pos] = v; + ios_read(s, (char*)jl_data_ptr(v), nby); } else { if (mode == MODE_AST && dt == jl_globalref_type) { @@ -1382,21 +1362,22 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t jl_value_t *var = jl_deserialize_value(s, NULL); return jl_module_globalref((jl_module_t*)mod, (jl_sym_t*)var); } - v = jl_new_struct_uninit(dt); - if (usetable) - backref_list.items[pos] = v; char *data = (char*)jl_data_ptr(v); for(i=0; i < nf; i++) { - jl_set_nth_field(v, i, jl_deserialize_value(s, - (dt->fields[i].isptr) ? (jl_value_t**)(data+jl_field_offset(dt, i)) : NULL)); + if (dt->fields[i].isptr) { + jl_value_t **fld = (jl_value_t**)(data+jl_field_offset(dt, i)); + *fld = jl_deserialize_value(s, fld); + } + else { + jl_set_nth_field(v, i, jl_deserialize_value(s, NULL)); + } + } + if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK)) { + if (jl_is_mtable(v)) + arraylist_push(&methtable_list, v); // will resort this table, later + if (dt == jl_typename_type) + ((jl_typename_t*)v)->uid = jl_assign_type_uid(); // make sure this has a new uid } - if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) && jl_is_mtable(v)) - arraylist_push(&methtable_list, v); - } - // TODO: put WeakRefs on the weak_refs list - if (mode == MODE_MODULE) { - dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type); - assert((jl_value_t*)dt == jl_typeof(v)); } return v; } @@ -1429,10 +1410,12 @@ void jl_deserialize_lambdas_from_mod(ios_t *s) int8_t iskw = read_int8(s); assert(jl_is_gf(gf)); if (iskw) { - if (!jl_gf_mtable(gf)->kwsorter) { - jl_gf_mtable(gf)->kwsorter = jl_new_generic_function(jl_gf_name(gf)); + jl_methtable_t* mt = jl_gf_mtable(gf); + if (!mt->kwsorter) { + mt->kwsorter = jl_new_generic_function(jl_gf_name(gf)); + jl_gc_wb(mt, mt->kwsorter); } - gf = jl_gf_mtable(gf)->kwsorter; + gf = mt->kwsorter; assert(jl_is_gf(gf)); } jl_tupletype_t *types = (jl_tupletype_t*)jl_deserialize_value(s, NULL); @@ -1499,7 +1482,8 @@ static void jl_finalize_serializer(ios_t *f) { assert(ptrhash_get(&backref_table, jl_cellref(jl_module_init_order, i)) != HT_NOTFOUND); } } - jl_serialize_value(f, jl_module_init_order); + if (mode != MODE_MODULE) + jl_serialize_value(f, jl_module_init_order); // record list of reinitialization functions l = reinit_list.len; @@ -1520,12 +1504,11 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) { break; } case 2: { // reinsert v (const) into parent as sym - jl_module_t *parent = (jl_module_t*)jl_deserialize_value(f, NULL); - jl_sym_t *sym = (jl_sym_t*)jl_deserialize_value(f, NULL); - jl_binding_t *b = jl_get_binding_wr(parent, sym); + jl_module_t *mod = (jl_module_t*)v; + jl_binding_t *b = jl_get_binding_wr(mod->parent, mod->name); jl_declare_constant(b); // this can throw if (b->value != NULL) { - jl_printf(JL_STDERR, "Warning: replacing module %s\n", sym->name); + jl_printf(JL_STDERR, "Warning: replacing module %s\n", mod->name->name); } b->value = v; jl_gc_wb_binding(b, v); @@ -1544,7 +1527,8 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) { } } static void jl_finalize_deserializer(ios_t *f) { - jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); + if (mode != MODE_MODULE) + jl_module_init_order = (jl_array_t*)jl_deserialize_value(f, NULL); // run reinitialization functions int pos = read_int32(f); @@ -1602,7 +1586,7 @@ void jl_save_system_image_to_stream(ios_t *f) write_int32(f, jl_get_t_uid_ctr()); write_int32(f, jl_get_gs_ctr()); - jl_finalize_serializer(f); + jl_finalize_serializer(f); // done with f htable_reset(&backref_table, 0); arraylist_free(&reinit_list); @@ -1763,8 +1747,10 @@ DLLEXPORT jl_value_t *jl_ast_rettype(jl_lambda_info_t *li, jl_value_t *ast) JL_SIGATOMIC_BEGIN(); DUMP_MODES last_mode = mode; mode = MODE_AST; - if (li->module->constant_table == NULL) + if (li->module->constant_table == NULL) { li->module->constant_table = jl_alloc_cell_1d(0); + jl_gc_wb(li->module, li->module->constant_table); + } tree_literal_values = li->module->constant_table; ios_t src; jl_array_t *bytes = (jl_array_t*)ast; @@ -1856,18 +1842,19 @@ DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) htable_new(&backref_table, 5000); ptrhash_put(&backref_table, jl_main_module, (void*)(uintptr_t)0); backref_table_numel = 1; + jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; - jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; - jl_serialize_value(&f, worklist); + jl_finalize_serializer(&f); // done with MODE_MODULE + reinit_list.len = 0; mode = MODE_MODULE_POSTWORK; jl_serialize_lambdas_from_mod(&f, jl_main_module); jl_serialize_value(&f, NULL); // signal end of lambdas - jl_finalize_serializer(&f); + jl_finalize_serializer(&f); // done with f mode = last_mode; jl_gc_enable(en); @@ -1908,7 +1895,8 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) size_t i = 0; while (i < flagref_list.len) { jl_value_t **loc = (jl_value_t**)flagref_list.items[i++]; - jl_value_t *v, *o = *loc; + int offs = (int)(intptr_t)flagref_list.items[i++]; + jl_value_t *v, *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; jl_datatype_t *dt; if (jl_is_datatype(o)) { dt = (jl_datatype_t*)o; @@ -1918,44 +1906,43 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) dt = (jl_datatype_t*)jl_typeof(o); v = o; } + assert(dt); jl_datatype_t *t = (jl_datatype_t*)jl_cache_type_(dt); - int offs = (int)(intptr_t)flagref_list.items[i++]; if (t != dt) { - jl_set_typeof(dt, (jl_value_t*)(ptrint_t)6); // invalidate the old value to help catch errors + jl_set_typeof(dt, (jl_value_t*)(ptrint_t)0x10); // invalidate the old value to help catch errors if ((jl_value_t*)dt == o) { if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } if (t->instance != v) { - jl_set_typeof(v, (jl_value_t*)(ptrint_t)4); // invalidate the old value to help catch errors + jl_set_typeof(v, (jl_value_t*)(ptrint_t)0x20); // invalidate the old value to help catch errors if (v == o) { - if (loc) *loc = v; - if (offs > 0) backref_list.items[offs] = v; + *loc = t->instance; + if (offs > 0) backref_list.items[offs] = t->instance; } } size_t j = i; while (j < flagref_list.len) { - if (flagref_list.items[j] == dt) { + jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; + int offs = (int)(intptr_t)flagref_list.items[j+1]; + if (*loc == (jl_value_t*)dt) { if (t != dt) { - jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; - int offs = (int)(intptr_t)flagref_list.items[j+1]; if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } - else if (flagref_list.items[j] == v) { + else if (*loc == v) { if (t->instance != v) { - jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; - int offs = (int)(intptr_t)flagref_list.items[j+1]; - if (loc) *loc = v; - if (offs > 0) backref_list.items[offs] = v; + *loc = t->instance; + if (offs > 0) backref_list.items[offs] = t->instance; } } else { j += 2; continue; } + // delete this item from the flagref list, so it won't be re-encountered later flagref_list.len -= 2; if (j >= flagref_list.len) break; @@ -1963,6 +1950,7 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) flagref_list.items[j+1] = flagref_list.items[flagref_list.len+1]; } } + jl_finalize_deserializer(f); // done with MODE_MODULE // at this point, the AST is fully reconstructed, but still completely disconnected // in postwork mode, all of the interconnects will be created @@ -2045,7 +2033,7 @@ void jl_init_serializer(void) jl_function_type, jl_simplevector_type, jl_array_type, jl_expr_type, (void*)LongSymbol_tag, (void*)LongSvec_tag, (void*)LongExpr_tag, (void*)LiteralVal_tag, - (void*)SmallInt64_tag, (void*)UNUSED_tag, + (void*)SmallInt64_tag, (void*)SmallDataType_tag, (void*)Int32_tag, (void*)Array1d_tag, (void*)Singleton_tag, jl_module_type, jl_tvar_type, jl_lambda_info_type, (void*)CommonSym_tag, (void*)NearbyGlobal_tag, jl_globalref_type, diff --git a/src/init.c b/src/init.c index b18edf0a32083..ea07afd31290a 100644 --- a/src/init.c +++ b/src/init.c @@ -1104,7 +1104,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) jl_current_module = jl_core_module; jl_root_task->current_module = jl_current_module; - jl_load("boot.jl"); + jl_load("boot.jl", sizeof("boot.jl")); jl_get_builtin_hooks(); jl_boot_file_loaded = 1; jl_init_box_caches(); @@ -1289,28 +1289,43 @@ static void julia_save() if (jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL) jl_compile_all(); - ios_t *s = NULL; - if (jl_options.outputo) - s = jl_create_system_image(); - - if (jl_options.outputji) { - if (s == NULL) { - jl_save_system_image(jl_options.outputji); - } - else { - ios_t f; - if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) - jl_errorf("Cannot open system image file \"%s\" for writing.\n", jl_options.outputji); - ios_write(&f, (const char*)s->buf, s->size); - ios_close(&f); + if (jl_options.incremental) { + jl_array_t *worklist = jl_module_init_order; + if (!worklist) { + jl_printf(JL_STDERR, "incremental output requested, but no modules defined during run"); + return; } + if (jl_options.outputji) + jl_save_incremental(jl_options.outputji, worklist); + if (jl_options.outputbc) + jl_printf(JL_STDERR, "incremental output to a .bc file is not implemented"); + if (jl_options.outputo) + jl_printf(JL_STDERR, "incremental output to a .o file is not implemented"); } + else { + ios_t *s = NULL; + if (jl_options.outputo) + s = jl_create_system_image(); - if (jl_options.outputbc) - jl_dump_bitcode((char*)jl_options.outputbc); + if (jl_options.outputji) { + if (s == NULL) { + jl_save_system_image(jl_options.outputji); + } + else { + ios_t f; + if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) + jl_errorf("Cannot open system image file \"%s\" for writing.\n", jl_options.outputji); + ios_write(&f, (const char*)s->buf, s->size); + ios_close(&f); + } + } - if (jl_options.outputo) - jl_dump_objfile((char*)jl_options.outputo, 0, (const char*)s->buf, s->size); + if (jl_options.outputbc) + jl_dump_bitcode((char*)jl_options.outputbc); + + if (jl_options.outputo) + jl_dump_objfile((char*)jl_options.outputo, 0, (const char*)s->buf, s->size); + } } jl_function_t *jl_typeinf_func=NULL; diff --git a/src/julia.h b/src/julia.h index 7865134908567..186326c880c46 100644 --- a/src/julia.h +++ b/src/julia.h @@ -687,6 +687,7 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) #define jl_cell_data(a) ((jl_value_t**)((jl_array_t*)a)->data) #define jl_string_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) +#define jl_string_len(s) (jl_array_len(((jl_array_t*)(s)->fieldptr[0]))) #define jl_iostr_data(s) ((char*)((jl_array_t*)(s)->fieldptr[0])->data) #define jl_gf_mtable(f) ((jl_methtable_t*)((jl_function_t*)(f))->env) @@ -1194,7 +1195,7 @@ void jl_compile(jl_function_t *f); DLLEXPORT jl_value_t *jl_toplevel_eval(jl_value_t *v); DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex); jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e); -DLLEXPORT jl_value_t *jl_load(const char *fname); +DLLEXPORT jl_value_t *jl_load(const char *fname, size_t len); jl_value_t *jl_parse_eval_all(const char *fname, size_t len); jl_value_t *jl_interpret_toplevel_thunk(jl_lambda_info_t *lam); jl_value_t *jl_interpret_toplevel_thunk_with(jl_lambda_info_t *lam, @@ -1526,6 +1527,7 @@ typedef struct { const char *outputbc; const char *outputo; const char *outputji; + int8_t incremental; } jl_options_t; extern DLLEXPORT jl_options_t jl_options; diff --git a/src/julia_internal.h b/src/julia_internal.h index ac861689fe7b6..960574e1e97f0 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -107,6 +107,8 @@ jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i); void jl_compute_field_offsets(jl_datatype_t *st); jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz); +extern jl_array_t *jl_module_init_order; + #ifdef JL_USE_INTEL_JITEVENTS extern char jl_using_intel_jitevents; #endif diff --git a/src/toplevel.c b/src/toplevel.c index f09c103f0c4b3..134f077842c35 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -70,8 +70,6 @@ jl_module_t *jl_new_main_module(void) return old_main; } -extern jl_array_t *jl_module_init_order; - // load time init procedure: in build mode, only record order void jl_module_load_time_initialize(jl_module_t *m) { @@ -567,7 +565,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) } JL_CATCH { jl_stop_parsing(); - fn = jl_pchar_to_string(fname, strlen(fname)); + fn = jl_pchar_to_string(fname, len); ln = jl_box_long(jl_lineno); jl_lineno = last_lineno; jl_filename = last_filename; @@ -586,7 +584,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len) return result; } -jl_value_t *jl_load(const char *fname) +jl_value_t *jl_load(const char *fname, size_t len) { if (jl_current_module->istopmod) { jl_printf(JL_STDOUT, "%s\r\n", fname); @@ -602,7 +600,7 @@ jl_value_t *jl_load(const char *fname) if (jl_start_parsing_file(fpath) != 0) { jl_errorf("could not open file %s", fpath); } - jl_value_t *result = jl_parse_eval_all(fpath, strlen(fpath)); + jl_value_t *result = jl_parse_eval_all(fpath, len); if (fpath != fname) free(fpath); return result; } @@ -610,7 +608,7 @@ jl_value_t *jl_load(const char *fname) // load from filename given as a ByteString object DLLEXPORT jl_value_t *jl_load_(jl_value_t *str) { - return jl_load(jl_string_data(str)); + return jl_load(jl_string_data(str), jl_string_len(str)); } // type definition ------------------------------------------------------------ diff --git a/test/choosetests.jl b/test/choosetests.jl index 8dee768042cfa..a8a65d93b16ce 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -21,7 +21,7 @@ function choosetests(choices = []) "arrayops", "tuple", "subarray", "reduce", "reducedim", "random", "abstractarray", "intfuncs", "simdloop", "blas", "sparse", "bitarray", "copy", "math", "fastmath", "functional", - "operators", "path", "ccall", "parse", + "operators", "path", "ccall", "parse", "loading", "bigint", "sorting", "statistics", "spawn", "backtrace", "priorityqueue", "file", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", diff --git a/test/core.jl b/test/core.jl index 125743f0e592c..b7470c3318132 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1048,8 +1048,6 @@ let @test Sum < -0.69 end -include("test_sourcepath.jl") - # issue #2509 immutable Foo2509; foo::Int; end @test Foo2509(1) != Foo2509(2) diff --git a/test/loading.jl b/test/loading.jl new file mode 100644 index 0000000000000..938b5f2fa461e --- /dev/null +++ b/test/loading.jl @@ -0,0 +1,29 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +using Base.Test + +expect_load_from = 1 +include("test_sourcepath.jl") # defines "path", consumes "expect_load_from" + +thefname = "the fname!//\\&\0\1*" +@test include_string("@__FILE__", thefname) == ":string:\0$thefname" +@test include_string("@__FILE__") == ":string:\0none" +@test basename(@__FILE__) == "loading.jl" +if myid() != 1 +@test startswith(@__FILE__, ":node 1:\0") +else +@test isabspath(@__FILE__) +end + +# the following may fail if this worker is not on the same machine as 1 +# since it tries to include the file off the local worker +expect_load_from = myid() +include("test_sourcepath.jl", (expect_load_from, dirname(path[2]))) +if nprocs() > 1 + expect_load_from = filter(x -> x != myid(), procs())[rand(1:(nprocs()-1))] + include("test_sourcepath.jl", (expect_load_from, dirname(path[2]))) + @spawnat 1 begin + eval(Main, :(expect_load_from = 1)) + include("test_sourcepath.jl") # defines "path", consumes "expect_load_from" + end +end diff --git a/test/parallel.jl b/test/parallel.jl index 50dbd87bcae4c..27e70247ecf75 100644 --- a/test/parallel.jl +++ b/test/parallel.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # NOTE: worker processes cannot add more workers, only the client process can. -require("testdefs.jl") +using Base.Test if nworkers() < 3 remotecall_fetch(1, () -> addprocs(3 - nworkers())) diff --git a/test/runtests.jl b/test/runtests.jl index 6c40e8973454c..4d901bb4ed6d7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,18 +2,18 @@ include("choosetests.jl") tests, net_on = choosetests(ARGS) -cd(dirname(@__FILE__)) do - n = 1 - if net_on - n = min(8, CPU_CORES, length(tests)) - n > 1 && addprocs(n; exeflags=`--check-bounds=yes --depwarn=error`) - blas_set_num_threads(1) - end - - @everywhere include("testdefs.jl") +n = 1 +if net_on + n = min(8, CPU_CORES, length(tests)) + n > 1 && addprocs(n; exeflags=`--check-bounds=yes --depwarn=error`) + blas_set_num_threads(1) +end - reduce(propagate_errors, nothing, pmap(runtests, tests; err_retry=false, err_stop=true)) +@everywhere include("testdefs.jl") - @unix_only n > 1 && rmprocs(workers(), waitfor=5.0) - println(" \033[32;1mSUCCESS\033[0m") +let cwd = pwd() + reduce(propagate_errors, nothing, pmap(test->runtests(test, cwd), tests; err_retry=false, err_stop=true)) end + +@unix_only n > 1 && rmprocs(workers(), waitfor=5.0) +println(" \033[32;1mSUCCESS\033[0m") diff --git a/test/test_sourcepath.jl b/test/test_sourcepath.jl index 88af5cff0e680..8d6aa11ebd8c1 100644 --- a/test/test_sourcepath.jl +++ b/test/test_sourcepath.jl @@ -1,6 +1,13 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # source path in tasks -path = Base.source_path() -@test endswith(path, joinpath("test","test_sourcepath.jl")) +path = Base.source_path() # this variable is leaked to the source script +@test length(path) == 2 +@test path[1] == expect_load_from # this variable is set in the source script +@test endswith(path[2], joinpath("test","test_sourcepath.jl")) @test yieldto(@task Base.source_path()) == path +if expect_load_from != myid() + @test startswith(@__FILE__, ":node $expect_load_from:\0") +else + @test isabspath(@__FILE__) +end diff --git a/test/testdefs.jl b/test/testdefs.jl index e7ddcda7635cf..2f5ab572f7230 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -2,9 +2,9 @@ using Base.Test -function runtests(name) +function runtests(name, cwd) @printf(" \033[1m*\033[0m \033[31m%-20s\033[0m", name) - tt = @elapsed Core.include(abspath("$name.jl")) + tt = @elapsed include("$name.jl", (1, cwd)) @printf(" in %6.2f seconds\n", tt) nothing end diff --git a/ui/repl.c b/ui/repl.c index aabe69a47acf3..2ebd44594690e 100644 --- a/ui/repl.c +++ b/ui/repl.c @@ -86,6 +86,7 @@ static const char opts[] = " --output-o name Generate an object file (including system image data)\n" " --output-ji name Generate a system image data file (.ji)\n" " --output-bc name Generate LLVM bitcode (.bc)\n\n" + " --output-incremental=no Generate an incremental output file (rather than complete)\n\n" // instrumentation options " --code-coverage={none|user|all}, --code-coverage\n" @@ -113,7 +114,8 @@ void parse_opts(int *argcp, char ***argvp) opt_handle_signals, opt_output_o, opt_output_ji, - opt_use_precompiled + opt_use_precompiled, + opt_incremental }; static char* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:Ob:"; static struct option longopts[] = { @@ -146,6 +148,7 @@ void parse_opts(int *argcp, char ***argvp) { "output-bc", required_argument, 0, opt_output_bc }, { "output-o", required_argument, 0, opt_output_o }, { "output-ji", required_argument, 0, opt_output_ji }, + { "output-incremental",required_argument, 0, opt_incremental }, { "depwarn", required_argument, 0, opt_depwarn }, { "inline", required_argument, 0, opt_inline }, { "math-mode", required_argument, 0, opt_math_mode }, @@ -330,6 +333,14 @@ void parse_opts(int *argcp, char ***argvp) jl_options.outputji = optarg; if (!imagepathspecified) jl_options.image_file = NULL; break; + case opt_incremental: + if (!strcmp(optarg,"yes")) + jl_options.incremental = 1; + else if (!strcmp(optarg,"no")) + jl_options.incremental = 0; + else + jl_errorf("julia: invalid argument to --output-incremental={yes|no} (%s)\n", optarg); + break; case opt_depwarn: if (!strcmp(optarg,"yes")) jl_options.depwarn = JL_OPTIONS_DEPWARN_ON; @@ -394,7 +405,7 @@ static int exec_program(char *program) jl_value_t *errs = jl_stderr_obj(); jl_value_t *e = jl_exception_in_transit; if (errs != NULL) { - jl_show(jl_stderr_obj(), e); + jl_show(errs, e); } else { jl_printf(JL_STDERR, "error during bootstrap:\n"); @@ -406,7 +417,7 @@ static int exec_program(char *program) JL_EH_POP(); return 1; } - jl_load(program); + jl_load(program, strlen(program)); } JL_CATCH { err = 1; From fee6081b889db36b39d30e9fdcbd1bdf27432b3f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 9 Jul 2015 16:56:42 -0400 Subject: [PATCH 09/28] fix a bug where jl_typeof(jl_emptytuple) was a singleton type missing from the serializer known objects list this also serializes the datatype along with any singleton. this simplifies the back-and-forth a bit, but more importantly, during incremental compilation, it was possible to encounter a singleton without ever encountering its datatype. it is now trivially possible to remove the Singleton_tag --- src/dump.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/dump.c b/src/dump.c index 9cb3d657b1cfe..1d7bb76016156 100644 --- a/src/dump.c +++ b/src/dump.c @@ -794,7 +794,14 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) else { if (v == t->instance) { assert(mode != MODE_MODULE_POSTWORK); + if (mode == MODE_MODULE) { + // also flag this in the backref table as special + uptrint_t *bp = (uptrint_t*)ptrhash_bp(&backref_table, v); + assert(*bp != (uptrint_t)HT_NOTFOUND); + *bp |= 1; + } writetag(s, (jl_value_t*)Singleton_tag); + jl_serialize_value(s, t); return; } if (t->size <= 255) { @@ -983,7 +990,6 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) if (has_instance) { assert(mode != MODE_MODULE_POSTWORK); // there shouldn't be an instance on a type with uid = 0 dt->instance = jl_deserialize_value(s, &dt->instance); - jl_set_typeof(dt->instance, dt); jl_gc_wb(dt, dt->instance); } if (tag == 5) { @@ -1382,7 +1388,13 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return v; } else if (vtag == (jl_value_t*)Singleton_tag) { - assert(mode != MODE_MODULE_POSTWORK); + if (mode == MODE_MODULE_POSTWORK) { + uptrint_t pos = backref_list.len; + arraylist_push(&backref_list, NULL); + jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); + backref_list.items[pos] = dt->instance; + return dt->instance; + } jl_value_t *v = (jl_value_t*)jl_gc_alloc_0w(); if (usetable) { uptrint_t pos = backref_list.len; @@ -1393,6 +1405,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t arraylist_push(&flagref_list, (void*)pos); } } + jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, NULL); // no loc, since if dt is replaced, then dt->instance would be also + jl_set_typeof(v, dt); return v; } assert(0); @@ -1926,13 +1940,14 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) while (j < flagref_list.len) { jl_value_t **loc = (jl_value_t**)flagref_list.items[j]; int offs = (int)(intptr_t)flagref_list.items[j+1]; - if (*loc == (jl_value_t*)dt) { + jl_value_t *o = loc ? *loc : (jl_value_t*)backref_list.items[offs]; + if ((jl_value_t*)dt == o) { if (t != dt) { if (loc) *loc = (jl_value_t*)t; if (offs > 0) backref_list.items[offs] = t; } } - else if (*loc == v) { + else if (v == o) { if (t->instance != v) { *loc = t->instance; if (offs > 0) backref_list.items[offs] = t->instance; @@ -2037,6 +2052,7 @@ void jl_init_serializer(void) (void*)Int32_tag, (void*)Array1d_tag, (void*)Singleton_tag, jl_module_type, jl_tvar_type, jl_lambda_info_type, (void*)CommonSym_tag, (void*)NearbyGlobal_tag, jl_globalref_type, + // everything above here represents a class of object rather only than a literal jl_emptysvec, jl_emptytuple, jl_false, jl_true, jl_nothing, jl_any_type, call_sym, goto_ifnot_sym, return_sym, body_sym, line_sym, @@ -2093,7 +2109,7 @@ void jl_init_serializer(void) jl_ANY_flag, jl_array_any_type, jl_intrinsic_type, jl_method_type, jl_methtable_type, jl_voidpointer_type, jl_newvarnode_type, jl_array_symbol_type, jl_anytuple_type, jl_tparam0(jl_anytuple_type), - + jl_typeof(jl_emptytuple), jl_symbol_type->name, jl_gensym_type->name, jl_tuple_typename, jl_ref_type->name, jl_pointer_type->name, jl_simplevector_type->name, jl_datatype_type->name, jl_uniontype_type->name, jl_array_type->name, From 22dfad54137180245db7bb630a9a3f19485e10d6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 9 Jul 2015 22:42:45 -0400 Subject: [PATCH 10/28] record binding module in new generic functions --- src/dump.c | 46 +++++++++++++++++++++++++++++++++----------- src/gf.c | 18 +++++++---------- src/jltypes.c | 14 ++++++++------ src/julia.h | 3 ++- src/julia_internal.h | 1 - src/toplevel.c | 13 +++++++++---- 6 files changed, 61 insertions(+), 34 deletions(-) diff --git a/src/dump.c b/src/dump.c index 1d7bb76016156..9839c739d3a7f 100644 --- a/src/dump.c +++ b/src/dump.c @@ -721,6 +721,18 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else if (jl_is_function(v)) { writetag(s, jl_function_type); + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) { + if (jl_is_gf(v)) { + jl_methtable_t *mt = jl_gf_mtable(v); + if (mt->module && !module_in_worklist(mt->module)) { + write_int8(s, 1); + jl_serialize_value(s, (jl_value_t*)mt->module); + jl_serialize_value(s, (jl_value_t*)mt->name); + return; + } + } + write_int8(s, 0); + } jl_function_t *f = (jl_function_t*)v; jl_serialize_value(s, (jl_value_t*)f->linfo); jl_serialize_value(s, f->env); @@ -846,20 +858,22 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } } -static void jl_serialize_methtable_from_mod(ios_t *s, jl_module_t *m, jl_sym_t *name, jl_methtable_t *mt, int8_t iskw) +static void jl_serialize_methtable_from_mod(ios_t *s, jl_methtable_t *mt, int8_t iskw) { if (iskw) { if (!mt->kwsorter) return; assert(jl_is_gf(mt->kwsorter)); + assert(mt->module == jl_gf_mtable(mt->kwsorter)->module); mt = jl_gf_mtable(mt->kwsorter); assert(!mt->kwsorter); } + assert(mt->module); jl_methlist_t *ml = mt->defs; while (ml != (void*)jl_nothing) { if (module_in_worklist(ml->func->linfo->module)) { - jl_serialize_value(s, m); - jl_serialize_value(s, name); + jl_serialize_value(s, mt->module); + jl_serialize_value(s, mt->name); write_int8(s, iskw); jl_serialize_value(s, ml->sig); jl_serialize_value(s, ml->func); @@ -882,15 +896,13 @@ static void jl_serialize_lambdas_from_mod(ios_t *s, jl_module_t *m) if (table[i] != HT_NOTFOUND) { jl_binding_t *b = (jl_binding_t*)table[i]; if (b->owner == m && b->value && b->constp) { - if (jl_is_function(b->value)) { + if (jl_is_function(b->value) && jl_is_gf(b->value)) { jl_function_t *gf = (jl_function_t*)b->value; - if (jl_is_gf(gf)) { - // TODO: verify this is not an alternative name for the gf - jl_methtable_t *mt = jl_gf_mtable(gf); - jl_serialize_methtable_from_mod(s, m, b->name, mt, 0); - jl_serialize_methtable_from_mod(s, m, b->name, mt, 1); + jl_methtable_t *mt = jl_gf_mtable(gf); + if (mt->name == b->name && mt->module == m) { + jl_serialize_methtable_from_mod(s, mt, 0); + jl_serialize_methtable_from_mod(s, mt, 1); } - //TODO: look in datatype cache? } else if (jl_is_module(b->value)) { jl_module_t *child = (jl_module_t*)b->value; @@ -1194,6 +1206,18 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)tv; } else if (vtag == (jl_value_t*)jl_function_type) { + if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) { + int ref_only = read_int8(s); + if (ref_only) { + int pos = backref_list.len; + arraylist_push(&backref_list, NULL); + jl_module_t *module = (jl_module_t*)jl_deserialize_value(s, NULL); + jl_sym_t *name = (jl_sym_t*)jl_deserialize_value(s, NULL); + jl_value_t *f = jl_get_global(module, name); + backref_list.items[pos] = f; + return f; + } + } jl_function_t *f = (jl_function_t*)newobj((jl_value_t*)jl_function_type, NWORDS(sizeof(jl_function_t))); if (usetable) @@ -1426,7 +1450,7 @@ void jl_deserialize_lambdas_from_mod(ios_t *s) if (iskw) { jl_methtable_t* mt = jl_gf_mtable(gf); if (!mt->kwsorter) { - mt->kwsorter = jl_new_generic_function(jl_gf_name(gf)); + mt->kwsorter = jl_new_generic_function(jl_gf_name(gf), mt->module); jl_gc_wb(mt, mt->kwsorter); } gf = mt->kwsorter; diff --git a/src/gf.c b/src/gf.c index 96bebce10ad3e..b3b0ef408311b 100644 --- a/src/gf.c +++ b/src/gf.c @@ -38,11 +38,12 @@ static jl_value_t *jl_apply_unspecialized(jl_function_t *meth, jl_value_t **args } -static jl_methtable_t *new_method_table(jl_sym_t *name) +static jl_methtable_t *new_method_table(jl_sym_t *name, jl_module_t *module) { jl_methtable_t *mt = (jl_methtable_t*)jl_gc_allocobj(sizeof(jl_methtable_t)); jl_set_typeof(mt, jl_methtable_type); mt->name = name; + mt->module = module; mt->defs = (jl_methlist_t*)jl_nothing; mt->cache = (jl_methlist_t*)jl_nothing; mt->cache_arg1 = (jl_array_t*)jl_nothing; @@ -1753,7 +1754,7 @@ jl_value_t *jl_gf_invoke(jl_function_t *gf, jl_tupletype_t *types, JL_GC_PUSH3(&tpenv, &newsig, &tt); tt = arg_type_tuple(args, nargs); if (m->invokes == (void*)jl_nothing) { - m->invokes = new_method_table(mt->name); + m->invokes = new_method_table(mt->name, mt->module); jl_gc_wb(m, m->invokes); update_max_args(m->invokes, tt); // this private method table has just this one definition @@ -1794,18 +1795,13 @@ void print_func_loc(JL_STREAM *s, jl_lambda_info_t *li) } } -void jl_initialize_generic_function(jl_function_t *f, jl_sym_t *name) -{ - f->fptr = jl_apply_generic; - f->env = (jl_value_t*)new_method_table(name); - jl_gc_wb(f, f->env); -} - -jl_function_t *jl_new_generic_function(jl_sym_t *name) +jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module) { jl_function_t *f = jl_new_closure(jl_apply_generic, NULL, NULL); JL_GC_PUSH1(&f); - jl_initialize_generic_function(f, name); + f->fptr = jl_apply_generic; + f->env = (jl_value_t*)new_method_table(name, module); + jl_gc_wb(f, f->env); JL_GC_POP(); return f; } diff --git a/src/jltypes.c b/src/jltypes.c index 9e15ad69c6a7c..fc7bc198e30a9 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3290,13 +3290,14 @@ void jl_init_types(void) jl_methtable_type = jl_new_datatype(jl_symbol("MethodTable"), jl_any_type, jl_emptysvec, - jl_svec(7, jl_symbol("name"), jl_symbol("defs"), + jl_svec(8, jl_symbol("name"), jl_symbol("defs"), jl_symbol("cache"), jl_symbol("cache_arg1"), - jl_symbol("cache_targ"), - jl_symbol("max_args"), jl_symbol("kwsorter")), - jl_svec(7, jl_sym_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type, jl_long_type, - jl_any_type), + jl_symbol("cache_targ"), jl_symbol("max_args"), + jl_symbol("kwsorter"), jl_symbol("module")), + jl_svec(8, jl_sym_type, jl_any_type, + jl_any_type, jl_any_type, + jl_any_type, jl_long_type, + jl_any_type, jl_any_type), 0, 1, 6); tv = jl_svec2(tvar("T"), tvar("N")); @@ -3381,6 +3382,7 @@ void jl_init_types(void) jl_svec(2, jl_module_type, jl_sym_type), 0, 0, 2); jl_svecset(jl_typename_type->types, 1, jl_module_type); + jl_svecset(jl_methtable_type->types, 7, jl_module_type); jl_lambda_info_type = jl_new_datatype(jl_symbol("LambdaStaticData"), diff --git a/src/julia.h b/src/julia.h index 186326c880c46..da5afc4d3b7bf 100644 --- a/src/julia.h +++ b/src/julia.h @@ -351,6 +351,7 @@ typedef struct _jl_methtable_t { jl_array_t *cache_targ; ptrint_t max_args; // max # of non-vararg arguments in a signature jl_function_t *kwsorter; // keyword argument sorter function + jl_module_t *module; // used for incremental serialization to locate original binding #ifdef JL_GF_PROFILE int ncalls; #endif @@ -924,7 +925,7 @@ DLLEXPORT jl_sym_t *jl_gensym(void); DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, int32_t len); DLLEXPORT jl_sym_t *jl_get_root_symbol(void); jl_expr_t *jl_exprn(jl_sym_t *head, size_t n); -jl_function_t *jl_new_generic_function(jl_sym_t *name); +jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module); void jl_add_method(jl_function_t *gf, jl_tupletype_t *types, jl_function_t *meth, jl_svec_t *tvars, int8_t isstaged); DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_value_t **bp, jl_value_t *bp_owner, diff --git a/src/julia_internal.h b/src/julia_internal.h index 960574e1e97f0..018ea1308e77f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -100,7 +100,6 @@ jl_datatype_t *jl_inst_concrete_tupletype_v(jl_value_t **p, size_t np); jl_datatype_t *jl_inst_concrete_tupletype(jl_svec_t *p); void jl_set_datatype_super(jl_datatype_t *tt, jl_value_t *super); -void jl_initialize_generic_function(jl_function_t *f, jl_sym_t *name); void jl_add_constructors(jl_datatype_t *t); jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i); diff --git a/src/toplevel.c b/src/toplevel.c index 134f077842c35..f4f65f6cdaa57 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -679,7 +679,8 @@ DLLEXPORT jl_value_t *jl_generic_function_def(jl_sym_t *name, jl_value_t **bp, j if (bnd) bnd->constp = 1; if (*bp == NULL) { - gf = (jl_value_t*)jl_new_generic_function(name); + jl_module_t *module = (bnd ? bnd->owner : NULL); + gf = (jl_value_t*)jl_new_generic_function(name, module); *bp = gf; if (bp_owner) jl_gc_wb(bp_owner, gf); } @@ -709,6 +710,7 @@ DLLEXPORT jl_value_t *jl_method_def(jl_sym_t *name, jl_value_t **bp, jl_value_t jl_svec_t *argdata, jl_function_t *f, jl_value_t *isstaged, jl_value_t *call_func, int iskw) { + jl_module_t *module = (bnd ? bnd->owner : NULL); // argdata is svec({types...}, svec(typevars...)) jl_tupletype_t *argtypes = (jl_tupletype_t*)jl_svecref(argdata,0); jl_svec_t *tvars = (jl_svec_t*)jl_svecref(argdata,1); @@ -780,8 +782,11 @@ DLLEXPORT jl_value_t *jl_method_def(jl_sym_t *name, jl_value_t **bp, jl_value_t } } if (iskw) { - bp = (jl_value_t**)&((jl_methtable_t*)((jl_function_t*)gf)->env)->kwsorter; - bp_owner = (jl_value_t*)((jl_function_t*)gf)->env; + jl_methtable_t *mt = jl_gf_mtable(gf); + assert(!module); + module = mt->module; + bp = (jl_value_t**)&mt->kwsorter; + bp_owner = (jl_value_t*)mt; gf = *bp; } } @@ -814,7 +819,7 @@ DLLEXPORT jl_value_t *jl_method_def(jl_sym_t *name, jl_value_t **bp, jl_value_t bnd->constp = 1; } if (*bp == NULL) { - gf = (jl_value_t*)jl_new_generic_function(name); + gf = (jl_value_t*)jl_new_generic_function(name, module); *bp = gf; if (bp_owner) jl_gc_wb(bp_owner, gf); } From b3423f220b28da7cff15f338186fe64ad1945bf9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 10 Jul 2015 15:06:51 -0400 Subject: [PATCH 11/28] empty typename->cache so that flagref can rebuild it cleanly --- src/dump.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/dump.c b/src/dump.c index 9839c739d3a7f..411740d67694f 100644 --- a/src/dump.c +++ b/src/dump.c @@ -805,7 +805,6 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else { if (v == t->instance) { - assert(mode != MODE_MODULE_POSTWORK); if (mode == MODE_MODULE) { // also flag this in the backref table as special uptrint_t *bp = (uptrint_t*)ptrhash_bp(&backref_table, v); @@ -1050,6 +1049,7 @@ jl_array_t *jl_eqtable_put(jl_array_t *h, void *key, void *val); static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t **loc); static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) { + assert(!ios_eof(s)); uint8_t tag = read_uint8(s); if (tag == Null_tag) return NULL; @@ -1405,8 +1405,12 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK)) { if (jl_is_mtable(v)) arraylist_push(&methtable_list, v); // will resort this table, later - if (dt == jl_typename_type) - ((jl_typename_t*)v)->uid = jl_assign_type_uid(); // make sure this has a new uid + if (dt == jl_typename_type) { + jl_typename_t* tn = (jl_typename_t*)v; + tn->uid = jl_assign_type_uid(); // make sure this has a new uid + tn->cache = jl_emptysvec; // the cache is refilled later (tag 5) + tn->linearcache = jl_emptysvec; // the cache is refilled later (tag 5) + } } } return v; From 6ceff1cb9a0c6d58c6541067220dcf2c7dc7ed6e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 10 Jul 2015 22:38:21 -0400 Subject: [PATCH 12/28] fix dt->uid assignment --- src/dump.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/dump.c b/src/dump.c index 411740d67694f..0245273dc5c6c 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1008,6 +1008,7 @@ static jl_value_t *jl_deserialize_datatype(ios_t *s, int pos, jl_value_t **loc) assert(mode != MODE_MODULE_POSTWORK); arraylist_push(&flagref_list, loc); arraylist_push(&flagref_list, (void*)(uptrint_t)pos); + dt->uid = jl_assign_type_uid(); // make sure this has a new uid early so it goes in the type cache correctly } if (nf > 0) { @@ -1723,9 +1724,7 @@ void jl_restore_system_image_from_stream(ios_t *f) // cache builtin parametric types for(int i=0; i < jl_array_len(datatype_list); i++) { jl_value_t *v = jl_cellref(datatype_list, i); - uint32_t uid = ((jl_datatype_t*)v)->uid; jl_cache_type_((jl_datatype_t*)v); - ((jl_datatype_t*)v)->uid = uid; } datatype_list = NULL; From 39bf8197c6a8bbd07a82eff06b97d6b4c9a8d1f4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 10 Jul 2015 23:22:53 -0400 Subject: [PATCH 13/28] fix a bug with remote restore-incremental return type handling --- base/loading.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 3f3fa4de84706..bc5e8c1cf81b7 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -47,7 +47,7 @@ end function _include_from_serialized(content::Vector{UInt8}) m = ccall(:jl_restore_incremental_from_buf, UInt, (Ptr{Uint8},Int), content, sizeof(content)) - return m + return m != 0 end function _require_from_serialized(node::Int, path_to_try::ByteString) @@ -63,8 +63,8 @@ function _require_from_serialized(node::Int, path_to_try::ByteString) refs = Any[ @spawnat p _include_from_serialized(content) for p in others] successes = Any[ fetch(ref) for ref in refs ] for (id, ref) in zip(others, refs) - if fetch(ref) != 0 - warn("node state is inconsistent: node $i failed to load serialized cache from :node $id:$path_to_try.") + if fetch(ref) + warn("node state is inconsistent: node $id failed to load serialized cache from :node $node:$path_to_try.") end end return true From d81f6d2fcaa1ec77402e1e9526886441944ebf24 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 11 Jul 2015 00:41:00 -0400 Subject: [PATCH 14/28] allow proper backref handling of the element that happened to land at index HT_NOTFOUND --- src/dump.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/dump.c b/src/dump.c index 0245273dc5c6c..6b4bbe4ca8773 100644 --- a/src/dump.c +++ b/src/dump.c @@ -290,9 +290,10 @@ static void jl_serialize_globalvals(ios_t *s) for(i=0; i < len; i+=2) { void *offs = p[i+1]; if (offs != HT_NOTFOUND) { + uintptr_t pos = offs - HT_NOTFOUND - 1; int32_t gv = jl_get_llvm_gv((jl_value_t*)p[i]); if (gv != 0) { - write_int32(s, (int)(intptr_t)offs); + write_int32(s, pos); write_int32(s, gv); } } @@ -445,7 +446,7 @@ static void jl_serialize_datatype(ios_t *s, jl_datatype_t *dt) // also flag this in the backref table as special uptrint_t *bp = (uptrint_t*)ptrhash_bp(&backref_table, dt); assert(*bp != (uptrint_t)HT_NOTFOUND); - *bp |= 1; + *bp |= 1; assert(((uptrint_t)HT_NOTFOUND)|1); } } else if (dt == jl_int32_type) @@ -507,8 +508,10 @@ static void jl_serialize_module(ios_t *s, jl_module_t *m) write_int8(s, ref_only); } jl_serialize_value(s, m->parent); - if (ref_only) + if (ref_only) { + assert(m->parent != m); return; + } size_t i; void **table = m->bindings.table; for(i=1; i < m->bindings.size; i+=2) { @@ -601,13 +604,14 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) else { bp = ptrhash_bp(&backref_table, v); if (*bp != HT_NOTFOUND) { + uintptr_t pos = *bp - HT_NOTFOUND - 1; if ((uptrint_t)*bp < 65536) { write_uint8(s, ShortBackRef_tag); - write_uint16(s, (uptrint_t)*bp); + write_uint16(s, pos); } else { write_uint8(s, BackRef_tag); - write_int32(s, (uptrint_t)*bp); + write_int32(s, pos); } return; } @@ -627,7 +631,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } if (mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK) pos <<= 1; - ptrhash_put(&backref_table, v, (void*)pos); + ptrhash_put(&backref_table, v, HT_NOTFOUND + pos + 1); } size_t i; @@ -809,7 +813,7 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) // also flag this in the backref table as special uptrint_t *bp = (uptrint_t*)ptrhash_bp(&backref_table, v); assert(*bp != (uptrint_t)HT_NOTFOUND); - *bp |= 1; + *bp |= 1; assert(((uptrint_t)HT_NOTFOUND)|1); } writetag(s, (jl_value_t*)Singleton_tag); jl_serialize_value(s, t); @@ -1881,7 +1885,7 @@ DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) JL_SIGATOMIC_BEGIN(); arraylist_new(&reinit_list, 0); htable_new(&backref_table, 5000); - ptrhash_put(&backref_table, jl_main_module, (void*)(uintptr_t)0); + ptrhash_put(&backref_table, jl_main_module, HT_NOTFOUND + 1); backref_table_numel = 1; jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("ObjectIdDict")) : NULL; From efcc709334ac8335360fa7eb1f8bb9bdda0b51ee Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 11 Jul 2015 01:02:07 -0400 Subject: [PATCH 15/28] sysimg does not need to call workspace() anymore. this sly module replacement can leak into the final environment and it invalidates a basic premise of incremental compilation (that modules are static) --- base/sysimg.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/base/sysimg.jl b/base/sysimg.jl index 9fd1bc1e15860..acff7bb926a62 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -1,7 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license import Core.Intrinsics.ccall -ccall(:jl_new_main_module, Any, ()) baremodule Base From e8a1c7440f47707be3329775fac91f0c4bf9c27d Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 11 Jul 2015 10:13:49 -0500 Subject: [PATCH 16/28] Add missing base_include method This fixes errors that crop up with multiple workers, e.g., ERROR: MethodError: `base_include` has no method matching base_include(::ASCIIString, ::ASCIIString, ::Tuple{Int64,ASCIIString}) Closest candidates are: base_include(::Nullable{Union{UTF8String,ASCIIString}}, ::AbstractString, ::Any) base_include(::AbstractString, ::Any) base_include(::Nullable{Union{UTF8String,ASCIIString}}, ::AbstractString) ... in eval at sysimg.jl:14 in anonymous at multi.jl:1303 in run_work_thunk at multi.jl:584 ... Perhaps you'd prefer a call-site fix? --- base/loading.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/loading.jl b/base/loading.jl index bc5e8c1cf81b7..917364bc97e42 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -229,6 +229,7 @@ function base_include(content::Nullable{ByteString}, path::AbstractString, relat end end end +base_include(content::AbstractString, path::AbstractString, relative_to=nothing) = base_include(Nullable{ByteString}(content), path, relative_to) base_include(path::AbstractString, relative_to=nothing) = base_include(Nullable{ByteString}(), path, relative_to) include_string(txt::AbstractString, fname::AbstractString="none") = base_include(Nullable{ByteString}(bytestring(txt)), bytestring(fname)) From c43f24451f61f2642039d2ee68349471ceb0e3b8 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 11 Jul 2015 10:14:33 -0500 Subject: [PATCH 17/28] Fix serialization warning from workers --- base/loading.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 917364bc97e42..194c7876fd4aa 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -61,9 +61,8 @@ function _require_from_serialized(node::Int, path_to_try::ByteString) if _include_from_serialized(content) others = filter(x -> x != myid(), procs()) refs = Any[ @spawnat p _include_from_serialized(content) for p in others] - successes = Any[ fetch(ref) for ref in refs ] for (id, ref) in zip(others, refs) - if fetch(ref) + if !fetch(ref) warn("node state is inconsistent: node $id failed to load serialized cache from :node $node:$path_to_try.") end end From f7ad4e0671698b2be474417b693114a9cb8f00f9 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 12 Jul 2015 12:51:46 -0500 Subject: [PATCH 18/28] Add `require` deprecations --- base/deprecated.jl | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 0c97fce868e66..87888ff3a1ebf 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -635,7 +635,6 @@ end @deprecate mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Integer}, s::IOStream, offset::FileOffset=position(s)) mmap(s, BitArray, dims, offset) @deprecate mmap_bitarray{N}(dims::NTuple{N,Integer}, s::IOStream, offset=position(s)) mmap(s, BitArray, dims, offset) - # T[a:b] and T[a:s:b] @noinline function getindex{T<:Union{Char,Number}}(::Type{T}, r::Range) depwarn("T[a:b] concatenation is deprecated; use T[a:b;] instead", :getindex) @@ -654,3 +653,32 @@ function getindex{T<:Union{Char,Number}}(::Type{T}, r1::Range, rs::Range...) end return a end + +function require(mod::AbstractString) + depwarn("`require` is deprecated, use `using` or `import` instead", :require) + require(symbol(require_filename(mod))) +end +function require(f::AbstractString, fs::AbstractString...) + require(f) + for fn in fs + require(fn) + end +end +export require +function require_filename(name::AbstractString) + # This function can be deleted when the deprecation for `require` + # is deleted. + # While we could also strip off the absolute path, the user may be + # deliberately directing to a different file than what got + # cached. So this takes a conservative approach. + if endswith(name, ".jl") + tmp = name[1:end-3] + for prefix in LOAD_CACHE_PATH + path = joinpath(prefix, tmp*".ji") + if isfile(path) + return tmp + end + end + end + name +end From e30ce8deeb6d5d6b8aae9f630d724fd2dfdaaa6e Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 12 Jul 2015 12:54:34 -0500 Subject: [PATCH 19/28] Use julia-versioning in libs path This way people who have two julia installations, release and master, won't have to recompile their packages all the time. (Relevant only once 0.4 or higher _is_ release, and master is something even newer.) --- base/client.jl | 2 +- base/loading.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/client.jl b/base/client.jl index 90ea29a9aa633..e4f9a8b36f3fa 100644 --- a/base/client.jl +++ b/base/client.jl @@ -332,7 +332,7 @@ function init_load_path() end push!(LOAD_PATH,abspath(JULIA_HOME,"..","local","share","julia","site",vers)) push!(LOAD_PATH,abspath(JULIA_HOME,"..","share","julia","site",vers)) - push!(LOAD_CACHE_PATH,abspath(homedir(),".julia",".libs")) + push!(LOAD_CACHE_PATH,abspath(homedir(),".julia","libs",vers)) push!(LOAD_CACHE_PATH,abspath(JULIA_HOME,"..","usr","lib","julia")) #TODO: fixme end diff --git a/base/loading.jl b/base/loading.jl index 194c7876fd4aa..1955d1f92b4b3 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -245,7 +245,7 @@ evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args... function create_expr_cache(m::Expr, name) cachepath = LOAD_CACHE_PATH[1] if !isdir(cachepath) - mkdir(cachepath) + mkpath(cachepath) end cachefile = abspath(cachepath, name*".ji") code_object = """ From 1b075436beb2d1483adf756cee8644e0b101d2a8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 13 Jul 2015 16:43:44 -0400 Subject: [PATCH 20/28] back out changes making base_include multinode-aware --- base/client.jl | 4 ++-- base/loading.jl | 48 ++++++++++++++++------------------------- test/loading.jl | 25 +++------------------ test/test_sourcepath.jl | 12 +++-------- test/testdefs.jl | 2 +- 5 files changed, 28 insertions(+), 63 deletions(-) diff --git a/base/client.jl b/base/client.jl index e4f9a8b36f3fa..5813aca22a8c0 100644 --- a/base/client.jl +++ b/base/client.jl @@ -274,8 +274,8 @@ let reqarg = Set(UTF8String["--home", "-H", end # load file immediately on all processors if opts.load != C_NULL - let file = bytestring(opts.load), cwd = pwd() - @everywhere include($file, (1, $cwd)) + let file = bytestring(abspath(opts.load)) + @everywhere include($file) end end # eval expression diff --git a/base/loading.jl b/base/loading.jl index 1955d1f92b4b3..5085dcbafab38 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -129,10 +129,10 @@ function require(mod::Symbol, cachecompile::Bool=false) if myid() == 1 && current_module() === Main && nprocs() > 1 # broadcast top-level import/using from node 1 (only) content = open(readall, path) - refs = Any[ @spawnat p eval(Main, :(Base.base_include($content, $path, (1, "")))) for p in procs() ] + refs = Any[ @spawnat p eval(Main, :(Base.include_string($content, $path))) for p in procs() ] for r in refs; wait(r); end else - eval(Main, :(Base.base_include($path, (1, "")))) + eval(Main, :(Base.base_include($path))) end finally loading = pop!(package_locks, mod) @@ -159,61 +159,52 @@ function source_path(default::Union{AbstractString,Void}=nothing) end macro __FILE__() - sp = source_path() - if !isa(sp, Tuple{Int,Any}) - return ":unknown:\0$sp" - elseif sp[1] === 0 - return ":string:\0$(sp[2])" - elseif sp[1] !== myid() - return ":node $(sp[1]):\0$(sp[2])" - else - return sp[2] - end + return source_path() end # base_include is the implementation for Base.include -# if relative_to::Void, it uses the working directory and node of the previous (recursive) include to find the file -# relative_to is a tuple of (node, cwd), where node can be 0 iff the content is from a string (sans a real path) +# if relative_to::Void, it uses the dirpath of the previous (recursive) include to find the file +# relative_to is an alternate , where node can be 0 iff the content is from a string (sans a real path) function base_include(content::Nullable{ByteString}, path::AbstractString, relative_to=nothing) prev = source_path(nothing) - local bytepath::ByteString, node::Int + local bytepath::ByteString if !isnull(content) # already have content, so no need for file-system operations if relative_to === nothing - node = 0 bytepath = bytestring(path) else - node = relative_to[1] - bytepath = bytestring(joinpath(relative_to[2],path)) + bytepath = bytestring(joinpath(relative_to, path)) end else if prev !== nothing && relative_to === nothing # if this is a recursive include without explicit relative_to location # pick up previous location for last include - relative_to = (prev[1], dirname(prev[2])) + relative_to = dirname(prev) end - if relative_to === nothing || relative_to[1] === 0 - # no previous path, pick up location from host - node = myid() - bytepath = bytestring(abspath(path)) + if relative_to === nothing + # no previous path, pick up absolute path location from node 1 + if myid() == 1 || isabspath(path) + bytepath = bytestring(abspath(path)) + else + bytepath = remotecall_fetch(1, path -> bytestring(abspath(path)), path) + end else - node = relative_to[1] - bytepath = bytestring(joinpath(relative_to[2],path)) + bytepath = bytestring(joinpath(relative_to, path)) end end tls = task_local_storage() - tls[:SOURCE_PATH] = (node, bytepath) + tls[:SOURCE_PATH] = bytepath try if isnull(content) - if myid() === node + if myid() === 1 # sleep a bit to process requests from other nodes nprocs()>1 && yield() result = Core.include(bytepath) nprocs()>1 && yield() return result else - content = Nullable{ByteString}(remotecall_fetch(node, readall, bytepath)) + content = Nullable{ByteString}(remotecall_fetch(1, readall, bytepath)) # fall-through now that content is assigned end end @@ -228,7 +219,6 @@ function base_include(content::Nullable{ByteString}, path::AbstractString, relat end end end -base_include(content::AbstractString, path::AbstractString, relative_to=nothing) = base_include(Nullable{ByteString}(content), path, relative_to) base_include(path::AbstractString, relative_to=nothing) = base_include(Nullable{ByteString}(), path, relative_to) include_string(txt::AbstractString, fname::AbstractString="none") = base_include(Nullable{ByteString}(bytestring(txt)), bytestring(fname)) diff --git a/test/loading.jl b/test/loading.jl index 938b5f2fa461e..5a86356decda3 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -2,28 +2,9 @@ using Base.Test -expect_load_from = 1 -include("test_sourcepath.jl") # defines "path", consumes "expect_load_from" - +include("test_sourcepath.jl") thefname = "the fname!//\\&\0\1*" -@test include_string("@__FILE__", thefname) == ":string:\0$thefname" -@test include_string("@__FILE__") == ":string:\0none" +@test include_string("@__FILE__", thefname) == "$thefname" +@test include_string("@__FILE__") == "none" @test basename(@__FILE__) == "loading.jl" -if myid() != 1 -@test startswith(@__FILE__, ":node 1:\0") -else @test isabspath(@__FILE__) -end - -# the following may fail if this worker is not on the same machine as 1 -# since it tries to include the file off the local worker -expect_load_from = myid() -include("test_sourcepath.jl", (expect_load_from, dirname(path[2]))) -if nprocs() > 1 - expect_load_from = filter(x -> x != myid(), procs())[rand(1:(nprocs()-1))] - include("test_sourcepath.jl", (expect_load_from, dirname(path[2]))) - @spawnat 1 begin - eval(Main, :(expect_load_from = 1)) - include("test_sourcepath.jl") # defines "path", consumes "expect_load_from" - end -end diff --git a/test/test_sourcepath.jl b/test/test_sourcepath.jl index 8d6aa11ebd8c1..837e89e126adf 100644 --- a/test/test_sourcepath.jl +++ b/test/test_sourcepath.jl @@ -1,13 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license # source path in tasks -path = Base.source_path() # this variable is leaked to the source script -@test length(path) == 2 -@test path[1] == expect_load_from # this variable is set in the source script -@test endswith(path[2], joinpath("test","test_sourcepath.jl")) +path = Base.source_path()::ByteString # this variable is leaked to the source script +@test endswith(path, joinpath("test","test_sourcepath.jl")) @test yieldto(@task Base.source_path()) == path -if expect_load_from != myid() - @test startswith(@__FILE__, ":node $expect_load_from:\0") -else - @test isabspath(@__FILE__) -end +@test isabspath(@__FILE__) diff --git a/test/testdefs.jl b/test/testdefs.jl index 2f5ab572f7230..141d72bdf275f 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -4,7 +4,7 @@ using Base.Test function runtests(name, cwd) @printf(" \033[1m*\033[0m \033[31m%-20s\033[0m", name) - tt = @elapsed include("$name.jl", (1, cwd)) + tt = @elapsed include("$name.jl", cwd) @printf(" in %6.2f seconds\n", tt) nothing end From d98075fcce0b3e6494256d53a5dfb4cd383b9bbd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 13 Jul 2015 17:16:17 -0400 Subject: [PATCH 21/28] revert all changes to include in this branch --- base/client.jl | 2 +- base/docs/basedocs.jl | 2 +- base/loading.jl | 79 ++++++++++++++----------------------------- base/precompile.jl | 4 +-- base/sysimg.jl | 2 +- test/loading.jl | 4 +-- test/runtests.jl | 4 +-- test/testdefs.jl | 4 +-- 8 files changed, 35 insertions(+), 66 deletions(-) diff --git a/base/client.jl b/base/client.jl index 5813aca22a8c0..54fb720ccf30e 100644 --- a/base/client.jl +++ b/base/client.jl @@ -274,7 +274,7 @@ let reqarg = Set(UTF8String["--home", "-H", end # load file immediately on all processors if opts.load != C_NULL - let file = bytestring(abspath(opts.load)) + let file = abspath(bytestring(opts.load)) @everywhere include($file) end end diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 59aaccd42729f..999c106205664 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -368,7 +368,7 @@ function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. """ -base_include +include_from_node1 """ 0 (zero; BrE: `/ˈzɪərəʊ/` or AmE: `/ˈziːroʊ/`) is both a number and the numerical digit used to represent that number in numerals. It fulfills a central role in mathematics as the additive identity of the integers, real numbers, and many other algebraic structures. As a digit, 0 is used as a placeholder in place value systems. Names for the number 0 in English include zero, nought or (US) naught (`/ˈnɔːt/`), nil, or — in contexts where at least one adjacent digit distinguishes it from the letter "O" — oh or o (`/ˈoʊ/`). Informal or slang terms for zero include zilch and zip. Ought and aught (/ˈɔːt/), as well as cipher, have also been used historically. diff --git a/base/loading.jl b/base/loading.jl index 5085dcbafab38..a42ec045126dd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -129,10 +129,10 @@ function require(mod::Symbol, cachecompile::Bool=false) if myid() == 1 && current_module() === Main && nprocs() > 1 # broadcast top-level import/using from node 1 (only) content = open(readall, path) - refs = Any[ @spawnat p eval(Main, :(Base.include_string($content, $path))) for p in procs() ] + refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ] for r in refs; wait(r); end else - eval(Main, :(Base.base_include($path))) + eval(Main, :(Base.include_from_node1($path))) end finally loading = pop!(package_locks, mod) @@ -144,7 +144,15 @@ const reload = require # remote/parallel load -function source_path(default::Union{AbstractString,Void}=nothing) +include_string(txt::ByteString, fname::ByteString) = + ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t), + txt, sizeof(txt), fname, sizeof(fname)) + +include_string(txt::AbstractString, fname::AbstractString) = include_string(bytestring(txt), bytestring(fname)) + +include_string(txt::AbstractString) = include_string(txt, "string") + +function source_path(default::Union{AbstractString,Void}="") t = current_task() while true s = t.storage @@ -158,69 +166,32 @@ function source_path(default::Union{AbstractString,Void}=nothing) end end -macro __FILE__() - return source_path() -end +macro __FILE__() source_path() end -# base_include is the implementation for Base.include -# if relative_to::Void, it uses the dirpath of the previous (recursive) include to find the file -# relative_to is an alternate , where node can be 0 iff the content is from a string (sans a real path) -function base_include(content::Nullable{ByteString}, path::AbstractString, relative_to=nothing) +function include_from_node1(path::AbstractString) prev = source_path(nothing) - local bytepath::ByteString - if !isnull(content) - # already have content, so no need for file-system operations - if relative_to === nothing - bytepath = bytestring(path) - else - bytepath = bytestring(joinpath(relative_to, path)) - end - else - if prev !== nothing && relative_to === nothing - # if this is a recursive include without explicit relative_to location - # pick up previous location for last include - relative_to = dirname(prev) - end - if relative_to === nothing - # no previous path, pick up absolute path location from node 1 - if myid() == 1 || isabspath(path) - bytepath = bytestring(abspath(path)) - else - bytepath = remotecall_fetch(1, path -> bytestring(abspath(path)), path) - end - else - bytepath = bytestring(joinpath(relative_to, path)) - end - end - + path = (prev == nothing) ? abspath(path) : joinpath(dirname(prev),path) tls = task_local_storage() - tls[:SOURCE_PATH] = bytepath + tls[:SOURCE_PATH] = path + local result try - if isnull(content) - if myid() === 1 - # sleep a bit to process requests from other nodes - nprocs()>1 && yield() - result = Core.include(bytepath) - nprocs()>1 && yield() - return result - else - content = Nullable{ByteString}(remotecall_fetch(1, readall, bytepath)) - # fall-through now that content is assigned - end + if myid()==1 + # sleep a bit to process file requests from other nodes + nprocs()>1 && sleep(0.005) + result = Core.include(path) + nprocs()>1 && sleep(0.005) + else + result = include_string(remotecall_fetch(1, readall, path), path) end - txt = get(content) - return ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t), - txt, sizeof(txt), bytepath, sizeof(bytepath)) finally - if prev === nothing + if prev == nothing delete!(tls, :SOURCE_PATH) else tls[:SOURCE_PATH] = prev end end + result end -base_include(path::AbstractString, relative_to=nothing) = base_include(Nullable{ByteString}(), path, relative_to) -include_string(txt::AbstractString, fname::AbstractString="none") = base_include(Nullable{ByteString}(bytestring(txt)), bytestring(fname)) function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[]) return eval(Module(:__anon__), diff --git a/base/precompile.jl b/base/precompile.jl index acb84f1808248..fdfa20deffe4d 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -251,8 +251,8 @@ precompile(Base.ht_keyindex2, (Dict{UInt8, Any}, UInt8)) precompile(Base.in, (Char, ASCIIString)) precompile(Base.in, (Int, Base.UnitRange{Int})) precompile(Base.in, (UInt8, Base.KeyIterator{Dict{UInt8, Any}})) -precompile(Base.base_include, (ASCIIString,)) -precompile(Base.base_include, (UTF8String,)) +precompile(Base.include_from_node1, (ASCIIString,)) +precompile(Base.include_from_node1, (UTF8String,)) precompile(Base.init_stdio, (Ptr{Void},)) precompile(Base.input_color, ()) precompile(Base.insert!, (Array{Any,1}, Int, Base.GlobalRef)) diff --git a/base/sysimg.jl b/base/sysimg.jl index acff7bb926a62..afe363fab0248 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -313,7 +313,7 @@ function __init__() init_parallel() end -include = base_include +include = include_from_node1 include("precompile.jl") end # baremodule Base diff --git a/test/loading.jl b/test/loading.jl index 5a86356decda3..cc6be3aa95f34 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -4,7 +4,7 @@ using Base.Test include("test_sourcepath.jl") thefname = "the fname!//\\&\0\1*" -@test include_string("@__FILE__", thefname) == "$thefname" -@test include_string("@__FILE__") == "none" +@test include_string("include_string_test() = @__FILE__", thefname)() == Base.source_path() +@test include_string("Base.source_path()", thefname) == Base.source_path() @test basename(@__FILE__) == "loading.jl" @test isabspath(@__FILE__) diff --git a/test/runtests.jl b/test/runtests.jl index 4d901bb4ed6d7..edd1a572296e7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,9 +11,7 @@ end @everywhere include("testdefs.jl") -let cwd = pwd() - reduce(propagate_errors, nothing, pmap(test->runtests(test, cwd), tests; err_retry=false, err_stop=true)) -end +reduce(propagate_errors, nothing, pmap(test->runtests(test), tests; err_retry=false, err_stop=true)) @unix_only n > 1 && rmprocs(workers(), waitfor=5.0) println(" \033[32;1mSUCCESS\033[0m") diff --git a/test/testdefs.jl b/test/testdefs.jl index 141d72bdf275f..15c97c57b438f 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -2,9 +2,9 @@ using Base.Test -function runtests(name, cwd) +function runtests(name) @printf(" \033[1m*\033[0m \033[31m%-20s\033[0m", name) - tt = @elapsed include("$name.jl", cwd) + tt = @elapsed include("$name.jl") @printf(" in %6.2f seconds\n", tt) nothing end From 3c58804beb482ab3b5b5c4a9042e024eb15b18f1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 13 Jul 2015 20:24:51 -0400 Subject: [PATCH 22/28] minor cleanups and add `compile` entry point --- base/client.jl | 4 ++-- base/exports.jl | 1 + base/loading.jl | 27 +++++++++++---------------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/base/client.jl b/base/client.jl index 54fb720ccf30e..918d1812961fa 100644 --- a/base/client.jl +++ b/base/client.jl @@ -274,8 +274,8 @@ let reqarg = Set(UTF8String["--home", "-H", end # load file immediately on all processors if opts.load != C_NULL - let file = abspath(bytestring(opts.load)) - @everywhere include($file) + @sync for p in procs() + @async remotecall_fetch(p, include, bytestring(opts.load)) end end # eval expression diff --git a/base/exports.jl b/base/exports.jl index dff4317e1548b..59a97a269a5cf 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1097,6 +1097,7 @@ export include, include_string, reload, + compile, # RTS internals finalizer, diff --git a/base/loading.jl b/base/loading.jl index a42ec045126dd..9d3da57e7029a 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -97,9 +97,8 @@ end # to synchronize multiple tasks trying to import/using something package_locks = Dict{Symbol,Condition}() -# setting cachecompile=true will force a compile of a cache file # require always works in Main scope and loads files from node 1 -function require(mod::Symbol, cachecompile::Bool=false) +function require(mod::Symbol) loading = get(package_locks, mod, false) if loading !== false # load already in progress for this module @@ -109,7 +108,7 @@ function require(mod::Symbol, cachecompile::Bool=false) package_locks[mod] = Condition() try - if !cachecompile && _require_from_serialized(1, mod) + if _require_from_serialized(1, mod) return end @@ -117,15 +116,6 @@ function require(mod::Symbol, cachecompile::Bool=false) path = find_in_node_path(name, 1) path === nothing && throw(ArgumentError("$name not found in path")) - if cachecompile || JLOptions().incremental != 0 - # spawn off a new incremental compile task from node 1 for recursive `require` calls - cachefile = remotecall_fetch(1, create_expr_cache, path, name) - if !_require_from_serialized(1, cachefile) - error("Warning: require failed to create a precompiled cache file") - end - return - end - if myid() == 1 && current_module() === Main && nprocs() > 1 # broadcast top-level import/using from node 1 (only) content = open(readall, path) @@ -203,7 +193,7 @@ function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[]) end evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...]) -function create_expr_cache(m::Expr, name) +function create_expr_cache(file::AbstractString, name) cachepath = LOAD_CACHE_PATH[1] if !isdir(cachepath) mkpath(cachepath) @@ -233,7 +223,7 @@ function create_expr_cache(m::Expr, name) task_local_storage()[:SOURCE_PATH] = $(source) end) end - serialize(io, m) + serialize(io, :(Base.include($(abspath(file))))) if source !== nothing serialize(io, quote delete!(task_local_storage(), :SOURCE_PATH) @@ -244,6 +234,11 @@ function create_expr_cache(m::Expr, name) return cachefile end -function create_expr_cache(file::AbstractString, name) - return create_expr_cache(:(Base.include($(abspath(file)))), name) +compile(mod::Symbol) = compile(string(mod)) +function compile(name::AbstractString) + path = find_in_node_path(name, 1) + path === nothing && throw(ArgumentError("$name not found in path")) + # spawn off a new incremental compile task from node 1 + return remotecall_fetch(1, create_expr_cache, path, name) + # TODO: error/warning on failure? end From 824846c76d0f970cd7d5eea9154c7f03d5019371 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 15 Jul 2015 17:59:00 -0400 Subject: [PATCH 23/28] incremental serialization inferface cleanup --- base/exports.jl | 1 - base/loading.jl | 53 ++++++++++++++++++++++++++++++++----------------- src/init.c | 6 +++--- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 59a97a269a5cf..dff4317e1548b 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1097,7 +1097,6 @@ export include, include_string, reload, - compile, # RTS internals finalizer, diff --git a/base/loading.jl b/base/loading.jl index 9d3da57e7029a..cfda5a66dab21 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -63,7 +63,7 @@ function _require_from_serialized(node::Int, path_to_try::ByteString) refs = Any[ @spawnat p _include_from_serialized(content) for p in others] for (id, ref) in zip(others, refs) if !fetch(ref) - warn("node state is inconsistent: node $id failed to load serialized cache from :node $node:$path_to_try.") + warn("node state is inconsistent: node $id failed to load cache from $path_to_try") end end return true @@ -90,15 +90,26 @@ function _require_from_serialized(node::Int, mod::Symbol) path_to_try === nothing && return false if _require_from_serialized(node, path_to_try) return true + else + warn("deserialization checks failed while attempting to load cache from $path_to_try") end end end # to synchronize multiple tasks trying to import/using something package_locks = Dict{Symbol,Condition}() +package_loaded = Set{Symbol}() # require always works in Main scope and loads files from node 1 function require(mod::Symbol) + if !(mod in package_loaded) + push!(package_loaded, mod) + reload(mod) + end + nothing +end + +function reload(mod::Symbol) loading = get(package_locks, mod, false) if loading !== false # load already in progress for this module @@ -109,13 +120,20 @@ function require(mod::Symbol) try if _require_from_serialized(1, mod) + return true + end + if JLOptions().incremental != 0 + # spawn off a new incremental compile task from node 1 for recursive `require` calls + cachefile = compile(mod) + if !_require_from_serialized(1, cachefile) + warn("require failed to create a precompiled cache file") + end return end name = string(mod) path = find_in_node_path(name, 1) path === nothing && throw(ArgumentError("$name not found in path")) - if myid() == 1 && current_module() === Main && nprocs() > 1 # broadcast top-level import/using from node 1 (only) content = open(readall, path) @@ -130,7 +148,6 @@ function require(mod::Symbol) end nothing end -const reload = require # remote/parallel load @@ -193,19 +210,14 @@ function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[]) end evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...]) -function create_expr_cache(file::AbstractString, name) - cachepath = LOAD_CACHE_PATH[1] - if !isdir(cachepath) - mkpath(cachepath) - end - cachefile = abspath(cachepath, name*".ji") +function create_expr_cache(input::AbstractString, output::AbstractString) code_object = """ while !eof(STDIN) eval(Main, deserialize(STDIN)) end """ io, pobj = open(detach(setenv(`$(julia_cmd()) - --output-ji $cachefile --output-incremental=yes + --output-ji $output --output-incremental=yes --startup-file=no --history-file=no --eval $code_object`, ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w") @@ -223,7 +235,7 @@ function create_expr_cache(file::AbstractString, name) task_local_storage()[:SOURCE_PATH] = $(source) end) end - serialize(io, :(Base.include($(abspath(file))))) + serialize(io, :(Base.include($(abspath(input))))) if source !== nothing serialize(io, quote delete!(task_local_storage(), :SOURCE_PATH) @@ -231,14 +243,19 @@ function create_expr_cache(file::AbstractString, name) end close(io) wait(pobj) - return cachefile + return pobj end -compile(mod::Symbol) = compile(string(mod)) -function compile(name::AbstractString) - path = find_in_node_path(name, 1) +function compile(mod::Symbol) + myid() == 1 || error("can only compile from node 1") + name = string(mod) + path = find_in_path(name) path === nothing && throw(ArgumentError("$name not found in path")) - # spawn off a new incremental compile task from node 1 - return remotecall_fetch(1, create_expr_cache, path, name) - # TODO: error/warning on failure? + cachepath = LOAD_CACHE_PATH[1] + if !isdir(cachepath) + mkpath(cachepath) + end + cachefile = abspath(cachepath, name*".ji") + create_expr_cache(path, cachefile) + return cachefile end diff --git a/src/init.c b/src/init.c index ea07afd31290a..df2dd2670a725 100644 --- a/src/init.c +++ b/src/init.c @@ -1292,15 +1292,15 @@ static void julia_save() if (jl_options.incremental) { jl_array_t *worklist = jl_module_init_order; if (!worklist) { - jl_printf(JL_STDERR, "incremental output requested, but no modules defined during run"); + jl_printf(JL_STDERR, "warning: incremental output requested, but no modules defined during run\n"); return; } if (jl_options.outputji) jl_save_incremental(jl_options.outputji, worklist); if (jl_options.outputbc) - jl_printf(JL_STDERR, "incremental output to a .bc file is not implemented"); + jl_printf(JL_STDERR, "warning: incremental output to a .bc file is not implemented\n"); if (jl_options.outputo) - jl_printf(JL_STDERR, "incremental output to a .o file is not implemented"); + jl_printf(JL_STDERR, "warning: incremental output to a .o file is not implemented\n"); } else { ios_t *s = NULL; From 69acadd06edbe7848485566c0d5caa9a982308df Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jul 2015 13:28:39 -0400 Subject: [PATCH 24/28] avoid serializing ghost fields --- src/dump.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/dump.c b/src/dump.c index 6b4bbe4ca8773..128b1d13c8ab4 100644 --- a/src/dump.c +++ b/src/dump.c @@ -854,7 +854,9 @@ static void jl_serialize_value_(ios_t *s, jl_value_t *v) } else { for(size_t i=0; i < nf; i++) { - jl_serialize_value(s, jl_get_nth_field(v, i)); + if (t->fields[i].size > 0) { + jl_serialize_value(s, jl_get_nth_field(v, i)); + } } } } @@ -1392,19 +1394,16 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t ios_read(s, (char*)jl_data_ptr(v), nby); } else { - if (mode == MODE_AST && dt == jl_globalref_type) { - jl_value_t *mod = jl_deserialize_value(s, NULL); - jl_value_t *var = jl_deserialize_value(s, NULL); - return jl_module_globalref((jl_module_t*)mod, (jl_sym_t*)var); - } char *data = (char*)jl_data_ptr(v); for(i=0; i < nf; i++) { - if (dt->fields[i].isptr) { - jl_value_t **fld = (jl_value_t**)(data+jl_field_offset(dt, i)); - *fld = jl_deserialize_value(s, fld); - } - else { - jl_set_nth_field(v, i, jl_deserialize_value(s, NULL)); + if (dt->fields[i].size > 0) { + if (dt->fields[i].isptr) { + jl_value_t **fld = (jl_value_t**)(data+jl_field_offset(dt, i)); + *fld = jl_deserialize_value(s, fld); + } + else { + jl_set_nth_field(v, i, jl_deserialize_value(s, NULL)); + } } } if ((mode == MODE_MODULE || mode == MODE_MODULE_POSTWORK)) { From 198a29c3c6e568f6c41f40e4f874ade67d4b3519 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jul 2015 15:28:33 -0400 Subject: [PATCH 25/28] restore jl_globalref_type cache optimization for MODE_AST --- src/dump.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/dump.c b/src/dump.c index 128b1d13c8ab4..9941ab8dcdbf6 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1356,13 +1356,19 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return jl_module_globalref(tree_enclosing_module, (jl_sym_t*)sym); } else if (vtag == (jl_value_t*)jl_globalref_type) { - jl_value_t *v = jl_new_struct_uninit(jl_globalref_type); - if (usetable) + if (usetable) { + jl_value_t *v = jl_new_struct_uninit(jl_globalref_type); arraylist_push(&backref_list, v); - jl_value_t* *data = jl_data_ptr(v); - data[0] = jl_deserialize_value(s, &data[0]); - data[1] = jl_deserialize_value(s, &data[1]); - return v; + jl_value_t* *data = jl_data_ptr(v); + data[0] = jl_deserialize_value(s, &data[0]); + data[1] = jl_deserialize_value(s, &data[1]); + return v; + } + else { + jl_value_t *mod = jl_deserialize_value(s, NULL); + jl_value_t *var = jl_deserialize_value(s, NULL); + return jl_module_globalref((jl_module_t*)mod, (jl_sym_t*)var); + } } else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s) : read_int32(s)); From 81d5ee96700a6b0cf71f697b2e44869dbfafa3a7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jul 2015 16:59:40 -0400 Subject: [PATCH 26/28] more agressive error messages when stuff is going to break incremental compilation also fix a bug where you could replace a const global with a new module also fix the handling of module init order in the incremental deserializer for an edge case --- base/loading.jl | 2 +- src/dump.c | 11 +++++++++-- src/toplevel.c | 17 ++++++++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index cfda5a66dab21..7b79fc7006f10 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -220,7 +220,7 @@ function create_expr_cache(input::AbstractString, output::AbstractString) --output-ji $output --output-incremental=yes --startup-file=no --history-file=no --eval $code_object`, - ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w") + ["JULIA_HOME=$JULIA_HOME", "HOME=$(homedir())"])), "w", STDOUT) serialize(io, quote empty!(Base.LOAD_PATH) append!(Base.LOAD_PATH, $LOAD_PATH) diff --git a/src/dump.c b/src/dump.c index 9941ab8dcdbf6..6586a95585bf0 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1555,11 +1555,17 @@ static void jl_reinit_item(ios_t *f, jl_value_t *v, int how) { jl_gc_wb(v, *a); break; } - case 2: { // reinsert v (const) into parent as sym + case 2: { // reinsert module v into parent (const) jl_module_t *mod = (jl_module_t*)v; jl_binding_t *b = jl_get_binding_wr(mod->parent, mod->name); jl_declare_constant(b); // this can throw if (b->value != NULL) { + if (!jl_is_module(b->value)) { + jl_errorf("invalid redefinition of constant %s", mod->name->name); // this also throws + } + if (jl_generating_output() && jl_options.incremental) { + jl_errorf("error: cannot replace module %s during incremental compile", mod->name->name); + } jl_printf(JL_STDERR, "Warning: replacing module %s\n", mod->name->name); } b->value = v; @@ -1939,6 +1945,7 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) int en = jl_gc_enable(0); DUMP_MODES last_mode = mode; mode = MODE_MODULE; + jl_array_t *last_module_init_order = jl_module_init_order; jl_array_t *restored = NULL; restored = (jl_array_t*)jl_deserialize_value(f, (jl_value_t**)&restored); @@ -2049,7 +2056,7 @@ static jl_array_t *_jl_restore_incremental(ios_t *f) JL_SIGATOMIC_END(); jl_init_restored_modules(); - jl_module_init_order = NULL; + jl_module_init_order = last_module_init_order; return restored; } diff --git a/src/toplevel.c b/src/toplevel.c index f4f65f6cdaa57..f849e201bb631 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -48,6 +48,9 @@ void jl_add_standard_imports(jl_module_t *m) jl_module_t *jl_new_main_module(void) { + if (jl_generating_output() && jl_options.incremental) + jl_error("cannot call workspace() in incremental compile mode"); + // switch to a new top-level module if (jl_current_module != jl_main_module && jl_current_module != NULL) jl_error("Main can only be replaced from the top level"); @@ -109,9 +112,17 @@ jl_value_t *jl_eval_module_expr(jl_expr_t *ex) jl_module_t *parent_module = jl_current_module; jl_binding_t *b = jl_get_binding_wr(parent_module, name); jl_declare_constant(b); - if (b->value != NULL && !jl_generating_output()) { - // suppress warning "replacing module Core.Inference" during bootstrapping - jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); + if (b->value != NULL) { + if (!jl_is_module(b->value)) { + jl_errorf("invalid redefinition of constant %s", name->name); + } + if (jl_generating_output() && jl_options.incremental) { + jl_errorf("error: cannot replace module %s during incremental compile", name->name); + } + if (!jl_generating_output()) { + // suppress warning "replacing module Core.Inference" during bootstrapping + jl_printf(JL_STDERR, "Warning: replacing module %s\n", name->name); + } } jl_module_t *newm = jl_new_module(name); newm->parent = parent_module; From 1cc8eb3776a82494a9deb5e516d297983a2fa3b7 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Jul 2015 22:54:38 -0400 Subject: [PATCH 27/28] test toplevel_load directly, rather than using current_module() == Main to determine whether to broadcast require --- base/deprecated.jl | 2 ++ base/exports.jl | 1 - base/loading.jl | 27 ++++++++++++--------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 87888ff3a1ebf..90f4f92f0edf6 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -682,3 +682,5 @@ function require_filename(name::AbstractString) end name end +const reload = require +export reload diff --git a/base/exports.jl b/base/exports.jl index dff4317e1548b..4dfe4b6d5088f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1096,7 +1096,6 @@ export evalfile, include, include_string, - reload, # RTS internals finalizer, diff --git a/base/loading.jl b/base/loading.jl index 7b79fc7006f10..45467c526d861 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -50,8 +50,8 @@ function _include_from_serialized(content::Vector{UInt8}) return m != 0 end -function _require_from_serialized(node::Int, path_to_try::ByteString) - if myid() == 1 && current_module() === Main && nprocs() > 1 +function _require_from_serialized(node::Int, path_to_try::ByteString, toplevel_load::Bool) + if toplevel_load && myid() == 1 && nprocs() > 1 # broadcast top-level import/using from node 1 (only) if node == myid() content = open(readbytes, path_to_try) @@ -82,13 +82,13 @@ function _require_from_serialized(node::Int, path_to_try::ByteString) return false end -function _require_from_serialized(node::Int, mod::Symbol) +function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool) name = string(mod) finder = @spawnat node @task find_in_cache_path(mod) # TODO: switch this to an explicit Channel while true path_to_try = remotecall_fetch(node, finder->consume(fetch(finder)), finder) path_to_try === nothing && return false - if _require_from_serialized(node, path_to_try) + if _require_from_serialized(node, path_to_try, toplevel_load) return true else warn("deserialization checks failed while attempting to load cache from $path_to_try") @@ -101,15 +101,9 @@ package_locks = Dict{Symbol,Condition}() package_loaded = Set{Symbol}() # require always works in Main scope and loads files from node 1 +toplevel_load = true function require(mod::Symbol) - if !(mod in package_loaded) - push!(package_loaded, mod) - reload(mod) - end - nothing -end - -function reload(mod::Symbol) + global toplevel_load loading = get(package_locks, mod, false) if loading !== false # load already in progress for this module @@ -118,14 +112,16 @@ function reload(mod::Symbol) end package_locks[mod] = Condition() + last = toplevel_load try - if _require_from_serialized(1, mod) + toplevel_load = false + if _require_from_serialized(1, mod, last) return true end if JLOptions().incremental != 0 # spawn off a new incremental compile task from node 1 for recursive `require` calls cachefile = compile(mod) - if !_require_from_serialized(1, cachefile) + if !_require_from_serialized(1, cachefile, last) warn("require failed to create a precompiled cache file") end return @@ -134,7 +130,7 @@ function reload(mod::Symbol) name = string(mod) path = find_in_node_path(name, 1) path === nothing && throw(ArgumentError("$name not found in path")) - if myid() == 1 && current_module() === Main && nprocs() > 1 + if last && myid() == 1 && nprocs() > 1 # broadcast top-level import/using from node 1 (only) content = open(readall, path) refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in procs() ] @@ -143,6 +139,7 @@ function reload(mod::Symbol) eval(Main, :(Base.include_from_node1($path))) end finally + toplevel_load = last loading = pop!(package_locks, mod) notify(loading, all=true) end From e2d842adeadfb6faca33260fd4190d89066bcafc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 17 Jul 2015 00:11:41 -0400 Subject: [PATCH 28/28] add incremental compile docs --- NEWS.md | 8 +++ base/client.jl | 2 +- doc/helpdb.jl | 160 ++++++++++++++++++++++++++--------------- doc/manual/modules.rst | 121 +++++++++++++++++++++++++------ doc/stdlib/base.rst | 12 ++-- 5 files changed, 221 insertions(+), 82 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3bd56ed843d6b..af23f70a0d3e2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -31,6 +31,12 @@ New language features * The syntax `function foo end` can be used to introduce a generic function without yet adding any methods ([#8283]). + * Incremental compilation of modules: ``Base.compile(module::Symbol)`` (stored in `~/.julia/lib/v0.4`) + + * See manual section on `Module initialization and precompilation` (under `Modules`) for details and errata. + + * New option `--compile-incremental={yes|no}` added to invoke the equivalent of ``Base.compile`` from the command line. + Language changes ---------------- @@ -450,6 +456,8 @@ Deprecated or removed * `sync_gc_total_bytes` -> `jl_gc_sync_total_bytes` + * `require(::AbstractString)` and `reload` (see news about addition of `compile`) + Julia v0.3.0 Release Notes ========================== diff --git a/base/client.jl b/base/client.jl index 918d1812961fa..77005e5f352fe 100644 --- a/base/client.jl +++ b/base/client.jl @@ -332,7 +332,7 @@ function init_load_path() end push!(LOAD_PATH,abspath(JULIA_HOME,"..","local","share","julia","site",vers)) push!(LOAD_PATH,abspath(JULIA_HOME,"..","share","julia","site",vers)) - push!(LOAD_CACHE_PATH,abspath(homedir(),".julia","libs",vers)) + push!(LOAD_CACHE_PATH,abspath(homedir(),".julia","lib",vers)) push!(LOAD_CACHE_PATH,abspath(JULIA_HOME,"..","usr","lib","julia")) #TODO: fixme end diff --git a/doc/helpdb.jl b/doc/helpdb.jl index c7bf85f074ac5..fc986af5dcbe3 100644 --- a/doc/helpdb.jl +++ b/doc/helpdb.jl @@ -1308,9 +1308,15 @@ Any[ "), -("Base","require","require(file::AbstractString...) +("Base","require","require(module::Symbol) - Load source files once, in the context of the \"Main\" module, on + This function is part of the implementation of \"using\" / + \"import\", if a module is not already defined in \"Main\". It can + also be called directly to force reloading a module, regardless of + whether it has been loaded before (for exmple, when interactively + developing libraries). + + Loads a source files, in the context of the \"Main\" module, on every active node, searching standard locations for files. \"require\" is considered a top-level operation, so it sets the current \"include\" path but does not use it to search for files @@ -1324,11 +1330,14 @@ Any[ "), -("Base","reload","reload(file::AbstractString) +("Base","compile","compile(module::String) - Like \"require\", except forces loading of files regardless of - whether they have been loaded before. Typically used when - interactively developing libraries. + Creates a precompiled cache file for module (see help for + \"require\") and all of its dependencies. This can be used to + reduce package load times. Cache files are stored in + LOAD_CACHE_PATH[1], which defaults to *~/.julia/lib/VERSION*. See + the manual section *Module initialization and precompilation* + (under *Modules*) for important notes. "), @@ -1345,7 +1354,7 @@ Any[ "), -("Base","include_string","include_string(code::AbstractString) +("Base","include_string","include_string(code::AbstractString[, filename]) Like \"include\", except reads code from the given string rather than from a file. Since there is no file path involved, no path @@ -5701,18 +5710,26 @@ Millisecond(v) "), -("Base","readbytes!","readbytes!(stream, b::Vector{UInt8}, nb=length(b)) +("Base","readbytes!","readbytes!(stream, b::Vector{UInt8}, nb=length(b); all=true) Read at most \"nb\" bytes from the stream into \"b\", returning the number of bytes read (increasing the size of \"b\" as needed). + See \"readbytes\" for a description of the \"all\" option. + "), -("Base","readbytes","readbytes(stream, nb=typemax(Int)) +("Base","readbytes","readbytes(stream, nb=typemax(Int); all=true) Read at most \"nb\" bytes from the stream, returning a \"Vector{UInt8}\" of the bytes read. + If \"all\" is true (the default), this function will block + repeatedly trying to read all requested bytes, until an error or + end-of-file occurs. If \"all\" is false, at most one \"read\" call + is performed, and the amount of data returned is device-dependent. + Note that not all stream types support the \"all\" option. + "), ("Base","position","position(s) @@ -5920,8 +5937,8 @@ Millisecond(v) ("Base","countlines","countlines(io[, eol::Char]) - Read io until the end of the stream/file and count the number of - non-empty lines. To specify a file pass the filename as the first + Read \"io\" until the end of the stream/file and count the number + of lines. To specify a file pass the filename as the first argument. EOL markers other than '\\n' are supported by passing them as the second argument. @@ -6387,31 +6404,55 @@ popdisplay(d::Display) "), -("Base","mmap_array","mmap_array(type, dims, stream[, offset]) +("Base","Mmap","Mmap.Anonymous(name, readonly, create) + + Create an \"IO\"-like object for creating zeroed-out mmapped-memory + that is not tied to a file for use in \"Mmap.mmap\". Used by + \"SharedArray\" for creating shared memory arrays. + +"), + +("Base","Mmap","Mmap.mmap(io::Union(IOStream,AbstractString,Mmap.AnonymousMmap)[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) +Mmap.mmap(type::Type{Array{T, N}}, dims) Create an \"Array\" whose values are linked to a file, using memory-mapping. This provides a convenient way of working with data too large to fit in the computer's memory. - The type determines how the bytes of the array are interpreted. - Note that the file must be stored in binary format, and no format - conversions are possible (this is a limitation of operating - systems, not Julia). + The type is an \"Array{T,N}\" with a bits-type element of \"T\" and + dimension \"N\" that determines how the bytes of the array are + interpreted. Note that the file must be stored in binary format, + and no format conversions are possible (this is a limitation of + operating systems, not Julia). + + \"dims\" is a tuple or single \"Integer\" specifying the size or + length of the array. - \"dims\" is a tuple specifying the size of the array. + The file is passed via the stream argument, either as an open + \"IOStream\" or filename string. When you initialize the stream, + use \"\"r\"\" for a \"read-only\" array, and \"\"w+\"\" to create a + new array used to write values to disk. - The file is passed via the stream argument. When you initialize - the stream, use \"\"r\"\" for a \"read-only\" array, and \"\"w+\"\" - to create a new array used to write values to disk. + If no \"type\" argument is specified, the default is + \"Vector{UInt8}\". Optionally, you can specify an offset (in bytes) if, for example, you want to skip over a header in the file. The default value for - the offset is the current stream position. + the offset is the current stream position for an \"IOStream\". + + The \"grow\" keyword argument specifies whether the disk file + should be grown to accomodate the requested size of array (if the + total file size is < requested array size). Write privileges are + required to grow the file. + + The \"shared\" keyword argument specifies whether the resulting + \"Array\" and changes made to it will be visible to other processes + mapping the same file. For example, the following code: # Create a file for mmapping - # (you could alternatively use mmap_array to do this step, too) + # (you could alternatively use mmap to do this step, too) A = rand(1:20, 5, 30) s = open(\"/tmp/mmap.bin\", \"w+\") # We'll write the dimensions of the array as the first two Ints in the file @@ -6425,7 +6466,7 @@ popdisplay(d::Display) s = open(\"/tmp/mmap.bin\") # default is read-only m = read(s, Int) n = read(s, Int) - A2 = mmap_array(Int, (m,n), s) + A2 = Mmap.mmap(s, Matrix{Int}, (m,n)) creates a \"m\"-by-\"n\" \"Matrix{Int}\", linked to the file associated with stream \"s\". @@ -6437,22 +6478,21 @@ popdisplay(d::Display) "), -("Base","mmap_bitarray","mmap_bitarray([type], dims, stream[, offset]) +("Base","Mmap","Mmap.mmap(io, BitArray[, dims, offset]) Create a \"BitArray\" whose values are linked to a file, using memory-mapping; it has the same purpose, works in the same way, and - has the same arguments, as \"mmap_array()\", but the byte - representation is different. The \"type\" parameter is optional, - and must be \"Bool\" if given. + has the same arguments, as \"mmap()\", but the byte representation + is different. - **Example**: \"B = mmap_bitarray((25,30000), s)\" + **Example**: \"B = Mmap.mmap(s, BitArray, (25,30000))\" This would create a 25-by-30000 \"BitArray\", linked to the file associated with stream \"s\". "), -("Base","msync","msync(array) +("Base","Mmap","Mmap.sync!(array) Forces synchronization between the in-memory version of a memory- mapped \"Array\" or \"BitArray\" and the on-disk version. @@ -6794,12 +6834,18 @@ popdisplay(d::Display) \"RTLD_FIRST\". These are converted to the corresponding flags of the POSIX (and/or GNU libc and/or MacOS) dlopen command, if possible, or are ignored if the specified functionality is not - available on the current platform. The default is - \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL\". An important usage of - these flags, on POSIX platforms, is to specify - \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL\" in order for the library's - symbols to be available for usage in other shared libraries, in - situations where there are dependencies between shared libraries. + available on the current platform. The default flags are platform + specific. On MacOS the default \"dlopen\" flags are + \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL\" while on other platforms + the defaults are \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_LOCAL\". An + important usage of these flags is to specify non default behavior + for when the dynamic library loader binds library references to + exported symbols and if the bound references are put into process + local or global scope. For instance + \"RTLD_LAZY|RTLD_DEEPBIND|RTLD_GLOBAL\" allows the library's + symbols to be available for usage in other shared libraries, + addressing situations where there are dependencies between shared + libraries. "), @@ -6991,15 +7037,15 @@ popdisplay(d::Display) \"Complex{Float64}\" the return type is \"UmfpackLU\". Some examples are shown in the table below. - +-------------------------+---------------------------+----------------------------------------------+ - | Type of input \\\"A\\\" | Type of output \\\"F\\\" | Relationship between \\\"F\\\" and \\\"A\\\" | - +-------------------------+---------------------------+----------------------------------------------+ - | \\\"Matrix()\\\" | \\\"LU\\\" | \\\"F[:L]*F[:U] == A[F[:p], :]\\\" | - +-------------------------+---------------------------+----------------------------------------------+ - | \\\"Tridiagonal()\\\" | \\\"LU{T,Tridiagonal{T}}\\\" | N/A | - +-------------------------+---------------------------+----------------------------------------------+ - | \\\"SparseMatrixCSC()\\\" | \\\"UmfpackLU\\\" | \\\"F[:L]*F[:U] == F[:Rs] .* A[F[:p], F[:q]]\\\" | - +-------------------------+---------------------------+----------------------------------------------+ + +-------------------------+---------------------------+------------------------------------------------+ + | Type of input \\\"A\\\" | Type of output \\\"F\\\" | Relationship between \\\"F\\\" and \\\"A\\\" | + +-------------------------+---------------------------+------------------------------------------------+ + | \\\"Matrix()\\\" | \\\"LU\\\" | \\\"F[:L]*F[:U] == A[F[:p], :]\\\" | + +-------------------------+---------------------------+------------------------------------------------+ + | \\\"Tridiagonal()\\\" | \\\"LU{T,Tridiagonal{T}}\\\" | N/A | + +-------------------------+---------------------------+------------------------------------------------+ + | \\\"SparseMatrixCSC()\\\" | \\\"UmfpackLU\\\" | \\\"F[:L]*F[:U] == (F[:Rs] .* A)[F[:p], F[:q]]\\\" | + +-------------------------+---------------------------+------------------------------------------------+ The individual components of the factorization \"F\" can be accessed by indexing: @@ -10278,12 +10324,12 @@ popdisplay(d::Display) "), -("Base","median","median(v) +("Base","median","median(v[, region]) - Compute the median of a vector \"v\". \"NaN\" is returned if the - data contains any \"NaN\" values. For applications requiring the - handling of missing data, the \"DataArrays\" package is - recommended. + Compute the median of whole array \"v\", or optionally along the + dimensions in \"region\". \"NaN\" is returned if the data contains + any \"NaN\" values. For applications requiring the handling of + missing data, the \"DataArrays\" package is recommended. "), @@ -10562,7 +10608,7 @@ popdisplay(d::Display) the transform has conjugate symmetry in order to save roughly half the computational time and storage costs compared with \"fft()\". If \"A\" has size \"(n_1, ..., n_d)\", the result has size - \"(floor(n_1/2)+1, ..., n_d)\". + \"(div(n_1,2)+1, ..., n_d)\". The optional \"dims\" argument specifies an iterable subset of one or more dimensions of \"A\" to transform, similar to \"fft()\". @@ -10580,10 +10626,11 @@ popdisplay(d::Display) transform, defaulting to \"1:ndims(A)\". \"d\" is the length of the transformed real array along the - \"dims[1]\" dimension, which must satisfy \"d == - floor(size(A,dims[1])/2)+1\". (This parameter cannot be inferred - from \"size(A)\" due to the possibility of rounding by the - \"floor\" function here.) + \"dims[1]\" dimension, which must satisfy \"div(d,2)+1 == + size(A,dims[1])\". (This parameter cannot be inferred from + \"size(A)\" since both \"2*size(A,dims[1])-2\" as well as + \"2*size(A,dims[1])-1\" are valid sizes for the transformed real + array.) "), @@ -13235,8 +13282,9 @@ golden ("Base.Test","@test_throws","@test_throws(extype, ex) Test that the expression \"ex\" throws an exception of type - \"extype\" and calls the current handler to handle the result. - The default handler returns the exception if it is of the expected type. + \"extype\" and calls the current handler to handle the result. The + default handler returns the exception if it is of the expected + type. "), diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index 09e2acb909653..aa47052141b31 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -254,13 +254,21 @@ Module initialization and precompilation ---------------------------------------- Large modules can take several second to load because executing all of -the statements in a module often involves compiling a large amount of -code. However, Julia is progressively gaining more ability to cache -the parsed and compiled binary image of a package. Currently, this -requires one to recompile Julia after modifying the file -``base/userimg.jl`` to require the desired modules, but in a future -version of Julia the module caching will be simpler and more -automated. In order to make your module work with precompilation, +the statements in a module often involves compiling a large amount of code. +Julia provides the ability to create precompiled versions of modules +to reduce this time. + +There are two mechanisms that can achieve this: +incremental compile and custom system image. + +To create a custom system image that can be used to start julia with the -J option, +recompile Julia after modifying the file ``base/userimg.jl`` to require the desired modules. + +To create an incremental precompiled module file, +call ``Base.compile(modulename::Symbol)``. +The resulting cache files will be stored in ``Base.LOAD_CACHE_PATH[1]``. + +In order to make your module work with precompilation, however, you may need to change your module to explicitly separate any initialization steps that must occur at *runtime* from steps that can occur at *compile time*. For this purpose, Julia allows you to define @@ -293,18 +301,20 @@ in your module:: global const foo_data_ptr = ccall((:foo_data,:libfoo), Ptr{Void}, ()) end -(Notice that it is perfectly possible to define a global constant inside +Notice that it is perfectly possible to define a global inside a function like ``__init__``; this is one of the advantages of using a -dynamic language.) Obviously, any other constant in your module that -depends on ``foo_data_ptr`` would also have to be initialized in ``__init__``. +dynamic language. +Obviously, any other globals in your module that depends on ``foo_data_ptr`` +would also have to be initialized in ``__init__``. Constants involving most Julia objects that are not produced by ``ccall`` do not need to be placed in ``__init__``: their definitions -can be precompiled and loaded from the cached module image. (This -includes complicated heap-allocated objects like arrays.) However, -any routine that returns a raw pointer value must be called at runtime -for precompilation to work. This includes the Julia functions -``cfunction`` and ``pointer``. +can be precompiled and loaded from the cached module image. +This includes complicated heap-allocated objects like arrays. +However, any routine that returns a raw pointer value must be called +at runtime for precompilation to work +(Ptr objects will turn into null pointers unless they are hidden inside an isbits object). +This includes the return values of the Julia functions ``cfunction`` and ``pointer``. Dictionary and set types, or in general anything that depends on the output of a ``hash(key)`` method, are a trickier case. In the common @@ -314,9 +324,80 @@ they are safe to precompile. However, for a few other key types, such as ``Function`` or ``DataType`` and generic user-defined types where you haven't defined a ``hash`` method, the fallback ``hash`` method depends on the memory address of the object (via its ``object_id``) -and hence may change from run to run. If you have one of these key -types, or if you aren't sure, to be safe you can initialize dictionary -and set globals from within your ``__init__`` function. -Alternatively, you can use the ``ObjectIdDict`` dictionary type, which -is specially handled by precompilation so that it is safe to +and hence may change from run to run. +If you have one of these key types, or if you aren't sure, +to be safe you can initialize this dictionary from within your +``__init__`` function. +Alternatively, you can use the ``ObjectIdDict`` dictionary type, +which is specially handled by precompilation so that it is safe to initialize at compile-time. + +When using precompilation, it is important to keep a clear sense of the +distinction between the compilation phase and the execution phase. +In this mode, it will often be much more clearly apparent that +Julia is a compiler which allows execution of arbitrary Julia code, +not a standalone interpreter that also generates compiled code. + +Other known potential failure scenarios include: + +1. Global counters (for example, for uniquely identify objects) + Consider the following code snippet:: + + type UniquedById + myid::Int + let counter = 0 + UniquedById() = new(counter += 1) + end + end + + while the intent of this code was to give every instance a unique id, + the counter value is recorded at the end of compilation. + All subsequent usages of this incrementally compiled module + will start from that same counter value. + + One alternative is to store both ``current_module()`` and the current ``counter`` value, + however, it may be better to redesign the code to not depend on this global state. + +2. Associative collections (such as ``Dict`` and ``Set``) need to be re-hashed in ``__init__``. + (In the future, a mechanism may be provided to register an initializer function.) + +3. Depending on compile-time side-effects persisting through load-time. + Example include: + modifying arrays or other variables in other Julia modules; + maintaining handles to open files or devices; + storing pointers to other system resources (including memory); + +4. Creating accidental "copies" of global state from another module, + by referencing it directly instead of via its lookup path. + For example, (in global scope):: + #mystdout = Base.STDOUT #= will not work correctly, since this will copy Base.STDOUT into this module =# + # instead use accessor functions: + getstdout() = Base.STDOUT #= best option =# + # or move the assignment into the runtime: + __init__() = global mystdout = Base.STDOUT #= also works =# + +Several additional restrictions are placed on the operations that can be done while compiling code +to help the user avoid other wrong-behavior situations: + +1. Calling ``eval`` to cause a side-effect in another module. + This will also cause a warning to be emitted when the incremental compile flag is set. + +2. ``global const`` statements from local scope after ``__init__()`` has been started (see issue #12010 for plans to add an error for this) + +3. Replacing a module (or calling ``workspace()`` is a runtime error while doing an incremental compile. + +A few other points to be aware of: + +1. No code reload / cache invalidation is performed after changes are made to the source files themselves, + (including by ``Pkg.update``), and no cleanup is done after ``Pkg.rm`` + +2. The memory sharing behavior of a reshaped array is disregarded by precompilation (each view gets its own copy) + +3. Expecting the filesystem to be unchanged between compile-time and runtime + e.g. ``@__FILE__``/``source_path()`` to find resources at runtime, + or the BinDeps ``@checked_lib`` macro. Sometimes this is unavoidable. + However, when possible, it can be good practice to copy resources + into the module at compile-time so they won't need to be found at runtime. + +4. WeakRef objects and finalizers are not captured by currently handled by the serializer + (this will be fixed in an upcoming release). diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index e7ead054d2064..5ce55500c1167 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -78,21 +78,23 @@ Getting Around Return a string with the contents of the operating system clipboard ("paste"). -.. function:: require(file::AbstractString...) +.. function:: require(module::Symbol) - Load source files once, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. + This function is part of the implementation of ``using`` / ``import``, if a module is not already defined in ``Main``. It can also be called directly to force reloading a module, regardless of whether it has been loaded before (for exmple, when interactively developing libraries). + + Loads a source files, in the context of the ``Main`` module, on every active node, searching standard locations for files. ``require`` is considered a top-level operation, so it sets the current ``include`` path but does not use it to search for files (see help for ``include``). This function is typically used to load library code, and is implicitly called by ``using`` to load packages. When searching for files, ``require`` first looks in the current working directory, then looks for package code under ``Pkg.dir()``, then tries paths in the global array ``LOAD_PATH``. -.. function:: reload(file::AbstractString) +.. function:: compile(module::String) - Like ``require``, except forces loading of files regardless of whether they have been loaded before. Typically used when interactively developing libraries. + Creates a precompiled cache file for module (see help for ``require``) and all of its dependencies. This can be used to reduce package load times. Cache files are stored in LOAD_CACHE_PATH[1], which defaults to `~/.julia/lib/VERSION`. See the manual section `Module initialization and precompilation` (under `Modules`) for important notes. .. function:: include(path::AbstractString) Evaluate the contents of a source file in the current context. During including, a task-local include path is set to the directory containing the file. Nested calls to ``include`` will search relative to that path. All paths refer to files on node 1 when running in parallel, and files will be fetched from node 1. This function is typically used to load source interactively, or to combine files in packages that are broken into multiple source files. -.. function:: include_string(code::AbstractString) +.. function:: include_string(code::AbstractString, [filename]) Like ``include``, except reads code from the given string rather than from a file. Since there is no file path involved, no path processing or fetching from node 1 is done.