Skip to content

Commit 9bb8af8

Browse files
vtjnashtkelman
authored andcommitted
incremental deserialize: handle LambdaInfo identity uniquing
this works to avoid having `Expr(:invoke)` creating unintentional copies of LambdaInfo objects when they show up in the system image fix #18184 (cherry picked from commit b0e692a) ref #18191
1 parent e8285b9 commit 9bb8af8

File tree

7 files changed

+160
-49
lines changed

7 files changed

+160
-49
lines changed

doc/manual/modules.rst

+8
Original file line numberDiff line numberDiff line change
@@ -422,6 +422,14 @@ A few other points to be aware of:
422422
4. WeakRef objects and finalizers are not currently handled properly by the serializer
423423
(this will be fixed in an upcoming release).
424424

425+
5. It is usually best to avoid capturing references to instances of internal metadata objects such as
426+
Method, LambdaInfo, MethodTable, TypeMapLevel, TypeMapEntry
427+
and fields of those objects, as this can confuse the serializer
428+
and may not lead to the outcome you desire.
429+
It is not necessarily an error to do this,
430+
but you simply need to be prepared that the system will
431+
try to copy some of these and to create a single unique instance of others.
432+
425433
It is sometimes helpful during module development to turn off incremental precompilation.
426434
The command line flag ``--compilecache={yes|no}`` enables you to toggle module precompilation on and off.
427435
When Julia is started with ``--compilecache=no`` the serialized modules in the compile cache are ignored when loading modules and module dependencies.

src/alloc.c

+8-1
Original file line numberDiff line numberDiff line change
@@ -535,7 +535,7 @@ static jl_lambda_info_t *jl_copy_lambda(jl_lambda_info_t *linfo)
535535
}
536536

537537
// return a new lambda-info that has some extra static parameters merged in
538-
JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp)
538+
JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t *types, jl_svec_t *sp, int allow_exec)
539539
{
540540
jl_lambda_info_t *linfo = m->lambda_template;
541541
jl_lambda_info_t *new_linfo;
@@ -547,6 +547,13 @@ JL_DLLEXPORT jl_lambda_info_t *jl_get_specialized(jl_method_t *m, jl_tupletype_t
547547
new_linfo->def = m;
548548
new_linfo->sparam_vals = sp;
549549
}
550+
else if (!allow_exec) {
551+
new_linfo = jl_copy_lambda(linfo);
552+
new_linfo->specTypes = types;
553+
new_linfo->def = m;
554+
new_linfo->sparam_vals = sp;
555+
jl_set_lambda_code_null(new_linfo);
556+
}
550557
else {
551558
new_linfo = jl_instantiate_staged(m, types, sp);
552559
}

src/codegen.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -1050,7 +1050,7 @@ void *jl_get_llvmf(jl_tupletype_t *tt, bool getwrapper, bool getdeclarations)
10501050
linfo = jl_get_specialization1(tt);
10511051
if (linfo == NULL) {
10521052
linfo = jl_method_lookup_by_type(
1053-
((jl_datatype_t*)jl_tparam0(tt))->name->mt, tt, 0, 0);
1053+
((jl_datatype_t*)jl_tparam0(tt))->name->mt, tt, 0, 0, 1);
10541054
if (linfo == NULL || jl_has_call_ambiguities(tt, linfo->def)) {
10551055
JL_GC_POP();
10561056
return NULL;

src/dump.c

+119-29
Original file line numberDiff line numberDiff line change
@@ -854,6 +854,11 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
854854
writetag(s->s, jl_method_type);
855855
jl_method_t *m = (jl_method_t*)v;
856856
union jl_typemap_t *tf = &m->specializations;
857+
if (s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) {
858+
int external = !module_in_worklist(m->module);
859+
if (external)
860+
jl_error("support for serializing a direct reference to an external Method not implemented");
861+
}
857862
if (tf->unknown && tf->unknown != jl_nothing) {
858863
// go through the t-func cache, replacing ASTs with just return
859864
// types for abstract argument types. these ASTs are generally
@@ -879,6 +884,19 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
879884
else if (jl_is_lambda_info(v)) {
880885
writetag(s->s, jl_lambda_info_type);
881886
jl_lambda_info_t *li = (jl_lambda_info_t*)v;
887+
jl_serialize_value(s, (jl_value_t*)li->specTypes);
888+
write_int8(s->s, li->inferred);
889+
if (s->mode == MODE_MODULE || s->mode == MODE_MODULE_POSTWORK) {
890+
int external = li->def && !module_in_worklist(li->def->module);
891+
write_uint8(s->s, external);
892+
if (external) {
893+
// also flag this in the backref table as special
894+
uintptr_t *bp = (uintptr_t*)ptrhash_bp(&backref_table, v);
895+
assert(*bp != (uintptr_t)HT_NOTFOUND);
896+
*bp |= 1; assert(((uintptr_t)HT_NOTFOUND)|1);
897+
return;
898+
}
899+
}
882900
if (li->jlcall_api == 2)
883901
jl_serialize_value(s, jl_nothing);
884902
else
@@ -890,8 +908,6 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
890908
jl_serialize_value(s, li->rettype);
891909
jl_serialize_value(s, (jl_value_t*)li->sparam_syms);
892910
jl_serialize_value(s, (jl_value_t*)li->sparam_vals);
893-
jl_serialize_value(s, (jl_value_t*)li->specTypes);
894-
write_int8(s->s, li->inferred);
895911
write_int8(s->s, li->pure);
896912
write_int8(s->s, li->inlineable);
897913
write_int8(s->s, li->isva);
@@ -1389,7 +1405,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
13891405
isunboxed = !(elsize>>15);
13901406
elsize = elsize&0x7fff;
13911407
}
1392-
int pos = backref_list.len;
1408+
uintptr_t pos = backref_list.len;
13931409
if (usetable)
13941410
arraylist_push(&backref_list, NULL);
13951411
size_t *dims = (size_t*)alloca(ndims*sizeof(size_t));
@@ -1452,6 +1468,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
14521468
jl_method_t *m =
14531469
(jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t),
14541470
jl_method_type);
1471+
memset(m, 0, sizeof(jl_method_type));
14551472
if (usetable)
14561473
arraylist_push(&backref_list, m);
14571474
m->specializations.unknown = jl_deserialize_value(s, (jl_value_t**)&m->specializations);
@@ -1490,8 +1507,42 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
14901507
jl_lambda_info_t *li =
14911508
(jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t),
14921509
jl_lambda_info_type);
1510+
memset(li, 0, sizeof(jl_lambda_info_t));
1511+
uintptr_t pos = backref_list.len;
14931512
if (usetable)
14941513
arraylist_push(&backref_list, li);
1514+
1515+
li->specTypes = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&li->specTypes);
1516+
if (li->specTypes) jl_gc_wb(li, li->specTypes);
1517+
int inferred = read_int8(s->s);
1518+
li->inferred = inferred;
1519+
1520+
if (s->mode == MODE_MODULE) {
1521+
int external = read_uint8(s->s);
1522+
if (external) {
1523+
assert(loc != NULL);
1524+
arraylist_push(&flagref_list, loc);
1525+
arraylist_push(&flagref_list, (void*)pos);
1526+
return (jl_value_t*)li;
1527+
}
1528+
}
1529+
if (s->mode == MODE_MODULE_POSTWORK) {
1530+
int external = read_uint8(s->s);
1531+
if (external) {
1532+
jl_datatype_t *ftype = jl_first_argument_datatype((jl_value_t*)li->specTypes);
1533+
jl_methtable_t *mt = ftype->name->mt;
1534+
li = jl_method_lookup_by_type(mt, li->specTypes, 1, 0, 0);
1535+
assert(li);
1536+
backref_list.items[pos] = li;
1537+
// if it can be inferred but isn't, encourage codegen to infer it
1538+
if (inferred && !li->inferred) {
1539+
jl_set_lambda_code_null(li);
1540+
li->inferred = 1;
1541+
}
1542+
return (jl_value_t*)li;
1543+
}
1544+
}
1545+
14951546
li->code = jl_deserialize_value(s, &li->code); jl_gc_wb(li, li->code);
14961547
li->slotnames = (jl_array_t*)jl_deserialize_value(s, (jl_value_t**)&li->slotnames); jl_gc_wb(li, li->slotnames);
14971548
li->slottypes = jl_deserialize_value(s, &li->slottypes); jl_gc_wb(li, li->slottypes);
@@ -1503,10 +1554,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
15031554
jl_gc_wb(li, li->sparam_syms);
15041555
li->sparam_vals = (jl_svec_t*)jl_deserialize_value(s, (jl_value_t**)&li->sparam_vals);
15051556
jl_gc_wb(li, li->sparam_vals);
1506-
li->specTypes = (jl_tupletype_t*)jl_deserialize_value(s, (jl_value_t**)&li->specTypes);
1507-
if (li->specTypes) jl_gc_wb(li, li->specTypes);
15081557
li->unspecialized_ducttape = NULL;
1509-
li->inferred = read_int8(s->s);
15101558
li->pure = read_int8(s->s);
15111559
li->inlineable = read_int8(s->s);
15121560
li->isva = read_int8(s->s);
@@ -1530,7 +1578,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
15301578
return (jl_value_t*)li;
15311579
}
15321580
else if (vtag == (jl_value_t*)jl_module_type) {
1533-
int pos = backref_list.len;
1581+
uintptr_t pos = backref_list.len;
15341582
if (usetable)
15351583
arraylist_push(&backref_list, NULL);
15361584
jl_sym_t *mname = (jl_sym_t*)jl_deserialize_value(s, NULL);
@@ -1620,7 +1668,7 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
16201668
else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) {
16211669
int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s->s) : read_int32(s->s));
16221670
jl_value_t *v = jl_gc_alloc(ptls, sz, NULL);
1623-
int pos = backref_list.len;
1671+
uintptr_t pos = backref_list.len;
16241672
if (usetable)
16251673
arraylist_push(&backref_list, v);
16261674
jl_datatype_t *dt = (jl_datatype_t*)jl_deserialize_value(s, &jl_astaggedvalue(v)->type);
@@ -2309,30 +2357,72 @@ static void jl_recache_types(void)
23092357
int offs = (int)(intptr_t)flagref_list.items[i++];
23102358
jl_value_t *v, *o = loc ? *loc : (jl_value_t*)backref_list.items[offs];
23112359
jl_datatype_t *dt, *t;
2312-
if (jl_is_datatype(o)) {
2313-
dt = (jl_datatype_t*)o;
2314-
v = dt->instance;
2315-
assert(dt->uid == -1);
2316-
t = jl_recache_type(dt, i, NULL);
2317-
}
2318-
else {
2319-
dt = (jl_datatype_t*)jl_typeof(o);
2360+
if (jl_is_lambda_info(o)) {
2361+
// lookup the real LambdaInfo based on the placeholder specTypes
2362+
jl_lambda_info_t *li = (jl_lambda_info_t*)o;
2363+
int inferred = li->inferred;
2364+
jl_datatype_t *argtypes = jl_recache_type(li->specTypes, i, NULL);
2365+
jl_datatype_t *ftype = jl_first_argument_datatype((jl_value_t*)argtypes);
2366+
jl_methtable_t *mt = ftype->name->mt;
2367+
jl_set_typeof(li, (void*)(intptr_t)0x30); // invalidate the old value to help catch errors
2368+
li = jl_method_lookup_by_type(mt, argtypes, 1, 0, 0);
2369+
assert(li);
2370+
// if it can be inferred but isn't, encourage codegen to infer it
2371+
if (inferred && !li->inferred) {
2372+
jl_set_lambda_code_null(li);
2373+
li->inferred = 1;
2374+
}
2375+
// update the backref list
2376+
if (loc) *loc = (jl_value_t*)li;
2377+
if (offs > 0) backref_list.items[offs] = li;
23202378
v = o;
2321-
t = jl_recache_type(dt, i, v);
2322-
}
2323-
assert(dt);
2324-
if (t != dt) {
2325-
jl_set_typeof(dt, (void*)(intptr_t)0x10); // invalidate the old value to help catch errors
2326-
if ((jl_value_t*)dt == o) {
2327-
if (loc) *loc = (jl_value_t*)t;
2328-
if (offs > 0) backref_list.items[offs] = t;
2379+
size_t j = i;
2380+
while (j < flagref_list.len) {
2381+
jl_value_t **loc = (jl_value_t**)flagref_list.items[j];
2382+
int offs = (int)(intptr_t)flagref_list.items[j+1];
2383+
jl_value_t *o = loc ? *loc : (jl_value_t*)backref_list.items[offs];
2384+
if ((jl_value_t*)v == o) { // same item, update this entry
2385+
if (loc) *loc = (jl_value_t*)li;
2386+
if (offs > 0) backref_list.items[offs] = li;
2387+
// delete this item from the flagref list, so it won't be re-encountered later
2388+
flagref_list.len -= 2;
2389+
if (j >= flagref_list.len)
2390+
break;
2391+
flagref_list.items[j+0] = flagref_list.items[flagref_list.len+0];
2392+
flagref_list.items[j+1] = flagref_list.items[flagref_list.len+1];
2393+
}
2394+
else {
2395+
j += 2;
2396+
}
23292397
}
23302398
}
2331-
if (t->instance != v) {
2332-
jl_set_typeof(v, (void*)(intptr_t)0x20); // invalidate the old value to help catch errors
2333-
if (v == o) {
2334-
*loc = t->instance;
2335-
if (offs > 0) backref_list.items[offs] = t->instance;
2399+
else {
2400+
if (jl_is_datatype(o)) {
2401+
dt = (jl_datatype_t*)o;
2402+
v = dt->instance;
2403+
assert(dt->uid == -1);
2404+
t = jl_recache_type(dt, i, NULL);
2405+
}
2406+
else {
2407+
dt = (jl_datatype_t*)jl_typeof(o);
2408+
v = o;
2409+
assert(dt->instance);
2410+
t = jl_recache_type(dt, i, v);
2411+
}
2412+
assert(dt);
2413+
if (t != dt) {
2414+
jl_set_typeof(dt, (void*)(intptr_t)0x10); // invalidate the old value to help catch errors
2415+
if ((jl_value_t*)dt == o) {
2416+
if (loc) *loc = (jl_value_t*)t;
2417+
if (offs > 0) backref_list.items[offs] = t;
2418+
}
2419+
}
2420+
if (t->instance != v) {
2421+
jl_set_typeof(v, (void*)(intptr_t)0x20); // invalidate the old value to help catch errors
2422+
if (v == o) {
2423+
*loc = t->instance;
2424+
if (offs > 0) backref_list.items[offs] = t->instance;
2425+
}
23362426
}
23372427
}
23382428
}

0 commit comments

Comments
 (0)