-
-
Notifications
You must be signed in to change notification settings - Fork 5.6k
/
Copy pathjitlayers.cpp
2629 lines (2449 loc) · 112 KB
/
jitlayers.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// This file is a part of Julia. License is MIT: https://julialang.org/license
#include "llvm-version.h"
#include "platform.h"
#include <stdint.h>
#include <string>
#include "llvm/IR/Mangler.h"
#include <llvm/ADT/Statistic.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/TargetLibraryInfo.h>
#include <llvm/Analysis/TargetTransformInfo.h>
#include <llvm/ExecutionEngine/Orc/CompileUtils.h>
#include <llvm/ExecutionEngine/Orc/ExecutionUtils.h>
#include <llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h>
#include <llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h>
#if JL_LLVM_VERSION >= 180000
#include <llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h>
#include <llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h>
#include <llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h>
#endif
#if JL_LLVM_VERSION >= 190000
#include <llvm/ExecutionEngine/Orc/Debugging/VTuneSupportPlugin.h>
#include <llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderVTune.h>
#endif
#include <llvm/ExecutionEngine/Orc/ExecutorProcessControl.h>
#include <llvm/IR/Verifier.h>
#include <llvm/Support/DynamicLibrary.h>
#include <llvm/Support/FormattedStream.h>
#include <llvm/Support/SmallVectorMemoryBuffer.h>
#include <llvm/Support/raw_ostream.h>
#include <llvm/Transforms/Utils/Cloning.h>
#include <llvm/Transforms/Utils/ModuleUtils.h>
#include <llvm/Bitcode/BitcodeWriter.h>
// target machine computation
#include <llvm/CodeGen/TargetSubtargetInfo.h>
#include <llvm/MC/TargetRegistry.h>
#include <llvm/Target/TargetOptions.h>
#include <llvm/TargetParser/Host.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Object/SymbolSize.h>
using namespace llvm;
#include "jitlayers.h"
#include "julia_assert.h"
#include "processor.h"
#if JL_LLVM_VERSION >= 180000
# include <llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h>
#else
# include <llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h>
#endif
# include <llvm/ExecutionEngine/JITLink/EHFrameSupport.h>
# include <llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h>
# include <llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h>
# include <llvm/ExecutionEngine/SectionMemoryManager.h>
#define DEBUG_TYPE "julia_jitlayers"
STATISTIC(LinkedGlobals, "Number of globals linked");
STATISTIC(SpecFPtrCount, "Number of specialized function pointers compiled");
STATISTIC(UnspecFPtrCount, "Number of specialized function pointers compiled");
STATISTIC(ModulesAdded, "Number of modules added to the JIT");
STATISTIC(ModulesOptimized, "Number of modules optimized by the JIT");
STATISTIC(OptO0, "Number of modules optimized at level -O0");
STATISTIC(OptO1, "Number of modules optimized at level -O1");
STATISTIC(OptO2, "Number of modules optimized at level -O2");
STATISTIC(OptO3, "Number of modules optimized at level -O3");
STATISTIC(ModulesMerged, "Number of modules merged");
STATISTIC(InternedGlobals, "Number of global constants interned in the string pool");
#ifdef _COMPILER_MSAN_ENABLED_
// TODO: This should not be necessary on ELF x86_64, but LLVM's implementation
// of the TLS relocations is currently broken, so enable this unconditionally.
#define MSAN_EMUTLS_WORKAROUND 1
// See https://github.com/google/sanitizers/wiki/MemorySanitizerJIT
namespace msan_workaround {
extern "C" {
extern __thread unsigned long long __msan_param_tls[];
extern __thread unsigned int __msan_param_origin_tls[];
extern __thread unsigned long long __msan_retval_tls[];
extern __thread unsigned int __msan_retval_origin_tls;
extern __thread unsigned long long __msan_va_arg_tls[];
extern __thread unsigned int __msan_va_arg_origin_tls[];
extern __thread unsigned long long __msan_va_arg_overflow_size_tls;
extern __thread unsigned int __msan_origin_tls;
}
enum class MSanTLS
{
param = 1, // __msan_param_tls
param_origin, //__msan_param_origin_tls
retval, // __msan_retval_tls
retval_origin, //__msan_retval_origin_tls
va_arg, // __msan_va_arg_tls
va_arg_origin, // __msan_va_arg_origin_tls
va_arg_overflow_size, // __msan_va_arg_overflow_size_tls
origin, //__msan_origin_tls
};
static void *getTLSAddress(void *control)
{
auto tlsIndex = static_cast<MSanTLS>(reinterpret_cast<uintptr_t>(control));
switch(tlsIndex)
{
case MSanTLS::param: return reinterpret_cast<void *>(&__msan_param_tls);
case MSanTLS::param_origin: return reinterpret_cast<void *>(&__msan_param_origin_tls);
case MSanTLS::retval: return reinterpret_cast<void *>(&__msan_retval_tls);
case MSanTLS::retval_origin: return reinterpret_cast<void *>(&__msan_retval_origin_tls);
case MSanTLS::va_arg: return reinterpret_cast<void *>(&__msan_va_arg_tls);
case MSanTLS::va_arg_origin: return reinterpret_cast<void *>(&__msan_va_arg_origin_tls);
case MSanTLS::va_arg_overflow_size: return reinterpret_cast<void *>(&__msan_va_arg_overflow_size_tls);
case MSanTLS::origin: return reinterpret_cast<void *>(&__msan_origin_tls);
default:
assert(false && "BAD MSAN TLS INDEX");
return nullptr;
}
}
}
#endif
#ifdef _OS_OPENBSD_
extern "C" {
__int128 __divti3(__int128, __int128);
__int128 __modti3(__int128, __int128);
unsigned __int128 __udivti3(unsigned __int128, unsigned __int128);
unsigned __int128 __umodti3(unsigned __int128, unsigned __int128);
}
#endif
// Snooping on which functions are being compiled, and how long it takes
extern "C" JL_DLLEXPORT_CODEGEN
void jl_dump_compiles_impl(void *s)
{
**jl_ExecutionEngine->get_dump_compiles_stream() = (ios_t*)s;
}
extern "C" JL_DLLEXPORT_CODEGEN
void jl_dump_llvm_opt_impl(void *s)
{
**jl_ExecutionEngine->get_dump_llvm_opt_stream() = (ios_t*)s;
}
static void jl_decorate_module(Module &M) JL_NOTSAFEPOINT;
void jl_link_global(GlobalVariable *GV, void *addr) JL_NOTSAFEPOINT
{
++LinkedGlobals;
Constant *P = literal_static_pointer_val(addr, GV->getValueType());
GV->setInitializer(P);
GV->setDSOLocal(true);
if (jl_options.image_codegen) {
// If we are forcing imaging mode codegen for debugging,
// emit external non-const symbol to avoid LLVM optimizing the code
// similar to non-imaging mode.
assert(GV->hasExternalLinkage());
}
else {
GV->setConstant(true);
GV->setLinkage(GlobalValue::PrivateLinkage);
GV->setVisibility(GlobalValue::DefaultVisibility);
GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
}
}
// convert local roots into global roots, if they are needed
static void jl_optimize_roots(jl_codegen_params_t ¶ms, jl_method_instance_t *mi, Module &M)
{
JL_GC_PROMISE_ROOTED(params.temporary_roots); // rooted by caller
if (jl_array_dim0(params.temporary_roots) == 0)
return;
jl_method_t *m = mi->def.method;
if (jl_is_method(m))
// the method might have a root for this already; use it if so
JL_LOCK(&m->writelock);
for (size_t i = 0; i < jl_array_dim0(params.temporary_roots); i++) {
jl_value_t *val = jl_array_ptr_ref(params.temporary_roots, i);
auto ref = params.global_targets.find((void*)val);
if (ref == params.global_targets.end())
continue;
auto get_global_root = [val, m]() {
if (jl_is_globally_rooted(val))
return val;
if (jl_is_method(m) && m->roots) {
size_t j, len = jl_array_dim0(m->roots);
for (j = 0; j < len; j++) {
jl_value_t *mval = jl_array_ptr_ref(m->roots, j);
if (jl_egal(mval, val)) {
return mval;
}
}
}
return jl_as_global_root(val, 1);
};
jl_value_t *mval = get_global_root();
if (mval != val) {
GlobalVariable *GV = ref->second;
params.global_targets.erase(ref);
auto mref = params.global_targets.find((void*)mval);
if (mref != params.global_targets.end()) {
GV->replaceAllUsesWith(mref->second);
GV->eraseFromParent();
}
else {
params.global_targets[(void*)mval] = GV;
}
}
}
if (jl_is_method(m))
JL_UNLOCK(&m->writelock);
}
void jl_jit_globals(std::map<void *, GlobalVariable*> &globals) JL_NOTSAFEPOINT
{
for (auto &global : globals) {
jl_link_global(global.second, global.first);
}
}
// lock for places where only single threaded behavior is implemented, so we need GC support
static jl_mutex_t jitlock;
// locks for adding external code to the JIT atomically
static std::mutex extern_c_lock;
// locks and barriers for this state
static std::mutex engine_lock;
static std::condition_variable engine_wait;
static int threads_in_compiler_phase;
// the TSM for each codeinst
static SmallVector<orc::ThreadSafeModule,0> sharedmodules;
static DenseMap<jl_code_instance_t*, orc::ThreadSafeModule> emittedmodules;
// the invoke and specsig function names in the JIT
static DenseMap<jl_code_instance_t*, jl_llvm_functions_t> invokenames;
// everything that any thread wants to compile right now
static DenseSet<jl_code_instance_t*> compileready;
// everything that any thread has compiled recently
static DenseSet<jl_code_instance_t*> linkready;
// a map from a codeinst to the outgoing edges needed before linking it
static DenseMap<jl_code_instance_t*, SmallVector<jl_code_instance_t*,0>> complete_graph;
// the state for each codeinst and the number of unresolved edges (we don't
// really need this once JITLink is available everywhere, since every module
// is automatically complete, and we can emit any required fixups later as a
// separate module)
static DenseMap<jl_code_instance_t*, std::tuple<jl_codegen_params_t, int>> incompletemodules;
// the set of incoming unresolved edges resolved by a codeinstance
static DenseMap<jl_code_instance_t*, SmallVector<jl_code_instance_t*,0>> incomplete_rgraph;
// Lock hierarchy here:
// jitlock is outermost, can contain others and allows GC
// engine_lock is next
// ThreadSafeContext locks are next, they should not be nested (unless engine_lock is also held, but this may make TSAN sad anyways)
// extern_c_lock is next
// jl_ExecutionEngine internal locks are exclusive to this list, since OrcJIT promises to never hold a lock over a materialization unit:
// construct a query object from a query set and query handler
// lock the session
// lodge query against requested symbols, collect required materializers (if any)
// unlock the session
// dispatch materializers (if any)
// However, this guarantee relies on Julia releasing all TSC locks before causing any materialization units to be dispatched
// as materialization may need to acquire TSC locks.
static void finish_params(Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT
{
if (params._shared_module) {
sharedmodules.push_back(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx));
}
// In imaging mode, we can't inline global variable initializers in order to preserve
// the fiction that we don't know what loads from the global will return. Thus, we
// need to emit a separate module for the globals before any functions are compiled,
// to ensure that the globals are defined when they are compiled.
if (jl_options.image_codegen) {
if (!params.global_targets.empty()) {
void **globalslots = new void*[params.global_targets.size()];
void **slot = globalslots;
for (auto &global : params.global_targets) {
auto GV = global.second;
*slot = global.first;
jl_ExecutionEngine->addGlobalMapping(GV->getName(), (uintptr_t)slot);
slot++;
}
#ifdef __clang_analyzer__
static void **leaker = globalslots; // for the purpose of the analyzer, we need to expressly leak this variable or it thinks we forgot to free it
#endif
}
}
else {
StringMap<void*> NewGlobals;
for (auto &global : params.global_targets) {
NewGlobals[global.second->getName()] = global.first;
}
for (auto &GV : M->globals()) {
auto InitValue = NewGlobals.find(GV.getName());
if (InitValue != NewGlobals.end()) {
jl_link_global(&GV, InitValue->second);
}
}
}
}
static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t ¶ms, bool forceall=false) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER
{
jl_task_t *ct = jl_current_task;
decltype(params.workqueue) edges;
std::swap(params.workqueue, edges);
for (auto &it : edges) {
jl_code_instance_t *codeinst = it.first;
JL_GC_PROMISE_ROOTED(codeinst);
auto &proto = it.second;
// try to emit code for this item from the workqueue
StringRef invokeName = "";
StringRef preal_decl = "";
bool preal_specsig = false;
jl_callptr_t invoke = nullptr;
bool isedge = false;
assert(params.cache);
// Checking the cache here is merely an optimization and not strictly required
// But it must be consistent with the following invokenames lookup, which is protected by the engine_lock
uint8_t specsigflags;
void *fptr;
void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; // declare it is not a safepoint (or deadlock) in this file due to 0 parameter
jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0);
//if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr)
if (invoke == jl_fptr_args_addr) {
preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst);
}
else if (specsigflags & 0b1) {
preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst);
preal_specsig = true;
}
bool force = forceall || invoke != nullptr;
if (preal_decl.empty()) {
auto it = invokenames.find(codeinst);
if (it != invokenames.end()) {
auto &decls = it->second;
invokeName = decls.functionObject;
if (decls.functionObject == "jl_fptr_args") {
preal_decl = decls.specFunctionObject;
isedge = true;
}
else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") {
preal_decl = decls.specFunctionObject;
preal_specsig = true;
isedge = true;
}
force = true;
}
}
if (preal_decl.empty()) {
// there may be an equivalent method already compiled (or at least registered with the JIT to compile), in which case we should be using that instead
jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 1);
if ((jl_value_t*)compiled_ci != jl_nothing) {
codeinst = compiled_ci;
uint8_t specsigflags;
void *fptr;
jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0);
//if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr)
if (invoke == jl_fptr_args_addr) {
preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst);
}
else if (specsigflags & 0b1) {
preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst);
preal_specsig = true;
}
if (preal_decl.empty()) {
auto it = invokenames.find(codeinst);
if (it != invokenames.end()) {
auto &decls = it->second;
invokeName = decls.functionObject;
if (decls.functionObject == "jl_fptr_args") {
preal_decl = decls.specFunctionObject;
isedge = true;
}
else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") {
preal_decl = decls.specFunctionObject;
preal_specsig = true;
isedge = true;
}
}
}
}
}
if (!preal_decl.empty() || force) {
// if we have a prototype emitted, compare it to what we emitted earlier
Module *mod = proto.decl->getParent();
assert(proto.decl->isDeclaration());
Function *pinvoke = nullptr;
if (preal_decl.empty()) {
if (invoke != nullptr && invokeName.empty()) {
assert(invoke != jl_fptr_args_addr);
if (invoke == jl_fptr_sparam_addr)
invokeName = "jl_fptr_sparam";
else if (invoke == jl_f_opaque_closure_call_addr)
invokeName = "jl_f_opaque_closure_call";
else
invokeName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst);
}
pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params);
if (!proto.specsig)
proto.decl->replaceAllUsesWith(pinvoke);
isedge = false;
}
if (proto.specsig && !preal_specsig) {
// get or build an fptr1 that can invoke codeinst
if (pinvoke == nullptr)
pinvoke = get_or_emit_fptr1(preal_decl, mod);
// emit specsig-to-(jl)invoke conversion
proto.decl->setLinkage(GlobalVariable::InternalLinkage);
//protodecl->setAlwaysInline();
jl_init_function(proto.decl, params.TargetTriple);
// TODO: maybe this can be cached in codeinst->specfptr?
int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls)
jl_method_instance_t *mi = jl_get_ci_mi(codeinst);
size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed
bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure;
emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke);
jl_gc_unsafe_leave(ct->ptls, gc_state);
preal_decl = ""; // no need to fixup the name
}
if (!preal_decl.empty()) {
// merge and/or rename this prototype to the real function
if (Value *specfun = mod->getNamedValue(preal_decl)) {
if (proto.decl != specfun)
proto.decl->replaceAllUsesWith(specfun);
}
else {
proto.decl->setName(preal_decl);
}
}
if (proto.oc) { // additionally, if we are dealing with an OC constructor, then we might also need to fix up the fptr1 reference too
assert(proto.specsig);
StringRef ocinvokeDecl = invokeName;
if (invoke != nullptr && ocinvokeDecl.empty()) {
// check for some special tokens used by opaque_closure.c and convert those to their real functions
assert(invoke != jl_fptr_args_addr);
assert(invoke != jl_fptr_sparam_addr);
if (invoke == jl_fptr_interpret_call_addr)
ocinvokeDecl = "jl_fptr_interpret_call";
else if (invoke == jl_fptr_const_return_addr)
ocinvokeDecl = "jl_fptr_const_return";
else if (invoke == jl_f_opaque_closure_call_addr)
ocinvokeDecl = "jl_f_opaque_closure_call";
//else if (invoke == jl_interpret_opaque_closure_addr)
else
ocinvokeDecl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst);
}
// if OC expected a specialized specsig dispatch, but we don't have it, use the inner trampoline here too
// XXX: this invoke translation logic is supposed to exactly match new_opaque_closure
if (!preal_specsig || ocinvokeDecl == "jl_f_opaque_closure_call" || ocinvokeDecl == "jl_fptr_interpret_call" || ocinvokeDecl == "jl_fptr_const_return") {
if (pinvoke == nullptr)
ocinvokeDecl = get_or_emit_fptr1(preal_decl, mod)->getName();
else
ocinvokeDecl = pinvoke->getName();
}
assert(!ocinvokeDecl.empty());
assert(ocinvokeDecl != "jl_fptr_args");
assert(ocinvokeDecl != "jl_fptr_sparam");
// merge and/or rename this prototype to the real function
if (Value *specfun = mod->getNamedValue(ocinvokeDecl)) {
if (proto.oc != specfun)
proto.oc->replaceAllUsesWith(specfun);
}
else {
proto.oc->setName(ocinvokeDecl);
}
}
}
else {
isedge = true;
params.workqueue.push_back(it);
incomplete_rgraph[codeinst].push_back(callee);
}
if (isedge)
complete_graph[callee].push_back(codeinst);
}
return params.workqueue.size();
}
// move codeinst (and deps) from incompletemodules to emitted modules
// and populate compileready from complete_graph
static void prepare_compile(jl_code_instance_t *codeinst) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER
{
SmallVector<jl_code_instance_t*> workqueue;
workqueue.push_back(codeinst);
while (!workqueue.empty()) {
codeinst = workqueue.pop_back_val();
if (!invokenames.count(codeinst)) {
// this means it should be compiled already while the callee was in stasis
assert(jl_is_compiled_codeinst(codeinst));
continue;
}
// if this was incomplete, force completion now of it
auto it = incompletemodules.find(codeinst);
if (it != incompletemodules.end()) {
int waiting = 0;
auto &edges = complete_graph[codeinst];
auto edges_end = std::remove_if(edges.begin(), edges.end(), [&waiting, codeinst] (jl_code_instance_t *edge) JL_NOTSAFEPOINT -> bool {
auto &redges = incomplete_rgraph[edge];
// waiting += std::erase(redges, codeinst);
auto redges_end = std::remove(redges.begin(), redges.end(), codeinst);
if (redges_end != redges.end()) {
waiting += redges.end() - redges_end;
redges.erase(redges_end, redges.end());
assert(!invokenames.count(edge));
}
return !invokenames.count(edge);
});
edges.erase(edges_end, edges.end());
assert(waiting == std::get<1>(it->second));
std::get<1>(it->second) = 0;
auto ¶ms = std::get<0>(it->second);
params.tsctx_lock = params.tsctx.getLock();
waiting = jl_analyze_workqueue(codeinst, params, true); // may safepoint
assert(!waiting); (void)waiting;
Module *M = emittedmodules[codeinst].getModuleUnlocked();
finish_params(M, params);
incompletemodules.erase(it);
}
// and then indicate this should be compiled now
if (!linkready.count(codeinst) && compileready.insert(codeinst).second) {
auto edges = complete_graph.find(codeinst);
if (edges != complete_graph.end()) {
workqueue.append(edges->second);
}
}
}
}
// notify any other pending work that this edge now has code defined
static void complete_emit(jl_code_instance_t *edge) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER
{
auto notify = incomplete_rgraph.find(edge);
if (notify == incomplete_rgraph.end())
return;
auto redges = std::move(notify->second);
incomplete_rgraph.erase(notify);
for (size_t i = 0; i < redges.size(); i++) {
jl_code_instance_t *callee = redges[i];
auto it = incompletemodules.find(callee);
assert(it != incompletemodules.end());
if (--std::get<1>(it->second) == 0) {
auto ¶ms = std::get<0>(it->second);
params.tsctx_lock = params.tsctx.getLock();
assert(callee == it->first);
int waiting = jl_analyze_workqueue(callee, params); // may safepoint
assert(!waiting); (void)waiting;
Module *M = emittedmodules[callee].getModuleUnlocked();
finish_params(M, params);
incompletemodules.erase(it);
}
}
}
// set the invoke field for codeinst (and all deps, and assist with other pending work from other threads) now
static void jl_compile_codeinst_now(jl_code_instance_t *codeinst)
{
jl_unique_gcsafe_lock lock(engine_lock);
if (!invokenames.count(codeinst))
return;
threads_in_compiler_phase++;
prepare_compile(codeinst); // may safepoint
while (1) {
// TODO: split up this work by ThreadSafeContext, so two threads don't need to get the same locks and stall
if (!sharedmodules.empty()) {
auto TSM = sharedmodules.pop_back_val();
lock.native.unlock();
{
auto Lock = TSM.getContext().getLock();
jl_ExecutionEngine->optimizeDLSyms(*TSM.getModuleUnlocked()); // may safepoint
}
jl_ExecutionEngine->addModule(std::move(TSM));
lock.native.lock();
}
else if (!compileready.empty()) {
// move a function from compileready to linkready then compile it
auto compilenext = compileready.begin();
codeinst = *compilenext;
compileready.erase(compilenext);
auto TSMref = emittedmodules.find(codeinst);
assert(TSMref != emittedmodules.end());
auto TSM = std::move(TSMref->second);
linkready.insert(codeinst);
emittedmodules.erase(TSMref);
lock.native.unlock();
uint64_t start_time = jl_hrtime();
{
auto Lock = TSM.getContext().getLock();
jl_ExecutionEngine->optimizeDLSyms(*TSM.getModuleUnlocked()); // may safepoint
}
jl_ExecutionEngine->addModule(std::move(TSM)); // may safepoint
// If logging of the compilation stream is enabled,
// then dump the method-instance specialization type to the stream
jl_method_instance_t *mi = jl_get_ci_mi(codeinst);
if (jl_is_method(mi->def.method)) {
auto stream = *jl_ExecutionEngine->get_dump_compiles_stream();
if (stream) {
uint64_t end_time = jl_hrtime();
ios_printf(stream, "%" PRIu64 "\t\"", end_time - start_time);
jl_static_show((JL_STREAM*)stream, mi->specTypes);
ios_printf(stream, "\"\n");
}
}
lock.native.lock();
}
else {
break;
}
}
codeinst = nullptr;
// barrier until all threads have finished calling addModule
if (--threads_in_compiler_phase == 0) {
// the last thread out will finish linking everything
// then release all of the other threads
// move the function pointers out from invokenames to the codeinst
// batch compile job for all new functions
SmallVector<StringRef> NewDefs;
for (auto &this_code : linkready) {
auto it = invokenames.find(this_code);
assert(it != invokenames.end());
jl_llvm_functions_t &decls = it->second;
assert(!decls.functionObject.empty());
if (decls.functionObject != "jl_fptr_args" &&
decls.functionObject != "jl_fptr_sparam" &&
decls.functionObject != "jl_f_opaque_closure_call")
NewDefs.push_back(decls.functionObject);
if (!decls.specFunctionObject.empty())
NewDefs.push_back(decls.specFunctionObject);
}
auto Addrs = jl_ExecutionEngine->findSymbols(NewDefs);
size_t nextaddr = 0;
for (auto &this_code : linkready) {
auto it = invokenames.find(this_code);
assert(it != invokenames.end());
jl_llvm_functions_t &decls = it->second;
jl_callptr_t addr;
bool isspecsig = false;
if (decls.functionObject == "jl_fptr_args") {
addr = jl_fptr_args_addr;
}
else if (decls.functionObject == "jl_fptr_sparam") {
addr = jl_fptr_sparam_addr;
}
else if (decls.functionObject == "jl_f_opaque_closure_call") {
addr = jl_f_opaque_closure_call_addr;
}
else {
assert(NewDefs[nextaddr] == decls.functionObject);
addr = (jl_callptr_t)Addrs[nextaddr++];
assert(addr);
isspecsig = true;
}
if (!decls.specFunctionObject.empty()) {
void *prev_specptr = nullptr;
assert(NewDefs[nextaddr] == decls.specFunctionObject);
void *spec = (void*)Addrs[nextaddr++];
assert(spec);
if (jl_atomic_cmpswap_acqrel(&this_code->specptr.fptr, &prev_specptr, spec)) {
// only set specsig and invoke if we were the first to set specptr
jl_atomic_store_relaxed(&this_code->specsigflags, (uint8_t) isspecsig);
// we might overwrite invokeptr here; that's ok, anybody who relied on the identity of invokeptr
// either assumes that specptr was null, doesn't care about specptr,
// or will wait until specsigflags has 0b10 set before reloading invoke
jl_atomic_store_release(&this_code->invoke, addr);
jl_atomic_store_release(&this_code->specsigflags, (uint8_t) (0b10 | isspecsig));
}
else {
//someone else beat us, don't commit any results
while (!(jl_atomic_load_acquire(&this_code->specsigflags) & 0b10)) {
jl_cpu_pause();
}
addr = jl_atomic_load_relaxed(&this_code->invoke);
}
}
else {
jl_callptr_t prev_invoke = nullptr;
// Allow replacing addr if it is either nullptr or our special waiting placeholder.
if (!jl_atomic_cmpswap_acqrel(&this_code->invoke, &prev_invoke, addr)) {
if (prev_invoke == jl_fptr_wait_for_compiled_addr && !jl_atomic_cmpswap_acqrel(&this_code->invoke, &prev_invoke, addr)) {
addr = prev_invoke;
//TODO do we want to potentially promote invoke anyways? (e.g. invoke is jl_interpret_call or some other
//known lesser function)
}
}
}
invokenames.erase(it);
complete_graph.erase(this_code);
}
linkready.clear();
engine_wait.notify_all();
}
else while (threads_in_compiler_phase) {
lock.wait(engine_wait);
}
}
void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const DataLayout &DL);
extern "C" JL_DLLEXPORT_CODEGEN
void jl_emit_codeinst_to_jit_impl(
jl_code_instance_t *codeinst,
jl_code_info_t *src)
{
if (jl_is_compiled_codeinst(codeinst))
return;
{ // lock scope
jl_unique_gcsafe_lock lock(engine_lock);
if (invokenames.count(codeinst) || jl_is_compiled_codeinst(codeinst))
return;
}
JL_TIMING(CODEINST_COMPILE, CODEINST_COMPILE);
// emit the code in LLVM IR form to the new context
jl_codegen_params_t params(std::make_unique<LLVMContext>(), jl_ExecutionEngine->getDataLayout(), jl_ExecutionEngine->getTargetTriple()); // Locks the context
params.getContext().setDiscardValueNames(true);
params.cache = true;
params.imaging_mode = 0;
orc::ThreadSafeModule result_m =
jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, params.DL, params.TargetTriple);
params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0);
JL_GC_PUSH1(¶ms.temporary_roots);
jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints
if (!result_m) {
JL_GC_POP();
return;
}
jl_optimize_roots(params, jl_get_ci_mi(codeinst), *result_m.getModuleUnlocked()); // contains safepoints
params.temporary_roots = nullptr;
JL_GC_POP();
{ // drop lock before acquiring engine_lock
auto release = std::move(params.tsctx_lock);
}
jl_unique_gcsafe_lock lock(engine_lock);
if (invokenames.count(codeinst) || jl_is_compiled_codeinst(codeinst))
return; // destroy everything
const std::string &specf = decls.specFunctionObject;
const std::string &f = decls.functionObject;
assert(!f.empty());
// Prepare debug info to receive this function
// record that this function name came from this linfo,
// so we can build a reverse mapping for debug-info.
bool toplevel = !jl_is_method(jl_get_ci_mi(codeinst)->def.method);
if (!toplevel) {
// don't remember toplevel thunks because
// they may not be rooted in the gc for the life of the program,
// and the runtime doesn't notify us when the code becomes unreachable :(
if (!specf.empty())
jl_add_code_in_flight(specf, codeinst, params.DL);
if (f != "jl_fptr_args" && f != "jl_fptr_sparam")
jl_add_code_in_flight(f, codeinst, params.DL);
}
jl_callptr_t expected = NULL;
jl_atomic_cmpswap_relaxed(&codeinst->invoke, &expected, jl_fptr_wait_for_compiled_addr);
invokenames[codeinst] = std::move(decls);
complete_emit(codeinst);
params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock
int waiting = jl_analyze_workqueue(codeinst, params);
if (waiting) {
auto release = std::move(params.tsctx_lock); // unlock again before moving from it
incompletemodules.try_emplace(codeinst, std::move(params), waiting);
}
else {
finish_params(result_m.getModuleUnlocked(), params);
}
emittedmodules[codeinst] = std::move(result_m);
}
const char *jl_generate_ccallable(Module *llvmmod, void *sysimg_handle, jl_value_t *declrt, jl_value_t *sigt, jl_codegen_params_t ¶ms);
// compile a C-callable alias
extern "C" JL_DLLEXPORT_CODEGEN
int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void *sysimg, jl_value_t *declrt, jl_value_t *sigt)
{
auto ct = jl_current_task;
bool timed = (ct->reentrant_timing & 1) == 0;
if (timed)
ct->reentrant_timing |= 1;
uint64_t compiler_start_time = 0;
uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled);
if (measure_compile_time_enabled)
compiler_start_time = jl_hrtime();
jl_codegen_params_t *pparams = (jl_codegen_params_t*)p;
DataLayout DL = pparams ? pparams->DL : jl_ExecutionEngine->getDataLayout();
Triple TargetTriple = pparams ? pparams->TargetTriple : jl_ExecutionEngine->getTargetTriple();
orc::ThreadSafeContext ctx;
auto into = unwrap(llvmmod);
orc::ThreadSafeModule backing;
bool success = true;
const char *name = "";
if (into == NULL) {
ctx = pparams ? pparams->tsctx : jl_ExecutionEngine->makeContext();
backing = jl_create_ts_module("cextern", ctx, DL, TargetTriple);
into = &backing;
}
{ // params scope
jl_codegen_params_t params(into->getContext(), DL, TargetTriple);
if (pparams == NULL) {
params.cache = p == NULL;
params.imaging_mode = 0;
params.tsctx.getContext()->setDiscardValueNames(true);
pparams = ¶ms;
}
Module &M = *into->getModuleUnlocked();
assert(pparams->tsctx.getContext() == &M.getContext());
name = jl_generate_ccallable(&M, sysimg, declrt, sigt, *pparams);
if (!sysimg && !p) {
{ // drop lock to keep analyzer happy (since it doesn't know we have the only reference to it)
auto release = std::move(params.tsctx_lock);
}
{ // lock scope
jl_unique_gcsafe_lock lock(extern_c_lock);
if (jl_ExecutionEngine->getGlobalValueAddress(name))
success = false;
}
params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock
if (success && params.cache) {
size_t newest_world = jl_atomic_load_acquire(&jl_world_counter);
for (auto &it : params.workqueue) { // really just zero or one, and just the ABI not the rest of the metadata
jl_code_instance_t *codeinst = it.first;
JL_GC_PROMISE_ROOTED(codeinst);
jl_code_instance_t *newest_ci = jl_type_infer(jl_get_ci_mi(codeinst), newest_world, SOURCE_MODE_ABI);
if (newest_ci) {
if (jl_egal(codeinst->rettype, newest_ci->rettype))
it.first = codeinst;
jl_compile_codeinst_now(newest_ci);
}
}
jl_analyze_workqueue(nullptr, params, true);
assert(params.workqueue.empty());
finish_params(&M, params);
}
}
pparams = nullptr;
}
if (!sysimg && success && llvmmod == NULL) {
{ // lock scope
jl_unique_gcsafe_lock lock(extern_c_lock);
if (!jl_ExecutionEngine->getGlobalValueAddress(name)) {
{
auto Lock = backing.getContext().getLock();
jl_ExecutionEngine->optimizeDLSyms(*backing.getModuleUnlocked()); // safepoint
}
jl_ExecutionEngine->addModule(std::move(backing));
success = jl_ExecutionEngine->getGlobalValueAddress(name);
assert(success);
}
}
}
if (timed) {
if (measure_compile_time_enabled) {
auto end = jl_hrtime();
jl_atomic_fetch_add_relaxed(&jl_cumulative_compile_time, end - compiler_start_time);
}
ct->reentrant_timing &= ~1ull;
}
return success;
}
// declare a C-callable entry point; called during code loading from the toplevel
extern "C" JL_DLLEXPORT_CODEGEN
void jl_extern_c_impl(jl_value_t *declrt, jl_tupletype_t *sigt)
{
// validate arguments. try to do as many checks as possible here to avoid
// throwing errors later during codegen.
JL_TYPECHK(@ccallable, type, declrt);
if (!jl_is_tuple_type(sigt))
jl_type_error("@ccallable", (jl_value_t*)jl_anytuple_type_type, (jl_value_t*)sigt);
// check that f is a guaranteed singleton type
jl_datatype_t *ft = (jl_datatype_t*)jl_tparam0(sigt);
if (!jl_is_datatype(ft) || !jl_is_datatype_singleton(ft))
jl_error("@ccallable: function object must be a singleton");
// compute / validate return type
if (!jl_is_concrete_type(declrt) || jl_is_kind(declrt))
jl_error("@ccallable: return type must be concrete and correspond to a C type");
if (!jl_type_mappable_to_c(declrt))
jl_error("@ccallable: return type doesn't correspond to a C type");
// validate method signature
size_t i, nargs = jl_nparams(sigt);
for (i = 1; i < nargs; i++) {
jl_value_t *ati = jl_tparam(sigt, i);
if (!jl_is_concrete_type(ati) || jl_is_kind(ati) || !jl_type_mappable_to_c(ati))
jl_error("@ccallable: argument types must be concrete");
}
// save a record of this so that the alias is generated when we write an object file
jl_method_t *meth = (jl_method_t*)jl_methtable_lookup(ft->name->mt, (jl_value_t*)sigt, jl_atomic_load_acquire(&jl_world_counter));
if (!jl_is_method(meth))
jl_error("@ccallable: could not find requested method");
JL_GC_PUSH1(&meth);
meth->ccallable = jl_svec2(declrt, (jl_value_t*)sigt);
jl_gc_wb(meth, meth->ccallable);
JL_GC_POP();
// create the alias in the current runtime environment
int success = jl_compile_extern_c(NULL, NULL, NULL, declrt, (jl_value_t*)sigt);
if (!success)
jl_error("@ccallable was already defined for this method name");
}
extern "C" JL_DLLEXPORT_CODEGEN
int jl_compile_codeinst_impl(jl_code_instance_t *ci)
{
int newly_compiled = 0;
if (!jl_is_compiled_codeinst(ci)) {
++SpecFPtrCount;
uint64_t start = jl_typeinf_timing_begin();
jl_compile_codeinst_now(ci);
jl_typeinf_timing_end(start, 0);
newly_compiled = 1;
}
return newly_compiled;
}
extern "C" JL_DLLEXPORT_CODEGEN
void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec)
{
if (jl_atomic_load_relaxed(&unspec->invoke) != NULL) {
return;
}
auto ct = jl_current_task;
bool timed = (ct->reentrant_timing & 1) == 0;
if (timed)
ct->reentrant_timing |= 1;
uint64_t compiler_start_time = 0;
uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled);
if (measure_compile_time_enabled)
compiler_start_time = jl_hrtime();
jl_code_info_t *src = NULL;
JL_GC_PUSH1(&src);
jl_method_t *def = jl_get_ci_mi(unspec)->def.method;
if (jl_is_method(def)) {
src = (jl_code_info_t*)def->source;
if (src && (jl_value_t*)src != jl_nothing)
src = jl_uncompress_ir(def, NULL, (jl_value_t*)src);
}
else {
jl_method_instance_t *mi = jl_get_ci_mi(unspec);
jl_code_instance_t *uninferred = jl_cached_uninferred(jl_atomic_load_relaxed(&mi->cache), 1);
assert(uninferred);
src = (jl_code_info_t*)jl_atomic_load_relaxed(&uninferred->inferred);
assert(src);
}
if (src) {
// TODO: first prepare recursive_compile_graph(unspec, src) before taking this lock to avoid recursion?
JL_LOCK(&jitlock); // TODO: use a better lock
if (!jl_is_compiled_codeinst(unspec)) {
assert(jl_is_code_info(src));
++UnspecFPtrCount;
jl_svec_t *edges = (jl_svec_t*)src->edges;
if (jl_is_svec(edges)) {
jl_atomic_store_release(&unspec->edges, edges); // n.b. this assumes the field was always empty svec(), which is not entirely true
jl_gc_wb(unspec, edges);
}
jl_debuginfo_t *debuginfo = src->debuginfo;
jl_atomic_store_release(&unspec->debuginfo, debuginfo); // n.b. this assumes the field was previously NULL, which is not entirely true
jl_gc_wb(unspec, debuginfo);
jl_emit_codeinst_to_jit(unspec, src);
jl_compile_codeinst_now(unspec);
}
JL_UNLOCK(&jitlock); // Might GC
}
JL_GC_POP();
jl_callptr_t null = nullptr;
// if we hit a codegen bug (or ran into a broken generated function or llvmcall), fall back to the interpreter as a last resort
jl_atomic_cmpswap(&unspec->invoke, &null, jl_fptr_interpret_call_addr);
if (timed) {
if (measure_compile_time_enabled) {
auto end = jl_hrtime();
jl_atomic_fetch_add_relaxed(&jl_cumulative_compile_time, end - compiler_start_time);
}
ct->reentrant_timing &= ~1ull;
}
}
// get a native disassembly for a compiled method
extern "C" JL_DLLEXPORT_CODEGEN
jl_value_t *jl_dump_method_asm_impl(jl_method_instance_t *mi, size_t world,
char emit_mc, char getwrapper, const char* asm_variant, const char *debuginfo, char binary)
{
// printing via disassembly
jl_code_instance_t *codeinst = jl_compile_method_internal(mi, world);
if (codeinst) {
uintptr_t fptr = (uintptr_t)jl_atomic_load_acquire(&codeinst->invoke);
uintptr_t specfptr = (uintptr_t)jl_atomic_load_relaxed(&codeinst->specptr.fptr);
if (getwrapper || specfptr == 0)
specfptr = fptr;
if (specfptr != 0)
return jl_dump_fptr_asm(specfptr, emit_mc, asm_variant, debuginfo, binary);
}
return jl_an_empty_string;
}