Add interactive option to UnstableOptions

If `interactive` is set to true, codegen will special case that to get
faster and for the resulting LLVM IR to be executable from external
programs such as REPL.

This commit also has the following changes:

- Added LLVM JIT-related API.

- Stopped distributing rustc crates as rmeta files. See added code
  comment and rust-lang#102065 for detailed info.

- Some visibility changes on existing APIs so they could be used by
  external programs.

Author: hkmatsumoto

Cargo.lock

@@ -4155,6 +4155,7 @@ dependencies = [
  "rustc_codegen_ssa",
  "rustc_driver",
  "rustc_driver_impl",
+ "rustc_interactive",
  "rustc_smir",
 ]
 
@@ -4807,6 +4808,18 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "rustc_interactive"
+version = "0.0.0"
+dependencies = [
+ "rustc_codegen_llvm",
+ "rustc_codegen_ssa",
+ "rustc_hir",
+ "rustc_llvm",
+ "rustc_middle",
+ "rustc_span",
+]
+
 [[package]]
 name = "rustc_interface"
 version = "0.0.0"

compiler/rustc/Cargo.toml

@@ -13,6 +13,9 @@ rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 # Make sure rustc_smir ends up in the sysroot, because this
 # crate is intended to be used by stable MIR consumers, which are not in-tree
 rustc_smir = { path = "../rustc_smir" }
+# Make sure rustc_interactive ends up in the sysroot, because this
+# crate is intended to be used by programs that use the compiler as a library, which are not in-tree.
+rustc_interactive = { path = "../rustc_interactive" }
 
 [dependencies.jemalloc-sys]
 version = "0.5.0"

compiler/rustc_codegen_llvm/src/callee.rs

@@ -4,12 +4,17 @@
 //! and methods are represented as just a fn ptr and not a full
 //! closure.
 
+use std::ffi::CString;
+
 use crate::abi::FnAbiLlvmExt;
 use crate::attributes;
+use crate::builder::Builder;
 use crate::common;
 use crate::context::CodegenCx;
 use crate::llvm;
 use crate::value::Value;
+use llvm::LLVMSearchForAddressOfSymbol;
+use rustc_codegen_ssa::base::codegen_instance;
 use rustc_codegen_ssa::traits::*;
 
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
@@ -204,6 +209,16 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) ->
             }
         }
 
+        if cx.sess().opts.unstable_opts.interactive {
+            let addr = unsafe {
+                let name = CString::new(sym).unwrap();
+                LLVMSearchForAddressOfSymbol(name.as_ptr())
+            };
+            if addr.is_null() {
+                codegen_instance::<Builder<'_, '_, '_>>(cx, instance);
+            }
+        }
+
         llfn
     };
 

compiler/rustc_codegen_llvm/src/context.rs

@@ -342,7 +342,7 @@ pub unsafe fn create_module<'ll>(
 }
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
-    pub(crate) fn new(
+    pub fn new(
         tcx: TyCtxt<'tcx>,
         codegen_unit: &'tcx CodegenUnit<'tcx>,
         llvm_module: &'ll crate::ModuleLlvm,

compiler/rustc_codegen_llvm/src/lib.rs

@@ -59,11 +59,11 @@ mod allocator;
 mod asm;
 mod attributes;
 mod base;
-mod builder;
+pub mod builder;
 mod callee;
 mod common;
 mod consts;
-mod context;
+pub mod context;
 mod coverageinfo;
 mod debuginfo;
 mod declare;
@@ -396,7 +396,7 @@ unsafe impl Send for ModuleLlvm {}
 unsafe impl Sync for ModuleLlvm {}
 
 impl ModuleLlvm {
-    fn new(tcx: TyCtxt<'_>, mod_name: &str) -> Self {
+    pub fn new(tcx: TyCtxt<'_>, mod_name: &str) -> Self {
         unsafe {
             let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names());
             let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _;
@@ -433,7 +433,7 @@ impl ModuleLlvm {
         }
     }
 
-    fn llmod(&self) -> &llvm::Module {
+    pub fn llmod(&self) -> &llvm::Module {
         unsafe { &*self.llmod_raw }
     }
 }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -669,6 +669,15 @@ pub struct OperandBundleDef<'a>(InvariantOpaque<'a>);
 #[repr(C)]
 pub struct Linker<'a>(InvariantOpaque<'a>);
 
+extern "C" {
+    pub type LLJIT;
+}
+unsafe impl Sync for LLJIT {}
+
+extern "C" {
+    pub type JITDylib;
+}
+
 extern "C" {
     pub type DiagnosticHandler;
 }
@@ -2491,4 +2500,16 @@ extern "C" {
         callback: GetSymbolsCallback,
         error_callback: GetSymbolsErrorCallback,
     ) -> *mut c_void;
+
+    pub fn LLVMRustCreateLLJIT() -> &'static mut LLJIT;
+    pub fn LLVMRustLLJITLoadDynamicLibrary(J: &LLJIT, FileName: *const c_char);
+    pub fn LLVMRustLLJITAddIRModule(J: &LLJIT, M: &Module);
+    pub fn LLVMRustLLJITLookup(J: &LLJIT, Name: *const c_char) -> u64;
+    pub fn LLVMRustSetLinkageForAllFunctions(M: &Module);
+    pub fn LLVMRustAddGlobalCtor(C: &Context, M: &Module, V: &Value);
+    pub fn LLVMRustRunCtors(J: &LLJIT);
+
+    pub fn LLVMDumpModule(M: &Module);
+
+    pub fn LLVMSearchForAddressOfSymbol(SymbolName: *const c_char) -> *mut c_void;
 }

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -747,6 +747,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
         let def = instance.map(|i| i.def);
 
+        if self.cx.sess().opts.unstable_opts.interactive
+        && let Some(instance) = instance {
+            match instance.def {
+                ty::InstanceDef::Intrinsic(_) => {}
+                _ => {
+                    self.cx.get_fn(instance);
+                }
+            }
+        }
+
         if let Some(ty::InstanceDef::DropGlue(_, None)) = def {
             // Empty drop glue; a no-op.
             let target = target.unwrap();

compiler/rustc_interactive/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "rustc_interactive"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+rustc_codegen_llvm = { path = "../rustc_codegen_llvm" }
+rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_llvm = { path = "../rustc_llvm" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_span = { path = "../rustc_span" }

compiler/rustc_interactive/src/lib.rs

@@ -0,0 +1,4 @@
+//! Utilities for interactively use the Rust compiler from outside of
+//! the source tree.
+
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc")]

compiler/rustc_llvm/build.rs

@@ -24,8 +24,18 @@ const OPTIONAL_COMPONENTS: &[&str] = &[
     "bpf",
 ];
 
-const REQUIRED_COMPONENTS: &[&str] =
-    &["ipo", "bitreader", "bitwriter", "linker", "asmparser", "lto", "coverage", "instrumentation"];
+const REQUIRED_COMPONENTS: &[&str] = &[
+    "ipo",
+    "bitreader",
+    "bitwriter",
+    "linker",
+    "asmparser",
+    "lto",
+    "coverage",
+    "instrumentation",
+    "executionengine",
+    "orcjit",
+];
 
 fn detect_llvm_link() -> (&'static str, &'static str) {
     // Force the link mode we want, preferring static by default, but

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -11,6 +11,8 @@
 #if LLVM_VERSION_GE(16, 0)
 #include "llvm/Support/ModRef.h"
 #endif
+#include "llvm/ExecutionEngine/Orc/LLJIT.h"
+#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/ObjectFile.h"
@@ -33,6 +35,7 @@
 using namespace llvm;
 using namespace llvm::sys;
 using namespace llvm::object;
+using namespace llvm::orc;
 
 // LLVMAtomicOrdering is already an enum - don't create another
 // one.
@@ -1996,3 +1999,88 @@ extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) {
 extern "C" bool LLVMRustIsBitcode(char *ptr, size_t len) {
   return identify_magic(StringRef(ptr, len)) == file_magic::bitcode;
 }
+
+extern "C" LLJIT *LLVMRustCreateLLJIT() {
+  Expected<std::unique_ptr<LLJIT>> J = LLJITBuilder().create();
+  if (!J)
+    report_fatal_error(toString(J.takeError()).c_str());
+
+  return J->release();
+}
+
+extern "C" void LLVMRustLLJITLoadDynamicLibrary(LLJIT *J,
+                                                const char *FileName) {
+  auto Generator = DynamicLibrarySearchGenerator::Load(
+      FileName, J->getDataLayout().getGlobalPrefix());
+  if (!Generator)
+    report_fatal_error(toString(Generator.takeError()).c_str());
+
+  J->getMainJITDylib().addGenerator(std::move(*Generator));
+}
+
+extern "C" void LLVMRustLLJITAddIRModule(LLJIT *J, LLVMModuleRef ModuleRef) {
+  auto M = std::unique_ptr<Module>(unwrap(ModuleRef));
+  auto C = std::make_unique<LLVMContext>();
+  ThreadSafeModule TSM = ThreadSafeModule(std::move(M), std::move(C));
+  auto Error = J->addIRModule(std::move(TSM));
+  if (Error) {
+    std::string errorString;
+    llvm::raw_string_ostream stream(errorString);
+    stream << Error;
+    report_fatal_error(errorString.c_str());
+  }
+}
+
+extern "C" uint64_t LLVMRustLLJITLookup(LLJIT *J, const char *Name) {
+  auto Addr = J->lookup(Name);
+  if (!Addr)
+    report_fatal_error(toString(Addr.takeError()).c_str());
+
+  return Addr->getValue();
+}
+
+extern "C" void LLVMRustSetLinkageForAllFunctions(LLVMModuleRef M) {
+  auto Module = unwrap(M);
+  for (auto F = Module->getFunctionList().begin();
+       F != Module->getFunctionList().end(); F++) {
+    (*F).setLinkage(GlobalValue::LinkageTypes::LinkOnceODRLinkage);
+  }
+}
+
+extern "C" void LLVMRustAddGlobalCtor(LLVMContextRef C, LLVMModuleRef M,
+                                      LLVMValueRef Fn) {
+  auto AddrSpace = unwrap(M)->getDataLayout().getProgramAddressSpace();
+  auto CtorFnPtrTy =
+      PointerType::get(unwrap<FunctionType>(LLVMTypeOf(Fn)), AddrSpace);
+
+  auto VoidPtrTy = PointerType::get(Type::getVoidTy(*unwrap(C)), AddrSpace);
+
+  auto CtorTy =
+      StructType::get(Type::getInt32Ty(*unwrap(C)), CtorFnPtrTy, VoidPtrTy);
+
+  auto Ctor = ConstantStruct::get(
+      CtorTy, {
+                  ConstantInt::get(Type::getInt32Ty(*unwrap(C)), 65535),
+                  ConstantExpr::getBitCast(unwrap<Constant>(Fn), CtorFnPtrTy),
+                  ConstantPointerNull::get(VoidPtrTy),
+              });
+
+  auto CtorsTy = ArrayType::get(CtorTy, 1);
+  auto Ctors = ConstantArray::get(CtorsTy, {Ctor});
+
+  auto GlobalCtors = unwrap(M)->getOrInsertGlobal("llvm.global_ctors", CtorsTy);
+  unwrap(M)
+      ->getGlobalVariable("llvm.global_ctors")
+      ->setLinkage(GlobalValue::LinkageTypes::AppendingLinkage);
+  unwrap(M)->getGlobalVariable("llvm.global_ctors")->setInitializer(Ctors);
+}
+
+extern "C" void LLVMRustRunCtors(LLJIT *J) {
+  auto Error = J->initialize(J->getMainJITDylib());
+  if (Error) {
+    std::string errorString;
+    raw_string_ostream stream(errorString);
+    stream << Error;
+    report_fatal_error(errorString.c_str());
+  }
+}

compiler/rustc_session/src/options.rs

@@ -1485,6 +1485,8 @@ options! {
          `=skip-entry`
          `=skip-exit`
          Multiple options can be combined with commas."),
+    interactive: bool = (false, parse_bool, [TRACKED],
+        "enable interactive mode (default: no)"),
     keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED],
         "keep hygiene data after analysis (default: no)"),
     layout_seed: Option<u64> = (None, parse_opt_number, [TRACKED],

src/bootstrap/compile.rs

@@ -1535,16 +1535,13 @@ pub fn run_cargo(
                 // During check builds we need to keep crate metadata
                 keep = true;
             } else if rlib_only_metadata {
-                if filename.contains("jemalloc_sys") || filename.contains("rustc_smir") {
-                    // jemalloc_sys and rustc_smir are not linked into librustc_driver.so,
-                    // so we need to distribute them as rlib to be able to use them.
-                    keep |= filename.ends_with(".rlib");
-                } else {
-                    // Distribute the rest of the rustc crates as rmeta files only to reduce
-                    // the tarball sizes by about 50%. The object files are linked into
-                    // librustc_driver.so, so it is still possible to link against them.
-                    keep |= filename.ends_with(".rmeta");
-                }
+                // Don't distribute rustc crates as rmeta files because doing
+                // so will require rustc_driver to be imported, which causes
+                // a linking error.
+                //
+                // See <https://github.com/rust-lang/rust/issues/102065> for
+                // more info.
+                keep |= filename.ends_with(".rlib");
             } else {
                 // In all other cases keep all rlibs
                 keep |= filename.ends_with(".rlib");