move the alloc_jemalloc allocator out of tree, default to alloc_system

See rust-lang#36963 for rationale

closes rust-lang#36963
fixes rust-lang#36488 (indirectly)
closes rust-lang#37975

Author: Jorge Aparicio

.gitmodules

@@ -8,9 +8,6 @@
 [submodule "src/rt/hoedown"]
 	path = src/rt/hoedown
 	url = https://github.com/rust-lang/hoedown.git
-[submodule "src/jemalloc"]
-	path = src/jemalloc
-	url = https://github.com/rust-lang/jemalloc.git
 [submodule "src/rust-installer"]
 	path = src/rust-installer
 	url = https://github.com/rust-lang/rust-installer.git

configure

@@ -647,7 +647,6 @@ opt_nosave debug-assertions 0 "build with debugging assertions"
 opt_nosave llvm-release-debuginfo 0 "build LLVM with debugger metadata"
 opt_nosave debuginfo 0 "build with debugger metadata"
 opt_nosave debuginfo-lines 0 "build with line number debugger metadata"
-opt_nosave debug-jemalloc 0 "build jemalloc with --enable-debug --enable-fill"
 
 valopt localstatedir "/var/lib" "local state directory"
 valopt sysconfdir "/etc" "install system configuration files"
@@ -656,7 +655,6 @@ valopt datadir "${CFG_PREFIX}/share" "install data"
 valopt infodir "${CFG_PREFIX}/share/info" "install additional info"
 valopt llvm-root "" "set LLVM root"
 valopt python "" "set path to python"
-valopt jemalloc-root "" "set directory where libjemalloc_pic.a is located"
 valopt build "${DEFAULT_BUILD}" "GNUs ./configure syntax LLVM build triple"
 valopt android-cross-path "" "Android NDK standalone path (deprecated)"
 valopt i686-linux-android-ndk "" "i686-linux-android NDK standalone path"
@@ -691,7 +689,6 @@ valopt default-ar     "ar" "the default ar"
 # (others are conditionally saved).
 opt_nosave manage-submodules 1 "let the build manage the git submodules"
 opt_nosave clang 0 "prefer clang to gcc for building the runtime"
-opt_nosave jemalloc 1 "build liballoc with jemalloc"
 opt elf-tls 1 "elf thread local storage on platforms where supported"
 opt full-bootstrap 0 "build three compilers instead of two"
 
@@ -771,7 +768,6 @@ if [ -n "$CFG_ENABLE_DEBUG" ]; then
 
     # Set following variables to 1 unless setting already provided
     enable_if_not_disabled debug-assertions
-    enable_if_not_disabled debug-jemalloc
     enable_if_not_disabled debuginfo
     enable_if_not_disabled llvm-assertions
 fi
@@ -785,7 +781,6 @@ if [ -n "$CFG_ENABLE_DEBUG_ASSERTIONS" ]; then putvar CFG_ENABLE_DEBUG_ASSERTION
 if [ -n "$CFG_ENABLE_LLVM_RELEASE_DEBUGINFO" ]; then putvar CFG_ENABLE_LLVM_RELEASE_DEBUGINFO; fi
 if [ -n "$CFG_ENABLE_DEBUGINFO" ]; then putvar CFG_ENABLE_DEBUGINFO; fi
 if [ -n "$CFG_ENABLE_DEBUGINFO_LINES" ]; then putvar CFG_ENABLE_DEBUGINFO_LINES; fi
-if [ -n "$CFG_ENABLE_DEBUG_JEMALLOC" ]; then putvar CFG_ENABLE_DEBUG_JEMALLOC; fi
 
 step_msg "looking for build programs"
 
@@ -1065,12 +1060,6 @@ else
     CFG_USING_LIBCPP="0"
 fi
 
-# Same with jemalloc.  save the setting here.
-if [ -n "$CFG_DISABLE_JEMALLOC" ]
-then
-    putvar CFG_DISABLE_JEMALLOC
-fi
-
 if [ -n "$CFG_LLVM_ROOT" -a -z "$CFG_DISABLE_LLVM_VERSION_CHECK" -a -e "$CFG_LLVM_ROOT/bin/llvm-config" ]
 then
     step_msg "using custom LLVM at $CFG_LLVM_ROOT"
@@ -1415,7 +1404,6 @@ if [ -n "$CFG_DISABLE_RUSTBUILD" ]; then
     for s in 0 1 2 3
     do
       make_dir $t/rt/stage$s
-      make_dir $t/rt/jemalloc
       make_dir $t/rt/compiler-rt
       for i in                                          \
         isaac sync test \
@@ -1503,10 +1491,6 @@ then
         msg "git: submodule deinit src/llvm"
         "${CFG_GIT}" submodule deinit src/llvm
     fi
-    if [ -n "${CFG_JEMALLOC_ROOT}" ]; then
-        msg "git: submodule deinit src/jemalloc"
-        "${CFG_GIT}" submodule deinit src/jemalloc
-    fi
 
     msg "git: submodule update"
     "${CFG_GIT}" submodule update

src/Cargo.lock

@@ -338,7 +338,7 @@ pub fn krate(build: &Build,
             ("libtest", "src/rustc/test_shim", String::new(), "test_shim")
         }
         Mode::Librustc => {
-            ("librustc", "src/rustc", build.rustc_features(), "rustc-main")
+            ("librustc", "src/rustc", String::new(), "rustc-main")
         }
         _ => panic!("can only test libraries"),
     };
@@ -373,12 +373,6 @@ pub fn krate(build: &Build,
             let mut visited = HashSet::new();
             let mut next = vec![root];
             while let Some(name) = next.pop() {
-                // Right now jemalloc is our only target-specific crate in the sense
-                // that it's not present on all platforms. Custom skip it here for now,
-                // but if we add more this probably wants to get more generalized.
-                if !name.contains("jemalloc") {
-                    cargo.arg("-p").arg(name);
-                }
                 for dep in build.crates[name].deps.iter() {
                     if visited.insert(dep) {
                         next.push(dep);

src/bootstrap/check.rs

@@ -47,11 +47,6 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
          .arg("--manifest-path")
          .arg(build.src.join("src/rustc/std_shim/Cargo.toml"));
 
-    if let Some(target) = build.config.target_config.get(target) {
-        if let Some(ref jemalloc) = target.jemalloc {
-            cargo.env("JEMALLOC_OVERRIDE", jemalloc);
-        }
-    }
     if target.contains("musl") {
         if let Some(p) = build.musl_root(target) {
             cargo.env("MUSL_ROOT", p);
@@ -177,8 +172,7 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) {
     build.clear_if_dirty(&out_dir, &libtest_stamp(build, compiler, target));
 
     let mut cargo = build.cargo(compiler, Mode::Librustc, target, "build");
-    cargo.arg("--features").arg(build.rustc_features())
-         .arg("--manifest-path")
+    cargo.arg("--manifest-path")
          .arg(build.src.join("src/rustc/Cargo.toml"));
 
     // Set some configuration variables picked up by build scripts and

src/bootstrap/compile.rs

@@ -77,8 +77,6 @@ pub struct Config {
     pub local_rebuild: bool,
 
     // libstd features
-    pub debug_jemalloc: bool,
-    pub use_jemalloc: bool,
     pub backtrace: bool, // support for RUST_BACKTRACE
 
     // misc
@@ -101,7 +99,6 @@ pub struct Config {
 #[derive(Default)]
 pub struct Target {
     pub llvm_config: Option<PathBuf>,
-    pub jemalloc: Option<PathBuf>,
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
@@ -179,8 +176,6 @@ struct Rust {
     debug_assertions: Option<bool>,
     debuginfo: Option<bool>,
     debuginfo_lines: Option<bool>,
-    debug_jemalloc: Option<bool>,
-    use_jemalloc: Option<bool>,
     backtrace: Option<bool>,
     default_linker: Option<String>,
     default_ar: Option<String>,
@@ -196,7 +191,6 @@ struct Rust {
 #[derive(RustcDecodable, Default)]
 struct TomlTarget {
     llvm_config: Option<String>,
-    jemalloc: Option<String>,
     cc: Option<String>,
     cxx: Option<String>,
     android_ndk: Option<String>,
@@ -207,7 +201,6 @@ impl Config {
     pub fn parse(build: &str, file: Option<PathBuf>) -> Config {
         let mut config = Config::default();
         config.llvm_optimize = true;
-        config.use_jemalloc = true;
         config.backtrace = true;
         config.rust_optimize = true;
         config.rust_optimize_tests = true;
@@ -303,8 +296,6 @@ impl Config {
             set(&mut config.rust_debuginfo_tests, rust.debuginfo_tests);
             set(&mut config.codegen_tests, rust.codegen_tests);
             set(&mut config.rust_rpath, rust.rpath);
-            set(&mut config.debug_jemalloc, rust.debug_jemalloc);
-            set(&mut config.use_jemalloc, rust.use_jemalloc);
             set(&mut config.backtrace, rust.backtrace);
             set(&mut config.channel, rust.channel.clone());
             config.rustc_default_linker = rust.default_linker.clone();
@@ -325,9 +316,6 @@ impl Config {
                 if let Some(ref s) = cfg.llvm_config {
                     target.llvm_config = Some(env::current_dir().unwrap().join(s));
                 }
-                if let Some(ref s) = cfg.jemalloc {
-                    target.jemalloc = Some(env::current_dir().unwrap().join(s));
-                }
                 if let Some(ref s) = cfg.android_ndk {
                     target.ndk = Some(env::current_dir().unwrap().join(s));
                 }
@@ -390,8 +378,6 @@ impl Config {
                 ("DEBUG_ASSERTIONS", self.rust_debug_assertions),
                 ("DEBUGINFO", self.rust_debuginfo),
                 ("DEBUGINFO_LINES", self.rust_debuginfo_lines),
-                ("JEMALLOC", self.use_jemalloc),
-                ("DEBUG_JEMALLOC", self.debug_jemalloc),
                 ("RPATH", self.rust_rpath),
                 ("OPTIMIZE_TESTS", self.rust_optimize_tests),
                 ("DEBUGINFO_TESTS", self.rust_debuginfo_tests),
@@ -476,11 +462,6 @@ impl Config {
                     let root = parse_configure_path(value);
                     target.llvm_config = Some(push_exe_path(root, &["bin", "llvm-config"]));
                 }
-                "CFG_JEMALLOC_ROOT" if value.len() > 0 => {
-                    let target = self.target_config.entry(self.build.clone())
-                                     .or_insert(Target::default());
-                    target.jemalloc = Some(parse_configure_path(value));
-                }
                 "CFG_ARM_LINUX_ANDROIDEABI_NDK" if value.len() > 0 => {
                     let target = "arm-linux-androideabi".to_string();
                     let target = self.target_config.entry(target)

src/bootstrap/config.rs

@@ -149,12 +149,6 @@
 # Whether or not line number debug information is emitted
 #debuginfo-lines = false
 
-# Whether or not jemalloc is built and enabled
-#use-jemalloc = true
-
-# Whether or not jemalloc is built with its debug option set
-#debug-jemalloc = false
-
 # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE)
 #backtrace = true
 
@@ -208,10 +202,6 @@
 # target.
 #llvm-config = "../path/to/llvm/root/bin/llvm-config"
 
-# Path to the custom jemalloc static library to link into the standard library
-# by default. This is only used if jemalloc is still enabled above
-#jemalloc = "/path/to/jemalloc/libjemalloc_pic.a"
-
 # If this target is for Android, this option will be required to specify where
 # the NDK for the target lives. This is used to find the C compiler to link and
 # build native code.

src/bootstrap/config.toml.example

@@ -206,8 +206,7 @@ pub fn rustc(build: &Build, stage: u32, target: &str) {
     }
     let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc");
     cargo.arg("--manifest-path")
-         .arg(build.src.join("src/rustc/Cargo.toml"))
-         .arg("--features").arg(build.rustc_features());
+         .arg(build.src.join("src/rustc/Cargo.toml"));
     build.run(&mut cargo);
     cp_r(&out_dir, &out)
 }

src/bootstrap/doc.rs

@@ -386,12 +386,6 @@ impl Build {
                 continue
             }
 
-            if submodule.path.components().any(|c| c == Component::Normal("jemalloc".as_ref())) &&
-                !self.config.use_jemalloc
-            {
-                continue
-            }
-
             // `submodule.path` is the relative path to a submodule (from the repository root)
             // `submodule_path` is the path to a submodule from the cwd
 
@@ -609,27 +603,12 @@ impl Build {
     /// library.
     fn std_features(&self) -> String {
         let mut features = "panic-unwind".to_string();
-        if self.config.debug_jemalloc {
-            features.push_str(" debug-jemalloc");
-        }
-        if self.config.use_jemalloc {
-            features.push_str(" jemalloc");
-        }
         if self.config.backtrace {
             features.push_str(" backtrace");
         }
         return features
     }
 
-    /// Get the space-separated set of activated features for the compiler.
-    fn rustc_features(&self) -> String {
-        let mut features = String::new();
-        if self.config.use_jemalloc {
-            features.push_str(" jemalloc");
-        }
-        return features
-    }
-
     /// Component directory that Cargo will produce output into (e.g.
     /// release/debug)
     fn cargo_dir(&self) -> &'static str {

src/bootstrap/lib.rs

@@ -136,14 +136,6 @@ pub fn check(build: &mut Build) {
         need_cmd(build.cxx(host).as_ref());
     }
 
-    // The msvc hosts don't use jemalloc, turn it off globally to
-    // avoid packaging the dummy liballoc_jemalloc on that platform.
-    for host in build.config.host.iter() {
-        if host.contains("msvc") {
-            build.config.use_jemalloc = false;
-        }
-    }
-
     // Externally configured LLVM requires FileCheck to exist
     let filecheck = build.llvm_filecheck(&build.config.build);
     if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests {