rustc: Link entire archives of native libraries

As discovered in #15460, a particular #[link(kind = "static", ...)] line is not
actually guaranteed to link the library at all. The reason for this is that if
the external library doesn't have any referenced symbols in the object generated
by rustc, the entire library is dropped by the linker.

For dynamic native libraries, this is solved by passing -lfoo for all downstream
compilations unconditionally. For static libraries in rlibs this is solved
because the entire archive is bundled in the rlib. The only situation in which
this was a problem was when a static native library was linked to a rust dynamic
library.

This commit brings the behavior of dylibs in line with rlibs by passing the
--whole-archive flag to the linker when linking native libraries. On OSX, this
uses the -force_load flag. This flag ensures that the entire archive is
considered candidate for being linked into the final dynamic library.

This is a breaking change because if any static library is included twice in the
same compilation unit then the linker will start emitting errors about duplicate
definitions now. The fix for this would involve only statically linking to a
library once.

Closes #15460
[breaking-change]

Author: alexcrichton

src/liballoc/heap.rs

@@ -136,6 +136,9 @@ mod imp {
     use libc::{c_char, c_int, c_void, size_t};
 
     #[link(name = "jemalloc", kind = "static")]
+    #[cfg(not(test))]
+    extern {}
+
     extern {
         fn je_mallocx(size: size_t, flags: c_int) -> *mut c_void;
         fn je_rallocx(ptr: *mut c_void, size: size_t,

src/librustc/back/link.rs

@@ -9,6 +9,7 @@
 // except according to those terms.
 
 use super::archive::{Archive, ArchiveBuilder, ArchiveConfig, METADATA_FILENAME};
+use super::archive;
 use super::rpath;
 use super::rpath::RPathConfig;
 use super::svh::Svh;
@@ -1597,29 +1598,61 @@ fn add_local_native_libraries(cmd: &mut Command, sess: &Session) {
     // For those that support this, we ensure we pass the option if the library
     // was flagged "static" (most defaults are dynamic) to ensure that if
     // libfoo.a and libfoo.so both exist that the right one is chosen.
-    let takes_hints = sess.targ_cfg.os != abi::OsMacos && sess.targ_cfg.os != abi::OsiOS;
+    let takes_hints = sess.targ_cfg.os != abi::OsMacos &&
+                      sess.targ_cfg.os != abi::OsiOS;
+
+    let libs = sess.cstore.get_used_libraries();
+    let libs = libs.borrow();
+
+    let mut staticlibs = libs.iter().filter_map(|&(ref l, kind)| {
+        if kind == cstore::NativeStatic {Some(l)} else {None}
+    });
+    let mut others = libs.iter().filter(|&&(_, kind)| {
+        kind != cstore::NativeStatic
+    });
+
+    // Platforms that take hints generally also support the --whole-archive
+    // flag. We need to pass this flag when linking static native libraries to
+    // ensure the entire library is included.
+    //
+    // For more details see #15460, but the gist is that the linker will strip
+    // away any unused objects in the archive if we don't otherwise explicitly
+    // reference them. This can occur for libraries which are just providing
+    // bindings, libraries with generic functions, etc.
+    if takes_hints {
+        cmd.arg("-Wl,--whole-archive").arg("-Wl,-Bstatic");
+    }
+    let search_path = archive_search_paths(sess);
+    for l in staticlibs {
+        if takes_hints {
+            cmd.arg(format!("-l{}", l));
+        } else {
+            // -force_load is the OSX equivalent of --whole-archive, but it
+            // involves passing the full path to the library to link.
+            let lib = archive::find_library(l.as_slice(),
+                                            sess.targ_cfg.os,
+                                            search_path.as_slice(),
+                                            &sess.diagnostic().handler);
+            let mut v = b"-Wl,-force_load,".to_vec();
+            v.push_all(lib.as_vec());
+            cmd.arg(v.as_slice());
+        }
+    }
+    if takes_hints {
+        cmd.arg("-Wl,--no-whole-archive").arg("-Wl,-Bdynamic");
+    }
 
-    for &(ref l, kind) in sess.cstore.get_used_libraries().borrow().iter() {
+    for &(ref l, kind) in others {
         match kind {
-            cstore::NativeUnknown | cstore::NativeStatic => {
-                if takes_hints {
-                    if kind == cstore::NativeStatic {
-                        cmd.arg("-Wl,-Bstatic");
-                    } else {
-                        cmd.arg("-Wl,-Bdynamic");
-                    }
-                }
-                cmd.arg(format!("-l{}", *l));
+            cstore::NativeUnknown => {
+                cmd.arg(format!("-l{}", l));
             }
             cstore::NativeFramework => {
-                cmd.arg("-framework");
-                cmd.arg(l.as_slice());
+                cmd.arg("-framework").arg(l.as_slice());
             }
+            cstore::NativeStatic => unreachable!(),
         }
     }
-    if takes_hints {
-        cmd.arg("-Wl,-Bdynamic");
-    }
 }
 
 // # Rust Crate linking

src/librustc_back/archive.rs

@@ -95,6 +95,30 @@ fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
     }
 }
 
+pub fn find_library(name: &str, os: abi::Os, search_paths: &[Path],
+                    handler: &ErrorHandler) -> Path {
+    let (osprefix, osext) = match os {
+        abi::OsWin32 => ("", "lib"), _ => ("lib", "a"),
+    };
+    // On Windows, static libraries sometimes show up as libfoo.a and other
+    // times show up as foo.lib
+    let oslibname = format!("{}{}.{}", osprefix, name, osext);
+    let unixlibname = format!("lib{}.a", name);
+
+    for path in search_paths.iter() {
+        debug!("looking for {} inside {}", name, path.display());
+        let test = path.join(oslibname.as_slice());
+        if test.exists() { return test }
+        if oslibname != unixlibname {
+            let test = path.join(unixlibname.as_slice());
+            if test.exists() { return test }
+        }
+    }
+    handler.fatal(format!("could not find native static library `{}`, \
+                           perhaps an -L flag is missing?",
+                          name).as_slice());
+}
+
 impl<'a> Archive<'a> {
     fn new(config: ArchiveConfig<'a>) -> Archive<'a> {
         let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
@@ -153,7 +177,9 @@ impl<'a> ArchiveBuilder<'a> {
     /// Adds all of the contents of a native library to this archive. This will
     /// search in the relevant locations for a library named `name`.
     pub fn add_native_library(&mut self, name: &str) -> io::IoResult<()> {
-        let location = self.find_library(name);
+        let location = find_library(name, self.archive.os,
+                                    self.archive.lib_search_paths.as_slice(),
+                                    self.archive.handler);
         self.add_archive(&location, name, [])
     }
 
@@ -285,28 +311,5 @@ impl<'a> ArchiveBuilder<'a> {
         }
         Ok(())
     }
-
-    fn find_library(&self, name: &str) -> Path {
-        let (osprefix, osext) = match self.archive.os {
-            abi::OsWin32 => ("", "lib"), _ => ("lib", "a"),
-        };
-        // On Windows, static libraries sometimes show up as libfoo.a and other
-        // times show up as foo.lib
-        let oslibname = format!("{}{}.{}", osprefix, name, osext);
-        let unixlibname = format!("lib{}.a", name);
-
-        for path in self.archive.lib_search_paths.iter() {
-            debug!("looking for {} inside {}", name, path.display());
-            let test = path.join(oslibname.as_slice());
-            if test.exists() { return test }
-            if oslibname != unixlibname {
-                let test = path.join(unixlibname.as_slice());
-                if test.exists() { return test }
-            }
-        }
-        self.archive.handler.fatal(format!("could not find native static library `{}`, \
-                                            perhaps an -L flag is missing?",
-                                           name).as_slice());
-    }
 }
 

src/librustrt/unwind.rs

@@ -159,6 +159,9 @@ pub unsafe fn try(f: ||) -> ::core::result::Result<(), Box<Any + Send>> {
     }
 
     #[link(name = "rustrt_native", kind = "static")]
+    #[cfg(not(test))]
+    extern {}
+
     extern {
         // Rust's try-catch
         // When f(...) returns normally, the return value is null.

src/libstd/rt/backtrace.rs

@@ -415,6 +415,9 @@ mod imp {
                           errnum: libc::c_int);
         enum backtrace_state {}
         #[link(name = "backtrace", kind = "static")]
+        #[cfg(not(test))]
+        extern {}
+
         extern {
             fn backtrace_create_state(filename: *const libc::c_char,
                                       threaded: libc::c_int,

src/libstd/rtdeps.rs

@@ -15,6 +15,7 @@
 #![experimental]
 
 // All platforms need to link to rustrt
+#[cfg(not(test))]
 #[link(name = "rust_builtin", kind = "static")]
 extern {}
 

src/test/run-make/extern-fn-with-union/test.rs

@@ -12,7 +12,6 @@ extern crate testcrate;
 
 use std::mem;
 
-#[link(name = "test", kind = "static")]
 extern {
     fn give_back(tu: testcrate::TestUnion) -> u64;
 }

src/test/run-make/issue-15460/Makefile

@@ -0,0 +1,6 @@
+-include ../tools.mk
+
+all: $(TMPDIR)/libfoo.a
+	$(RUSTC) foo.rs -C extra-filename=-383hf8
+	$(RUSTC) bar.rs
+	$(call RUN,bar)

src/test/run-make/issue-15460/bar.rs

@@ -0,0 +1,14 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate foo;
+fn main() {
+    unsafe { foo::foo() }
+}

src/test/run-make/issue-15460/foo.c

@@ -0,0 +1 @@
+void foo() {}

src/test/run-make/issue-15460/foo.rs

@@ -0,0 +1,16 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![crate_type = "dylib"]
+
+#[link(name = "foo", kind = "static")]
+extern {
+    pub fn foo();
+}

src/test/run-make/no-duplicate-libs/Makefile

@@ -1,6 +1,7 @@
 -include ../tools.mk
 
-all: $(call STATICLIB,foo) $(call STATICLIB,bar)
+all:
+	$(RUSTC) foo.rs
+	$(RUSTC) bar.rs
 	$(RUSTC) main.rs
 	$(call RUN,main)
-

src/test/run-make/no-duplicate-libs/bar.rs

@@ -0,0 +1,21 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![no_std]
+#![feature(lang_items)]
+#![crate_type = "dylib"]
+
+extern crate libc;
+
+#[no_mangle]
+pub extern fn bar() {}
+
+#[lang = "stack_exhausted"] fn stack_exhausted() {}
+#[lang = "eh_personality"] fn eh_personality() {}

src/test/run-make/no-duplicate-libs/foo.rs

@@ -0,0 +1,21 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![no_std]
+#![feature(lang_items)]
+#![crate_type = "dylib"]
+
+extern crate libc;
+
+#[no_mangle]
+pub extern fn foo() {}
+
+#[lang = "stack_exhausted"] fn stack_exhausted() {}
+#[lang = "eh_personality"] fn eh_personality() {}

src/test/run-pass/foreign-dupe.rs

@@ -22,7 +22,6 @@ mod rustrt1 {
 mod rustrt2 {
     extern crate libc;
 
-    #[link(name = "rust_test_helpers")]
     extern {
         pub fn rust_get_test_int() -> libc::intptr_t;
     }