Allow unbundled linking of rust libraries when using buck2

Summary: This diff ports the `native_unbundle_deps` feature from buck1 to buck2. The effect of this change is to make all `cxx_library` -> `rust_library` edges in the dep graph result in linking `rlib` artifacts rather than `staticlib` artifacts. The benefit being that we can then avoid issues with duplicate deps across libraries. rust-lang/rust#73632 has much more detail about this approach.

Reviewed By: zertosh

Differential Revision: D50523279

fbshipit-source-id: 16ee22db9140dba1e882d86aa376a03e010ef352

Author: capickett

rust/build.bzl

@@ -213,6 +213,7 @@ def generate_rustdoc_test(
     exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows"
 
     toolchain_info = compile_ctx.toolchain_info
+    native_unbundle_deps = toolchain_info.native_unbundle_deps
 
     resources = create_resource_db(
         ctx = ctx,
@@ -230,7 +231,7 @@ def generate_rustdoc_test(
     if link_style == LinkStyle("shared"):
         shlib_info = merge_shared_libraries(
             ctx.actions,
-            deps = inherited_shared_libs(ctx, include_doc_deps = True),
+            deps = inherited_shared_libs(ctx, native_unbundle_deps, include_doc_deps = True),
         )
         for soname, shared_lib in traverse_shared_library_info(shlib_info).items():
             shared_libs[soname] = shared_lib.lib
@@ -258,7 +259,7 @@ def generate_rustdoc_test(
             LinkArgs(flags = extra_link_args),
             get_link_args_for_strategy(
                 ctx,
-                inherited_merged_link_infos(ctx, include_doc_deps = True),
+                inherited_merged_link_infos(ctx, native_unbundle_deps, include_doc_deps = True),
                 # TODO(cjhopman): It's unclear how rust is using link_style. I'm not sure if it's intended to be a LibOutputStyle or a LinkStrategy.
                 to_link_strategy(link_style),
             ),
@@ -374,6 +375,7 @@ def rust_compile(
     exec_is_windows = ctx.attrs._exec_os_type[OsLookup].platform == "windows"
 
     toolchain_info = compile_ctx.toolchain_info
+    native_unbundle_deps = toolchain_info.native_unbundle_deps
 
     lints, clippy_lints = _lint_flags(compile_ctx)
 
@@ -444,6 +446,7 @@ def rust_compile(
                 ctx,
                 inherited_merged_link_infos(
                     ctx,
+                    native_unbundle_deps,
                     include_doc_deps = False,
                 ),
                 # TODO(cjhopman): It's unclear how rust is using link_style. I'm not sure if it's intended to be a LibOutputStyle or a LinkStrategy.

rust/build_params.bzl

@@ -124,8 +124,37 @@ def output_filename(cratename: str, emit: Emit, buildparams: BuildParams, extra:
 # Rule type - 'binary' also covers 'test'
 RuleType = enum("binary", "library")
 
-# What language we're generating artifacts to be linked with
-LinkageLang = enum("rust", "c++")
+# Controls how we build our rust libraries, largely dependent on whether rustc
+# or buck is driving the final linking and whether we are linking the artifact
+# into other rust targets.
+#
+# Rust: In this mode, we build rust libraries as rlibs. This is the primary
+# approach for building rust targets when the final link step is driven by
+# rustc (e.g. rust_binary, rust_unittest, etc).
+#
+# Native: In this mode, we build rust libraries as staticlibs, where rustc
+# will bundle all of this target's rust dependencies into a single library
+# artifact. This approach is the most standardized way to build rust libraries
+# for linkage in non-rust code.
+#
+# NOTE: This approach does not scale well. It's possible to end up with
+# non-rust target A depending on two rust targets B and C, which can cause
+# duplicate symbols if B and C share common rust dependencies.
+#
+# Native Unbundled: In this mode, we revert back to building as rlibs. This
+# approach mitigates the duplicate symbol downside of the "Native" approach.
+# However, this option is not formally supported by rustc, and depends on an
+# implementation detail of rlibs (they're effectively .a archives and can be
+# linked with other native code using the CXX linker).
+#
+# See https://github.com/rust-lang/rust/issues/73632 for more details on
+# stabilizing this approach.
+
+LinkageLang = enum(
+    "rust",
+    "native",
+    "native-unbundled",
+)
 
 _BINARY_SHARED = 0
 _BINARY_PIE = 1
@@ -231,10 +260,15 @@ _INPUTS = {
     ("binary", False, "static", "shared", "rust"): _BINARY_NON_PIE,
     ("binary", False, "static", "static", "rust"): _BINARY_NON_PIE,
     # Native linkable shared object
-    ("library", False, "shared", "any", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
-    ("library", False, "shared", "shared", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
-    ("library", False, "static", "shared", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
-    ("library", False, "static_pic", "shared", "c++"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "shared", "any", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "shared", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "static", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    ("library", False, "static_pic", "shared", "native"): _NATIVE_LINKABLE_SHARED_OBJECT,
+    # Native unbundled linkable shared object
+    ("library", False, "shared", "any", "native-unbundled"): _RUST_DYLIB_SHARED,
+    ("library", False, "shared", "shared", "native-unbundled"): _RUST_DYLIB_SHARED,
+    ("library", False, "static", "shared", "native-unbundled"): _RUST_DYLIB_SHARED,
+    ("library", False, "static_pic", "shared", "native-unbundled"): _RUST_DYLIB_SHARED,
     # Rust dylib shared object
     ("library", False, "shared", "any", "rust"): _RUST_DYLIB_SHARED,
     ("library", False, "shared", "shared", "rust"): _RUST_DYLIB_SHARED,
@@ -258,12 +292,19 @@ _INPUTS = {
     ("library", False, "static", "any", "rust"): _RUST_STATIC_NON_PIC_LIBRARY,
     ("library", False, "static", "static", "rust"): _RUST_STATIC_NON_PIC_LIBRARY,
     # Native linkable static_pic
-    ("library", False, "shared", "static", "c++"): _NATIVE_LINKABLE_STATIC_PIC,
-    ("library", False, "static_pic", "any", "c++"): _NATIVE_LINKABLE_STATIC_PIC,
-    ("library", False, "static_pic", "static", "c++"): _NATIVE_LINKABLE_STATIC_PIC,
+    ("library", False, "shared", "static", "native"): _NATIVE_LINKABLE_STATIC_PIC,
+    ("library", False, "static_pic", "any", "native"): _NATIVE_LINKABLE_STATIC_PIC,
+    ("library", False, "static_pic", "static", "native"): _NATIVE_LINKABLE_STATIC_PIC,
     # Native linkable static non-pic
-    ("library", False, "static", "any", "c++"): _NATIVE_LINKABLE_STATIC_NON_PIC,
-    ("library", False, "static", "static", "c++"): _NATIVE_LINKABLE_STATIC_NON_PIC,
+    ("library", False, "static", "any", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC,
+    ("library", False, "static", "static", "native"): _NATIVE_LINKABLE_STATIC_NON_PIC,
+    # Native Unbundled static_pic library
+    ("library", False, "shared", "static", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY,
+    ("library", False, "static_pic", "any", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY,
+    ("library", False, "static_pic", "static", "native-unbundled"): _RUST_STATIC_PIC_LIBRARY,
+    # Native Unbundled static (non-pic) library
+    ("library", False, "static", "any", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY,
+    ("library", False, "static", "static", "native-unbundled"): _RUST_STATIC_NON_PIC_LIBRARY,
 }
 
 # Check types of _INPUTS, writing these out as types is too verbose, but let's make sure we don't have any typos.

rust/link_info.bzl

@@ -172,7 +172,7 @@ RustCxxLinkGroupInfo = record(
     link_group_libs = field(dict[str, [LinkGroupLib, None]]),
     # mapping from target labels to the corresponding link group link_info
     labels_to_links_map = field(dict[Label, LinkGroupLinkInfo]),
-    # prefrred linkage mode for link group libraries
+    # preferred linkage mode for link group libraries
     link_group_preferred_linkage = field(dict[Label, Linkage]),
 )
 
@@ -265,8 +265,9 @@ def _create_linkable_graph(
     return linkable_graph
 
 # Returns native link dependencies.
-def _non_rust_link_deps(
+def _native_link_dependencies(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[Dependency]:
     """
     Return all first-order native linkable dependencies of all transitive Rust
@@ -276,30 +277,31 @@ def _non_rust_link_deps(
     looking for non-Rust native link infos (and terminating the search there).
     """
     first_order_deps = [dep.dep for dep in resolve_deps(ctx, include_doc_deps)]
-    return [
-        d
-        for d in first_order_deps
-        if RustLinkInfo not in d and MergedLinkInfo in d
-    ]
+
+    if native_unbundle_deps:
+        return [d for d in first_order_deps if MergedLinkInfo in d]
+    else:
+        return [
+            d
+            for d in first_order_deps
+            if RustLinkInfo not in d and MergedLinkInfo in d
+        ]
 
 # Returns native link dependencies.
-def _non_rust_link_infos(
+def _native_link_infos(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[MergedLinkInfo]:
     """
     Return all first-order native link infos of all transitive Rust libraries.
-
-    This emulates v1's graph walk, where it traverses through Rust libraries
-    looking for non-Rust native link infos (and terminating the search there).
-    MergedLinkInfo is a mapping from link style to all the transitive deps
-    rolled up in a tset.
     """
-    link_deps = _non_rust_link_deps(ctx, include_doc_deps)
+    link_deps = _native_link_dependencies(ctx, native_unbundle_deps, include_doc_deps)
     return [d[MergedLinkInfo] for d in link_deps]
 
 # Returns native link dependencies.
-def _non_rust_shared_lib_infos(
+def _native_shared_lib_infos(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[SharedLibraryInfo]:
     """
     Return all transitive shared libraries for non-Rust native linkabes.
@@ -308,11 +310,15 @@ def _non_rust_shared_lib_infos(
     Rust libraries to collect all transitive shared libraries.
     """
     first_order_deps = [dep.dep for dep in resolve_deps(ctx, include_doc_deps)]
-    return [
-        d[SharedLibraryInfo]
-        for d in first_order_deps
-        if RustLinkInfo not in d and SharedLibraryInfo in d
-    ]
+
+    if native_unbundle_deps:
+        return [d[SharedLibraryInfo] for d in first_order_deps if SharedLibraryInfo in d]
+    else:
+        return [
+            d[SharedLibraryInfo]
+            for d in first_order_deps
+            if RustLinkInfo not in d and SharedLibraryInfo in d
+        ]
 
 # Returns native link dependencies.
 def _rust_link_infos(
@@ -323,21 +329,21 @@ def _rust_link_infos(
 def normalize_crate(label: str) -> str:
     return label.replace("-", "_")
 
-# TODO(pickett): Currently this assumes the library target is being built as a
-# staticlib or cdylib.
-def inherited_exported_link_deps(ctx: AnalysisContext) -> list[Dependency]:
+def inherited_exported_link_deps(ctx: AnalysisContext, native_unbundle_deps: bool) -> list[Dependency]:
     deps = {}
-    for dep in _non_rust_link_deps(ctx):
+    for dep in _native_link_dependencies(ctx, native_unbundle_deps):
         deps[dep.label] = dep
-    for info in _rust_link_infos(ctx):
-        for dep in info.exported_link_deps:
-            deps[dep.label] = dep
+    if not native_unbundle_deps:
+        for info in _rust_link_infos(ctx):
+            for dep in info.exported_link_deps:
+                deps[dep.label] = dep
     return deps.values()
 
 def inherited_rust_cxx_link_group_info(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         link_style: [LinkStyle, None] = None) -> RustCxxLinkGroupInfo:
-    link_deps = inherited_exported_link_deps(ctx)
+    link_deps = inherited_exported_link_deps(ctx, native_unbundle_deps)
 
     # Assume a rust executable wants to use link groups if a link group map
     # is present
@@ -415,24 +421,24 @@ def inherited_rust_cxx_link_group_info(
         link_group_preferred_linkage = link_group_preferred_linkage,
     )
 
-# TODO(pickett): Currently this assumes the library target is being built as a
-# staticlib or cdylib.
 def inherited_merged_link_infos(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[MergedLinkInfo]:
     infos = []
-    infos.extend(_non_rust_link_infos(ctx, include_doc_deps))
-    infos.extend([d.merged_link_info for d in _rust_link_infos(ctx, include_doc_deps) if d.merged_link_info])
+    infos.extend(_native_link_infos(ctx, native_unbundle_deps, include_doc_deps))
+    if not native_unbundle_deps:
+        infos.extend([d.merged_link_info for d in _rust_link_infos(ctx, include_doc_deps) if d.merged_link_info])
     return infos
 
-# TODO(pickett): Currently this assumes the library target is being built as a
-# staticlib or cdylib.
 def inherited_shared_libs(
         ctx: AnalysisContext,
+        native_unbundle_deps: bool,
         include_doc_deps: bool = False) -> list[SharedLibraryInfo]:
     infos = []
-    infos.extend(_non_rust_shared_lib_infos(ctx, include_doc_deps))
-    infos.extend([d.shared_libs for d in _rust_link_infos(ctx, include_doc_deps)])
+    infos.extend(_native_shared_lib_infos(ctx, native_unbundle_deps, include_doc_deps))
+    if not native_unbundle_deps:
+        infos.extend([d.shared_libs for d in _rust_link_infos(ctx, include_doc_deps)])
     return infos
 
 def inherited_external_debug_info(

rust/rust_binary.bzl

@@ -97,6 +97,7 @@ def _rust_binary_common(
 
     target_os_type = ctx.attrs._target_os_type[OsLookup]
     linker_type = compile_ctx.cxx_toolchain_info.linker_info.type
+    native_unbundle_deps = compile_ctx.toolchain_info.native_unbundle_deps
 
     resources = flatten_dict(gather_resources(
         label = ctx.label,
@@ -133,6 +134,7 @@ def _rust_binary_common(
         if enable_link_groups(ctx, link_style, specified_link_style, is_binary = True):
             rust_cxx_link_group_info = inherited_rust_cxx_link_group_info(
                 ctx,
+                native_unbundle_deps,
                 link_style = link_style,
             )
             link_group_mappings = rust_cxx_link_group_info.link_group_info.mappings
@@ -147,7 +149,7 @@ def _rust_binary_common(
         if link_style == LinkStyle("shared") or rust_cxx_link_group_info != None:
             shlib_info = merge_shared_libraries(
                 ctx.actions,
-                deps = inherited_shared_libs(ctx),
+                deps = inherited_shared_libs(ctx, native_unbundle_deps),
             )
 
             link_group_ctx = LinkGroupContext(

rust/rust_library.bzl

@@ -138,15 +138,19 @@ def prebuilt_rust_library_impl(ctx: AnalysisContext) -> list[Provider]:
             pdb = None,
             external_debug_info = external_debug_info,
         )
+
+    # Prebuilt libraries only work in unbundled mode, as they only support `rlib`
+    # files today.
+    native_unbundle_deps = True
     providers.append(
         RustLinkInfo(
             crate = crate,
             styles = styles,
-            exported_link_deps = inherited_exported_link_deps(ctx),
-            merged_link_info = create_merged_link_info_for_propagation(ctx, inherited_merged_link_infos(ctx)),
+            exported_link_deps = inherited_exported_link_deps(ctx, native_unbundle_deps),
+            merged_link_info = create_merged_link_info_for_propagation(ctx, inherited_merged_link_infos(ctx, native_unbundle_deps)),
             shared_libs = merge_shared_libraries(
                 ctx.actions,
-                deps = inherited_shared_libs(ctx),
+                deps = inherited_shared_libs(ctx, native_unbundle_deps),
             ),
         ),
     )
@@ -237,7 +241,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]:
                 check_artifacts.update(meta.diag)
 
             rust_param_artifact[params] = _handle_rust_artifact(ctx, params, link, meta)
-        if LinkageLang("c++") in param_lang[params]:
+        if LinkageLang("native") in param_lang[params] or LinkageLang("native-unbundled") in param_lang[params]:
             native_param_artifact[params] = link
 
     # Among {rustdoc, doctests, macro expand}, doctests are the only one which
@@ -317,6 +321,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]:
     )
     providers += _rust_providers(
         ctx = ctx,
+        compile_ctx = compile_ctx,
         lang_style_param = lang_style_param,
         param_artifact = rust_param_artifact,
     )
@@ -364,8 +369,8 @@ def _build_params_for_styles(
 
     # Styles+lang linkage to params
     for linkage_lang in LinkageLang:
-        # Skip proc_macro + c++ combination
-        if ctx.attrs.proc_macro and linkage_lang == LinkageLang("c++"):
+        # Skip proc_macro + non-rust combinations
+        if ctx.attrs.proc_macro and linkage_lang != LinkageLang("rust"):
             continue
 
         for link_style in LinkStyle:
@@ -516,12 +521,14 @@ def _default_providers(
 
 def _rust_providers(
         ctx: AnalysisContext,
+        compile_ctx: CompileContext,
         lang_style_param: dict[(LinkageLang, LinkStyle), BuildParams],
         param_artifact: dict[BuildParams, RustLinkStyleInfo]) -> list[Provider]:
     """
     Return the set of providers for Rust linkage.
     """
     crate = attr_crate(ctx)
+    native_unbundle_deps = compile_ctx.toolchain_info.native_unbundle_deps
 
     style_info = {
         link_style: param_artifact[lang_style_param[(LinkageLang("rust"), link_style)]]
@@ -532,9 +539,9 @@ def _rust_providers(
     # non-Rust rules, found by walking through -- and ignoring -- Rust libraries
     # to find non-Rust native linkables and libraries.
     if not ctx.attrs.proc_macro:
-        inherited_link_deps = inherited_exported_link_deps(ctx)
-        inherited_link_infos = inherited_merged_link_infos(ctx)
-        inherited_shlibs = inherited_shared_libs(ctx)
+        inherited_link_deps = inherited_exported_link_deps(ctx, native_unbundle_deps)
+        inherited_link_infos = inherited_merged_link_infos(ctx, native_unbundle_deps)
+        inherited_shlibs = inherited_shared_libs(ctx, native_unbundle_deps)
     else:
         # proc-macros are just used by the compiler and shouldn't propagate
         # their native deps to the link line of the target.
@@ -566,14 +573,16 @@ def _native_providers(
     """
     Return the set of providers needed to link Rust as a dependency for native
     (ie C/C++) code, along with relevant dependencies.
-
-    TODO: This currently assumes `staticlib`/`cdylib` behaviour, where all
-    dependencies are bundled into the Rust crate itself. We need to break out of
-    this mode of operation.
     """
-    inherited_link_deps = inherited_exported_link_deps(ctx)
-    inherited_link_infos = inherited_merged_link_infos(ctx)
-    inherited_shlibs = inherited_shared_libs(ctx)
+
+    # If native_unbundle_deps is set on the the rust toolchain, then build this artifact
+    # using the "native-unbundled" linkage language. See LinkageLang docs for more details
+    native_unbundle_deps = compile_ctx.toolchain_info.native_unbundle_deps
+    lang = LinkageLang("native-unbundled") if native_unbundle_deps else LinkageLang("native")
+
+    inherited_link_deps = inherited_exported_link_deps(ctx, native_unbundle_deps)
+    inherited_link_infos = inherited_merged_link_infos(ctx, native_unbundle_deps)
+    inherited_shlibs = inherited_shared_libs(ctx, native_unbundle_deps)
     linker_info = compile_ctx.cxx_toolchain_info.linker_info
     linker_type = linker_info.type
 
@@ -593,7 +602,7 @@ def _native_providers(
     external_debug_infos = {}
     for output_style in LibOutputStyle:
         legacy_link_style = legacy_output_style_to_link_style(output_style)
-        params = lang_style_param[(LinkageLang("c++"), legacy_link_style)]
+        params = lang_style_param[(lang, legacy_link_style)]
         lib = param_artifact[params]
         libraries[output_style] = lib
 

rust/rust_toolchain.bzl

@@ -68,6 +68,10 @@ _rust_toolchain_attrs = {
     # Utilities used for building flagfiles containing dynamic crate names
     "concat_tool": None,
     "transitive_dependency_symlinks_tool": None,
+    # Passing true here enables the unstable feature using `rlib` format
+    # instead of `staticlib` when linking rust targets into native (e.g.
+    # C/C++) targets.
+    "native_unbundle_deps": False,
 }
 
 RustToolchainInfo = provider(fields = _rust_toolchain_attrs.keys())