ccinfo: when providing ccinfo, optionally include libstd and alloc (#624)

* ccinfo: when providing ccinfo, optionally include libstd and alloc

The new attribute on RustToolchain is the label of a target that
provides __rust_realloc et al, which allows ld(1) to use the .rlib
files directly without needing to involve rustc in the linking
step. This means Rust and C++ can be mixed in a cc_binary freely
without needing any staticlib-type crates, which avoids problems if
you have a cc_binary -> rust_library -> cc_library -> rust_library
situation.

* Add verification that all files were properly partitioned

* Disable print

* stdlib_ordering: new tests to ensure check stdlib rlib ordering

* native_deps: give up on testing cc_binary using rust_library

This only works on clang today, and can only work on clang for the
near-term. It's not clear how to write a clang-only test in this case,
so we just give up for now.

"I promise it works!" etc

Co-authored-by: Augie Fackler <[email protected]>
Co-authored-by: Marcel Hlopko <[email protected]>

Author: durin42

rust/private/rustc.bzl

@@ -693,6 +693,10 @@ def establish_cc_info(ctx, crate_info, toolchain, cc_toolchain, feature_configur
         if CcInfo in dep:
             cc_infos.append(dep[CcInfo])
 
+    if crate_info.type in ("rlib", "lib") and toolchain.libstd_and_allocator_ccinfo:
+        # TODO: if we already have an rlib in our deps, we could skip this
+        cc_infos.append(toolchain.libstd_and_allocator_ccinfo)
+
     return [cc_common.merge_cc_infos(cc_infos = cc_infos)]
 
 def add_edition_flags(args, crate):

rust/toolchain.bzl

@@ -1,5 +1,118 @@
 """The rust_toolchain rule definition and implementation."""
 
+load(
+    "//rust/private:utils.bzl",
+    "find_cc_toolchain",
+)
+
+def _make_dota(ctx, f):
+    """Add a symlink for a file that ends in .a, so it can be used as a staticlib.
+
+    Args:
+        ctx (ctx): The rule's context object.
+        f (File): The file to symlink.
+
+    Returns:
+        The symlink's File.
+    """
+    dot_a = ctx.actions.declare_file(f.basename + ".a", sibling = f)
+    ctx.actions.symlink(output = dot_a, target_file = f)
+    return dot_a
+
+def _ltl(library, ctx, cc_toolchain, feature_configuration):
+    return cc_common.create_library_to_link(
+        actions = ctx.actions,
+        feature_configuration = feature_configuration,
+        cc_toolchain = cc_toolchain,
+        static_library = library,
+        pic_static_library = library,
+    )
+
+def _make_libstd_and_allocator_ccinfo(ctx, rust_lib, allocator_library):
+    """Make the CcInfo (if possible) for libstd and allocator libraries.
+
+    Args:
+        ctx (ctx): The rule's context object.
+        rust_lib: The rust standard library.
+        allocator_library: The target to use for providing allocator functions.
+
+
+    Returns:
+        A CcInfo object for the required libraries, or None if no such libraries are available.
+    """
+    cc_toolchain, feature_configuration = find_cc_toolchain(ctx)
+    link_inputs = []
+    std_rlibs = [f for f in rust_lib.files.to_list() if f.basename.endswith(".rlib")]
+    if std_rlibs:
+        # std depends on everything
+        #
+        # core only depends on alloc, but we poke adler in there
+        # because that needs to be before miniz_oxide
+        #
+        # alloc depends on the allocator_library if it's configured, but we
+        # do that later.
+        dot_a_files = [_make_dota(ctx, f) for f in std_rlibs]
+
+        alloc_files = [f for f in dot_a_files if "alloc" in f.basename and "std" not in f.basename]
+        between_alloc_and_core_files = [f for f in dot_a_files if "compiler_builtins" in f.basename]
+        core_files = [f for f in dot_a_files if ("core" in f.basename or "adler" in f.basename) and "std" not in f.basename]
+        between_core_and_std_files = [
+            f
+            for f in dot_a_files
+            if "alloc" not in f.basename and "compiler_builtins" not in f.basename and "core" not in f.basename and "adler" not in f.basename and "std" not in f.basename
+        ]
+        std_files = [f for f in dot_a_files if "std" in f.basename]
+
+        partitioned_files_len = len(alloc_files) + len(between_alloc_and_core_files) + len(core_files) + len(between_core_and_std_files) + len(std_files)
+        if partitioned_files_len != len(dot_a_files):
+            partitioned = alloc_files + between_alloc_and_core_files + core_files + between_core_and_std_files + std_files
+            for f in sorted(partitioned):
+                # buildifier: disable=print
+                print("File partitioned: {}".format(f.basename))
+            fail("rust_toolchain couldn't properly partition rlibs in rust_lib. Partitioned {} out of {} files. This is probably a bug in the rule implementation.".format(partitioned_files_len, len(dot_a_files)))
+
+        alloc_inputs = depset(
+            [_ltl(f, ctx, cc_toolchain, feature_configuration) for f in alloc_files],
+        )
+        between_alloc_and_core_inputs = depset(
+            [_ltl(f, ctx, cc_toolchain, feature_configuration) for f in between_alloc_and_core_files],
+            transitive = [alloc_inputs],
+            order = "topological",
+        )
+        core_inputs = depset(
+            [_ltl(f, ctx, cc_toolchain, feature_configuration) for f in core_files],
+            transitive = [between_alloc_and_core_inputs],
+            order = "topological",
+        )
+        between_core_and_std_inputs = depset(
+            [_ltl(f, ctx, cc_toolchain, feature_configuration) for f in between_core_and_std_files],
+            transitive = [core_inputs],
+            order = "topological",
+        )
+        std_inputs = depset(
+            [_ltl(f, ctx, cc_toolchain, feature_configuration) for f in std_files],
+            transitive = [between_core_and_std_inputs],
+            order = "topological",
+        )
+
+        link_inputs.append(cc_common.create_linker_input(
+            owner = rust_lib.label,
+            libraries = std_inputs,
+        ))
+
+    allocator_inputs = None
+    if allocator_library:
+        allocator_inputs = [allocator_library[CcInfo].linking_context.linker_inputs]
+
+    libstd_and_allocator_ccinfo = None
+    if link_inputs:
+        return CcInfo(linking_context = cc_common.create_linking_context(linker_inputs = depset(
+            link_inputs,
+            transitive = allocator_inputs,
+            order = "topological",
+        )))
+    return None
+
 def _rust_toolchain_impl(ctx):
     """The rust_toolchain implementation
 
@@ -38,12 +151,17 @@ def _rust_toolchain_impl(ctx):
         default_edition = ctx.attr.default_edition,
         compilation_mode_opts = compilation_mode_opts,
         crosstool_files = ctx.files._crosstool,
+        libstd_and_allocator_ccinfo = _make_libstd_and_allocator_ccinfo(ctx, ctx.attr.rust_lib, ctx.attr.allocator_library),
     )
     return [toolchain]
 
 rust_toolchain = rule(
     implementation = _rust_toolchain_impl,
+    fragments = ["cpp"],
     attrs = {
+        "allocator_library": attr.label(
+            doc = "Target that provides allocator functions when rust_library targets are embedded in a cc_binary.",
+        ),
         "binary_ext": attr.string(
             doc = "The extension for binaries created from rustc.",
             mandatory = True,
@@ -128,6 +246,9 @@ rust_toolchain = rule(
                 "For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations"
             ),
         ),
+        "_cc_toolchain": attr.label(
+            default = "@bazel_tools//tools/cpp:current_cc_toolchain",
+        ),
         "_crosstool": attr.label(
             default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
         ),

test/unit/cc_info/cc_info_test.bzl

@@ -6,11 +6,11 @@ load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_
 def _is_dylib_on_windows(ctx):
     return ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo])
 
-def _assert_cc_info_has_library_to_link(env, tut, type):
+def _assert_cc_info_has_library_to_link(env, tut, type, ccinfo_count):
     asserts.true(env, CcInfo in tut, "rust_library should provide CcInfo")
     cc_info = tut[CcInfo]
     linker_inputs = cc_info.linking_context.linker_inputs.to_list()
-    asserts.equals(env, len(linker_inputs), 1)
+    asserts.equals(env, len(linker_inputs), ccinfo_count)
     library_to_link = linker_inputs[0].libraries[0]
     asserts.equals(env, False, library_to_link.alwayslink)
 
@@ -42,7 +42,7 @@ def _assert_cc_info_has_library_to_link(env, tut, type):
 def _rlib_provides_cc_info_test_impl(ctx):
     env = analysistest.begin(ctx)
     tut = analysistest.target_under_test(env)
-    _assert_cc_info_has_library_to_link(env, tut, "rlib")
+    _assert_cc_info_has_library_to_link(env, tut, "rlib", 2)
     return analysistest.end(env)
 
 def _bin_does_not_provide_cc_info_test_impl(ctx):
@@ -60,13 +60,13 @@ def _proc_macro_does_not_provide_cc_info_test_impl(ctx):
 def _cdylib_provides_cc_info_test_impl(ctx):
     env = analysistest.begin(ctx)
     tut = analysistest.target_under_test(env)
-    _assert_cc_info_has_library_to_link(env, tut, "cdylib")
+    _assert_cc_info_has_library_to_link(env, tut, "cdylib", 1)
     return analysistest.end(env)
 
 def _staticlib_provides_cc_info_test_impl(ctx):
     env = analysistest.begin(ctx)
     tut = analysistest.target_under_test(env)
-    _assert_cc_info_has_library_to_link(env, tut, "staticlib")
+    _assert_cc_info_has_library_to_link(env, tut, "staticlib", 1)
     return analysistest.end(env)
 
 rlib_provides_cc_info_test = analysistest.make(_rlib_provides_cc_info_test_impl)

test/unit/stdlib_ordering/BUILD

@@ -0,0 +1,4 @@
+load(":stdlib_ordering.bzl", "stdlib_ordering_suite")
+
+############################ UNIT TESTS #############################
+stdlib_ordering_suite(name = "stdlib_ordering_suite")

test/unit/stdlib_ordering/some_rlib.rs

@@ -0,0 +1,3 @@
+pub fn print_stuff() {
+    println!("stuff");
+}

test/unit/stdlib_ordering/stdlib_ordering.bzl

@@ -0,0 +1,63 @@
+"""Unittest to verify ordering of rust stdlib in rust_library() CcInfo"""
+load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts")
+load("//rust:defs.bzl", "rust_library")
+
+def _categorize_library(name):
+    """Given an rlib name, guess if it's std, core, or alloc."""
+    if "std" in name:
+        return "std"
+    if "core" in name:
+        return "core"
+    if "alloc" in name:
+        return "alloc"
+    if "compiler_builtins" in name:
+        return "compiler_builtins"
+    return "other"
+
+def _dedup_preserving_order(list):
+    """Given a list, deduplicate its elements preserving order."""
+    r = []
+    seen = {}
+    for e in list:
+        if e in seen:
+            continue
+        seen[e] = 1
+        r.append(e)
+    return r
+
+def _libstd_ordering_test_impl(ctx):
+    env = analysistest.begin(ctx)
+    tut = analysistest.target_under_test(env)
+    libs = [lib.static_library for li in tut[CcInfo].linking_context.linker_inputs.to_list() for lib in li.libraries]
+    rlibs = [_categorize_library(lib.basename) for lib in libs if ".rlib" in lib.basename]
+    set_to_check = _dedup_preserving_order([lib for lib in rlibs if lib != "other"])
+    asserts.equals(env, ["std", "core", "compiler_builtins", "alloc"], set_to_check)
+    return analysistest.end(env)
+
+libstd_ordering_test = analysistest.make(_libstd_ordering_test_impl)
+
+def _native_dep_test():
+    rust_library(
+        name = "some_rlib",
+        srcs = ["some_rlib.rs"],
+    )
+
+    libstd_ordering_test(
+        name = "libstd_ordering_test",
+        target_under_test = ":some_rlib",
+    )
+
+def stdlib_ordering_suite(name):
+    """Entry-point macro called from the BUILD file.
+
+    Args:
+        name: Name of the macro.
+    """
+    _native_dep_test()
+
+    native.test_suite(
+        name = name,
+        tests = [
+            ":libstd_ordering_test",
+        ],
+    )