coverage. Adapt to llvm optimization for conditions limits

Author: zhuyunxing

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1719,9 +1719,9 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        bitmap_bytes: &'ll Value,
+        bitmap_bits: &'ll Value,
     ) {
-        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes);
+        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits);
 
         assert!(llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later");
 
@@ -1730,7 +1730,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()],
             self.cx.type_void(),
         );
-        let args = &[fn_name, hash, bitmap_bytes];
+        let args = &[fn_name, hash, bitmap_bits];
         let args = self.check_call("call", llty, llfn, args);
 
         unsafe {
@@ -1750,29 +1750,22 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        bitmap_bytes: &'ll Value,
         bitmap_index: &'ll Value,
         mcdc_temp: &'ll Value,
     ) {
         debug!(
-            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp
+            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})",
+            fn_name, hash, bitmap_index, mcdc_temp
         );
         assert!(llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later");
 
         let llfn =
             unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) };
         let llty = self.cx.type_func(
-            &[
-                self.cx.type_ptr(),
-                self.cx.type_i64(),
-                self.cx.type_i32(),
-                self.cx.type_i32(),
-                self.cx.type_ptr(),
-            ],
+            &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()],
             self.cx.type_void(),
         );
-        let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp];
+        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
         let args = self.check_call("call", llty, llfn, args);
         unsafe {
             let _ = llvm::LLVMRustBuildCall(
@@ -1792,38 +1785,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        cond_loc: &'ll Value,
+        cond_index: &'ll Value,
         mcdc_temp: &'ll Value,
-        bool_value: &'ll Value,
     ) {
         debug!(
-            "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, cond_loc, mcdc_temp, bool_value
+            "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?})",
+            fn_name, hash, cond_index, mcdc_temp
         );
         assert!(llvm_util::get_version() >= (19, 0, 0), "MCDC intrinsics require LLVM 19 or later");
-        let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) };
-        let llty = self.cx.type_func(
-            &[
-                self.cx.type_ptr(),
-                self.cx.type_i64(),
-                self.cx.type_i32(),
-                self.cx.type_ptr(),
-                self.cx.type_i1(),
-            ],
-            self.cx.type_void(),
-        );
-        let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value];
-        self.check_call("call", llty, llfn, args);
-        unsafe {
-            let _ = llvm::LLVMRustBuildCall(
-                self.llbuilder,
-                llty,
-                llfn,
-                args.as_ptr() as *const &llvm::Value,
-                args.len() as c_uint,
-                [].as_ptr(),
-                0 as c_uint,
-            );
-        }
+        let align = self.tcx.data_layout.i32_align.abi;
+        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
+        let new_tv_index = self.add(current_tv_index, cond_index);
+        self.store(new_tv_index, mcdc_temp, align);
     }
 }

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -100,14 +100,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
         };
 
         // If there are no MC/DC bitmaps to set up, return immediately.
-        if function_coverage_info.mcdc_bitmap_bytes == 0 {
+        if function_coverage_info.mcdc_bitmap_bits == 0 {
             return;
         }
 
         let fn_name = self.get_pgo_func_name_var(instance);
         let hash = self.const_u64(function_coverage_info.function_source_hash);
-        let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes);
-        self.mcdc_parameters(fn_name, hash, bitmap_bytes);
+        let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32);
+        self.mcdc_parameters(fn_name, hash, bitmap_bits);
 
         // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps.
         let mut cond_bitmaps = vec![];
@@ -187,35 +187,30 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
             CoverageKind::ExpressionUsed { id } => {
                 func_coverage.mark_expression_id_seen(id);
             }
-            CoverageKind::CondBitmapUpdate { id, value, decision_depth } => {
+            CoverageKind::CondBitmapUpdate { index, decision_depth } => {
                 drop(coverage_map);
-                assert_ne!(
-                    id.as_u32(),
-                    0,
-                    "ConditionId of evaluated conditions should never be zero"
-                );
                 let cond_bitmap = coverage_context
                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)
                     .expect("mcdc cond bitmap should have been allocated for updating");
-                let cond_loc = bx.const_i32(id.as_u32() as i32 - 1);
-                let bool_value = bx.const_bool(value);
+                let cond_index = bx.const_i32(index as i32);
                 let fn_name = bx.get_pgo_func_name_var(instance);
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value);
+                bx.mcdc_condbitmap_update(fn_name, hash, cond_index, cond_bitmap);
             }
             CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
                 drop(coverage_map);
                 let cond_bitmap = coverage_context
                                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)
                                     .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap");
-                let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes;
-                assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range");
+                assert!(
+                    bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits,
+                    "bitmap index of the decision out of range"
+                );
 
                 let fn_name = bx.get_pgo_func_name_var(instance);
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                let bitmap_bytes = bx.const_u32(bitmap_bytes);
                 let bitmap_index = bx.const_u32(bitmap_idx);
-                bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap);
+                bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap);
             }
         }
     }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1635,7 +1635,6 @@ extern "C" {
     pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;
     pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value;
     pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value;
-    pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value;
 
     pub fn LLVMRustBuildCall<'a>(
         B: &Builder<'a>,

compiler/rustc_middle/src/mir/coverage.rs

@@ -136,8 +136,8 @@ pub enum CoverageKind {
 
     /// Marks the point in MIR control flow represented by a evaluated condition.
     ///
-    /// This is eventually lowered to `llvm.instrprof.mcdc.condbitmap.update` in LLVM IR.
-    CondBitmapUpdate { id: ConditionId, value: bool, decision_depth: u16 },
+    /// This is eventually lowered to instruments updating mcdc temp variables.
+    CondBitmapUpdate { index: u32, decision_depth: u16 },
 
     /// Marks the point in MIR control flow represented by a evaluated decision.
     ///
@@ -153,14 +153,8 @@ impl Debug for CoverageKind {
             BlockMarker { id } => write!(fmt, "BlockMarker({:?})", id.index()),
             CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
             ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
-            CondBitmapUpdate { id, value, decision_depth } => {
-                write!(
-                    fmt,
-                    "CondBitmapUpdate({:?}, {:?}, depth={:?})",
-                    id.index(),
-                    value,
-                    decision_depth
-                )
+            CondBitmapUpdate { index, decision_depth } => {
+                write!(fmt, "CondBitmapUpdate(index={:?}, depth={:?})", index, decision_depth)
             }
             TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
                 write!(fmt, "TestVectorUpdate({:?}, depth={:?})", bitmap_idx, decision_depth)
@@ -274,7 +268,7 @@ pub struct Mapping {
 pub struct FunctionCoverageInfo {
     pub function_source_hash: u64,
     pub num_counters: usize,
-    pub mcdc_bitmap_bytes: u32,
+    pub mcdc_bitmap_bits: usize,
     pub expressions: IndexVec<ExpressionId, Expression>,
     pub mappings: Vec<Mapping>,
     /// The depth of the deepest decision is used to know how many

compiler/rustc_mir_build/src/build/coverageinfo/mcdc.rs

@@ -3,6 +3,7 @@ use std::collections::VecDeque;
 use std::rc::Rc;
 
 use rustc_data_structures::fx::FxIndexMap;
+
 use rustc_middle::bug;
 use rustc_middle::mir::coverage::{
     BlockMarkerId, ConditionId, ConditionInfo, DecisionId, MCDCBranchSpan, MCDCDecisionSpan,
@@ -15,10 +16,9 @@ use rustc_span::Span;
 use crate::build::Builder;
 use crate::errors::{MCDCExceedsConditionLimit, MCDCExceedsDecisionDepth};
 
-/// The MCDC bitmap scales exponentially (2^n) based on the number of conditions seen,
-/// So llvm sets a maximum value prevents the bitmap footprint from growing too large without the user's knowledge.
-/// This limit may be relaxed if the [upstream change](https://github.com/llvm/llvm-project/pull/82448) is merged.
-const MAX_CONDITIONS_IN_DECISION: usize = 6;
+/// LLVM uses `i16` to represent condition id. Hence `i16::MAX` is the hard limit for number of
+/// conditions in a decision.
+const MAX_CONDITIONS_IN_DECISION: usize = i16::MAX as usize;
 
 /// MCDC allocates an i32 variable on stack for each depth. Ignore decisions nested too much to prevent it
 /// consuming excessive memory.