use crate::llvm;
use crate::builder::Builder;
use crate::common::CodegenCx;
use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion};
use crate::coverageinfo::map_data::FunctionCoverageCollector;
use libc::c_uint;
use rustc_codegen_ssa::traits::{
BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods,
StaticMethods,
};
use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
use rustc_llvm::RustString;
use rustc_middle::bug;
use rustc_middle::mir::coverage::CoverageKind;
use rustc_middle::ty::layout::HasTyCtxt;
use rustc_middle::ty::Instance;
use rustc_target::abi::{Align, Size};
use tracing::{debug, instrument};
use std::cell::RefCell;
pub(crate) mod ffi;
pub(crate) mod map_data;
pub mod mapgen;
pub struct CrateCoverageContext<'ll, 'tcx> {
pub(crate) function_coverage_map:
RefCell<FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>>>,
pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
pub(crate) mcdc_condition_bitmap_map: RefCell<FxHashMap<Instance<'tcx>, Vec<&'ll llvm::Value>>>,
}
impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
pub fn new() -> Self {
Self {
function_coverage_map: Default::default(),
pgo_func_name_var_map: Default::default(),
mcdc_condition_bitmap_map: Default::default(),
}
}
pub fn take_function_coverage_map(
&self,
) -> FxIndexMap<Instance<'tcx>, FunctionCoverageCollector<'tcx>> {
self.function_coverage_map.replace(FxIndexMap::default())
}
fn try_get_mcdc_condition_bitmap(
&self,
instance: &Instance<'tcx>,
decision_depth: u16,
) -> Option<&'ll llvm::Value> {
self.mcdc_condition_bitmap_map
.borrow()
.get(instance)
.and_then(|bitmap_map| bitmap_map.get(decision_depth as usize))
.copied() }
}
impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
pub(crate) fn coverageinfo_finalize(&self) {
mapgen::finalize(self)
}
fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
if let Some(coverage_context) = self.coverage_context() {
debug!("getting pgo_func_name_var for instance={:?}", instance);
let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut();
pgo_func_name_var_map
.entry(instance)
.or_insert_with(|| create_pgo_func_name_var(self, instance))
} else {
bug!("Could not get the `coverage_context`");
}
}
}
impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
fn init_coverage(&mut self, instance: Instance<'tcx>) {
let Some(function_coverage_info) =
self.tcx.instance_mir(instance.def).function_coverage_info.as_deref()
else {
return;
};
if function_coverage_info.mcdc_bitmap_bytes == 0 {
return;
}
let fn_name = self.get_pgo_func_name_var(instance);
let hash = self.const_u64(function_coverage_info.function_source_hash);
let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes);
self.mcdc_parameters(fn_name, hash, bitmap_bytes);
let mut cond_bitmaps = vec![];
for i in 0..function_coverage_info.mcdc_num_condition_bitmaps {
let align = self.tcx.data_layout.i32_align.abi;
let cond_bitmap = self.alloca(Size::from_bytes(4), align);
llvm::set_value_name(cond_bitmap, format!("mcdc.addr.{i}").as_bytes());
self.store(self.const_i32(0), cond_bitmap, align);
cond_bitmaps.push(cond_bitmap);
}
self.coverage_context()
.expect("always present when coverage is enabled")
.mcdc_condition_bitmap_map
.borrow_mut()
.insert(instance, cond_bitmaps);
}
#[instrument(level = "debug", skip(self))]
fn add_coverage(&mut self, instance: Instance<'tcx>, kind: &CoverageKind) {
let bx = self;
let Some(function_coverage_info) =
bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref()
else {
debug!("function has a coverage statement but no coverage info");
return;
};
let Some(coverage_context) = bx.coverage_context() else { return };
let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
let func_coverage = coverage_map
.entry(instance)
.or_insert_with(|| FunctionCoverageCollector::new(instance, function_coverage_info));
match *kind {
CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!(
"marker statement {kind:?} should have been removed by CleanupPostBorrowck"
),
CoverageKind::CounterIncrement { id } => {
func_coverage.mark_counter_id_seen(id);
drop(coverage_map);
let num_counters =
bx.tcx().coverage_ids_info(instance.def).max_counter_id.as_u32() + 1;
assert!(
num_counters as usize <= function_coverage_info.num_counters,
"num_counters disagreement: query says {num_counters} but function info only has {}",
function_coverage_info.num_counters
);
let fn_name = bx.get_pgo_func_name_var(instance);
let hash = bx.const_u64(function_coverage_info.function_source_hash);
let num_counters = bx.const_u32(num_counters);
let index = bx.const_u32(id.as_u32());
debug!(
"codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
fn_name, hash, num_counters, index,
);
bx.instrprof_increment(fn_name, hash, num_counters, index);
}
CoverageKind::ExpressionUsed { id } => {
func_coverage.mark_expression_id_seen(id);
}
CoverageKind::CondBitmapUpdate { id, value, decision_depth } => {
drop(coverage_map);
assert_ne!(
id.as_u32(),
0,
"ConditionId of evaluated conditions should never be zero"
);
let cond_bitmap = coverage_context
.try_get_mcdc_condition_bitmap(&instance, decision_depth)
.expect("mcdc cond bitmap should have been allocated for updating");
let cond_loc = bx.const_i32(id.as_u32() as i32 - 1);
let bool_value = bx.const_bool(value);
let fn_name = bx.get_pgo_func_name_var(instance);
let hash = bx.const_u64(function_coverage_info.function_source_hash);
bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value);
}
CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
drop(coverage_map);
let cond_bitmap = coverage_context
.try_get_mcdc_condition_bitmap(&instance, decision_depth)
.expect("mcdc cond bitmap should have been allocated for merging into the global bitmap");
let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes;
assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range");
let fn_name = bx.get_pgo_func_name_var(instance);
let hash = bx.const_u64(function_coverage_info.function_source_hash);
let bitmap_bytes = bx.const_u32(bitmap_bytes);
let bitmap_index = bx.const_u32(bitmap_idx);
bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap);
}
}
}
}
fn create_pgo_func_name_var<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
instance: Instance<'tcx>,
) -> &'ll llvm::Value {
let mangled_fn_name: &str = cx.tcx.symbol_name(instance).name;
let llfn = cx.get_fn(instance);
unsafe {
llvm::LLVMRustCoverageCreatePGOFuncNameVar(
llfn,
mangled_fn_name.as_ptr().cast(),
mangled_fn_name.len(),
)
}
}
pub(crate) fn write_filenames_section_to_buffer<'a>(
filenames: impl IntoIterator<Item = &'a str>,
buffer: &RustString,
) {
let (pointers, lengths) = filenames
.into_iter()
.map(|s: &str| (s.as_ptr().cast(), s.len()))
.unzip::<_, _, Vec<_>, Vec<_>>();
unsafe {
llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(
pointers.as_ptr(),
pointers.len(),
lengths.as_ptr(),
lengths.len(),
buffer,
);
}
}
pub(crate) fn write_mapping_to_buffer(
virtual_file_mapping: Vec<u32>,
expressions: Vec<CounterExpression>,
mapping_regions: Vec<CounterMappingRegion>,
buffer: &RustString,
) {
unsafe {
llvm::LLVMRustCoverageWriteMappingToBuffer(
virtual_file_mapping.as_ptr(),
virtual_file_mapping.len() as c_uint,
expressions.as_ptr(),
expressions.len() as c_uint,
mapping_regions.as_ptr(),
mapping_regions.len() as c_uint,
buffer,
);
}
}
pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 {
unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) }
}
pub(crate) fn mapping_version() -> u32 {
unsafe { llvm::LLVMRustCoverageMappingVersion() }
}
pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
cov_data_val: &'ll llvm::Value,
) {
let covmap_var_name = llvm::build_string(|s| unsafe {
llvm::LLVMRustCoverageWriteMappingVarNameToString(s);
})
.expect("Rust Coverage Mapping var name failed UTF-8 conversion");
debug!("covmap var name: {:?}", covmap_var_name);
let covmap_section_name = llvm::build_string(|s| unsafe {
llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s);
})
.expect("Rust Coverage section name failed UTF-8 conversion");
debug!("covmap section name: {:?}", covmap_section_name);
let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name);
llvm::set_initializer(llglobal, cov_data_val);
llvm::set_global_constant(llglobal, true);
llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
llvm::set_section(llglobal, &covmap_section_name);
llvm::set_alignment(llglobal, Align::EIGHT);
cx.add_used_global(llglobal);
}
pub(crate) fn save_func_record_to_mod<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
covfun_section_name: &str,
func_name_hash: u64,
func_record_val: &'ll llvm::Value,
is_used: bool,
) {
let func_record_var_name =
format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" });
debug!("function record var name: {:?}", func_record_var_name);
debug!("function record section name: {:?}", covfun_section_name);
let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name);
llvm::set_initializer(llglobal, func_record_val);
llvm::set_global_constant(llglobal, true);
llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage);
llvm::set_visibility(llglobal, llvm::Visibility::Hidden);
llvm::set_section(llglobal, covfun_section_name);
llvm::set_alignment(llglobal, Align::EIGHT);
llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name);
cx.add_used_global(llglobal);
}
pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String {
llvm::build_string(|s| unsafe {
llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s);
})
.expect("Rust Coverage function record section name failed UTF-8 conversion")
}