1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

use crate::inline;
use crate::pass_manager as pm;
use rustc_attr::InlineAttr;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::LocalDefId;
use rustc_middle::mir::visit::Visitor;
use rustc_middle::mir::*;
use rustc_middle::query::Providers;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::InliningThreshold;
use rustc_session::config::OptLevel;
use rustc_span::sym;

pub fn provide(providers: &mut Providers) {
    providers.cross_crate_inlinable = cross_crate_inlinable;
}

fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
    let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
    // If this has an extern indicator, then this function is globally shared and thus will not
    // generate cgu-internal copies which would make it cross-crate inlinable.
    if codegen_fn_attrs.contains_extern_indicator() {
        return false;
    }

    if tcx.intrinsic(def_id).is_some_and(|i| i.must_be_overridden) {
        return false;
    }

    // This just reproduces the logic from Instance::requires_inline.
    match tcx.def_kind(def_id) {
        DefKind::Ctor(..) | DefKind::Closure => return true,
        DefKind::Fn | DefKind::AssocFn => {}
        _ => return false,
    }

    // From this point on, it is valid to return true or false.
    if tcx.sess.opts.unstable_opts.cross_crate_inline_threshold == InliningThreshold::Always {
        return true;
    }

    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
        // Intrinsic fallback bodies are always cross-crate inlineable.
        // To ensure that the MIR inliner doesn't cluelessly try to inline fallback
        // bodies even when the backend would implement something better, we stop
        // the MIR inliner from ever inlining an intrinsic.
        return true;
    }

    // Obey source annotations first; this is important because it means we can use
    // #[inline(never)] to force code generation.
    match codegen_fn_attrs.inline {
        InlineAttr::Never => return false,
        InlineAttr::Hint | InlineAttr::Always => return true,
        _ => {}
    }

    // Don't do any inference when incremental compilation is enabled; the additional inlining that
    // inference permits also creates more work for small edits.
    if tcx.sess.opts.incremental.is_some() {
        return false;
    }

    // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
    // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
    // which is less confusing than having to also enable -Copt-level=1.
    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline)
    {
        return false;
    }

    if !tcx.is_mir_available(def_id) {
        return false;
    }

    let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold {
        InliningThreshold::Always => return true,
        InliningThreshold::Sometimes(threshold) => threshold,
        InliningThreshold::Never => return false,
    };

    let mir = tcx.optimized_mir(def_id);
    let mut checker =
        CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };
    checker.visit_body(mir);
    checker.calls == 0
        && checker.resumes == 0
        && checker.landing_pads == 0
        && checker.statements <= threshold
}

struct CostChecker<'b, 'tcx> {
    tcx: TyCtxt<'tcx>,
    callee_body: &'b Body<'tcx>,
    calls: usize,
    statements: usize,
    landing_pads: usize,
    resumes: usize,
}

impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
    fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
        // Don't count StorageLive/StorageDead in the inlining cost.
        match statement.kind {
            StatementKind::StorageLive(_)
            | StatementKind::StorageDead(_)
            | StatementKind::Deinit(_)
            | StatementKind::Nop => {}
            _ => self.statements += 1,
        }
    }

    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
        let tcx = self.tcx;
        match terminator.kind {
            TerminatorKind::Drop { ref place, unwind, .. } => {
                let ty = place.ty(self.callee_body, tcx).ty;
                if !ty.is_trivially_pure_clone_copy() {
                    self.calls += 1;
                    if let UnwindAction::Cleanup(_) = unwind {
                        self.landing_pads += 1;
                    }
                }
            }
            TerminatorKind::Call { unwind, .. } => {
                self.calls += 1;
                if let UnwindAction::Cleanup(_) = unwind {
                    self.landing_pads += 1;
                }
            }
            TerminatorKind::Assert { unwind, .. } => {
                self.calls += 1;
                if let UnwindAction::Cleanup(_) = unwind {
                    self.landing_pads += 1;
                }
            }
            TerminatorKind::UnwindResume => self.resumes += 1,
            TerminatorKind::InlineAsm { unwind, .. } => {
                self.statements += 1;
                if let UnwindAction::Cleanup(_) = unwind {
                    self.landing_pads += 1;
                }
            }
            TerminatorKind::Return => {}
            _ => self.statements += 1,
        }
    }
}