October 29, 2025
Better granularity with Inlining Information
Ever compared two benchmark runs and wondered why performance changed when you didn't touch the code? Often, the culprit is function inlining: a compiler optimization that can make frames appear to vanish from your flamegraphs.
Now, CodSpeed automatically detects and displays inlined frames in your flamegraphs, giving you complete visibility into how compiler optimizations affect your performance profile.
What you get
When viewing differential flamegraphs, inlined frames are explicitly marked, so you can instantly tell whether performance differences are due to:
- Compiler optimization changes between builds
- Different optimization flags (
-O2vs-O3) - Actual code changes in your application
This makes it easier to understand mysterious performance shifts, especially when comparing:
- Builds with different optimization levels
- Before and after dependency updates
- Code changes that may affect inlining decisions
How it works
The inlining information is automatically detected and displayed when using the CPU simulation instrument, no configuration needed. Inlined frames appear as marked nodes in your flamegraphs, making the call graph even more complete and accurate.
See it in action
Check out this real-world example from the
salsa project showing inlined frames in
differential flamegraphs:
Mode:
Origin
Function
Bottleneck
accumulator
accumulator::accumulator::{{closure}}::{{closure}} (100.00%)
salsa::function::accumulated::<impl salsa::function::IngredientImpl<C>>::accumulated_by (99.98%)
pop<salsa::key::DatabaseKeyIndex, alloc::alloc::Global> (1.22%) (new)
as_ptr<salsa::key::DatabaseKeyIndex, alloc::alloc::Global> (0.35%) (new)
add<salsa::key::DatabaseKeyIndex> (0.18%) (new)
read<salsa::key::DatabaseKeyIndex> (1.21%) (new)
hashbrown::map::HashMap<K,V,S,A>::insert (25.06%)
find_or_find_insert_slot<salsa::key::DatabaseKeyIndex, (), core::hash::BuildHasherDefault<rustc_hash::FxHasher>, alloc::alloc::Global, salsa::key::DatabaseKeyIndex> (1.05%) (new)
_mm_loadu_si128 (1.17%) (new)
match_tag (0.53%) (new)
next (0.35%) (new)
_mm_movemask_epi8 (0.17%) (new)
lowest_set_bit (0.17%) (new)
find_or_find_insert_slot_inner (2.44%) (new)
match_empty (0.33%) (new)
likely (0.51%) (new)
is_bucket_full (0.34%) (new)
is_full (0.17%) (new)
unlikely (0.17%) (new)
from (0.17%) (new)
wrapping_sub (0.17%) (new)
set_ctrl (0.84%) (new)
record_item_insert_at (0.84%) (new)
from_base_index<(salsa::key::DatabaseKeyIndex, ())> (0.35%) (new)
write<(salsa::key::DatabaseKeyIndex, ())> (7.59%) (new)
Unknown inlined function (5.04%) (new)
insert<salsa::key::DatabaseKeyIndex, core::hash::BuildHasherDefault<rustc_hash::FxHasher>, alloc::alloc::Global> (1.76%) (new)
as_slice<alloc::boxed::Box<dyn salsa::ingredient::Ingredient, alloc::alloc::Global>, alloc::alloc::Global> (0.33%) (new)
index<alloc::boxed::Box<dyn salsa::ingredient::Ingredient, alloc::alloc::Global>> (0.33%) (new)
as_ptr<alloc::boxed::Box<dyn salsa::ingredient::Ingredient, alloc::alloc::Global>, alloc::alloc::Global> (0.33%) (new)
index<alloc::boxed::Box<dyn salsa::ingredient::Ingredient, alloc::alloc::Global>, usize> (0.17%) (new)
as_ref<dyn salsa::ingredient::Ingredient, alloc::alloc::Global> (0.67%) (new)
salsa::ingredient::Ingredient::accumulated (0.34%)
salsa::function::IngredientImpl<C>::accumulated (45.17%)
view_caster<accumulator::infer_expression::infer_expression_Configuration_> (0.25%) (new)
get_zalsa_id (2.01%) (new)
memo_table_for<accumulator::Expression> (0.17%) (new)
eq (0.50%) (new)
get<salsa::function::memo::Memo<accumulator::infer_expression::infer_expression_Configuration_>> (2.46%) (new)
branch<&salsa::function::memo::Memo<accumulator::infer_expression::infer_expression_Configuration_>> (0.17%) (new)
database_key_index<accumulator::infer_expression::infer_expression_Configuration_> (0.33%) (new)
load (0.08%) (new)
fetch_hot<accumulator::infer_expression::infer_expression_Configuration_> (1.85%) (new)
update_shallow<accumulator::infer_expression::infer_expression_Configuration_> (0.17%) (new)
accumulated_map<accumulator::infer_expression::infer_expression_Configuration_> (1.84%) (new)
expect<&salsa::views::DatabaseDownCaster<dyn salsa::database::Database>> (0.08%) (new)
core::ops::function::FnOnce::call_once (0.34%)
{closure#0}<salsa::database_impl::DatabaseImpl> (0.25%) (new)
downcast_unchecked<dyn salsa::database::Database> (0.50%) (new)
salsa::zalsa::ZalsaDatabase::zalsas (0.50%)
inner<salsa::zalsa::Zalsa, alloc::alloc::Global> (0.00%) (new)
deref<salsa::zalsa::Zalsa, alloc::alloc::Global> (0.25%) (new)
zalsa_local<salsa::database_impl::DatabaseImpl> (0.25%) (new)
Unknown inlined function (8.28%) (new)
get<salsa::table::memo::MemoEntry, usize> (9.04%) (new)
new (0.67%) (new)
salsa::function::maybe_changed_after::<impl salsa::function::IngredientImpl<C>>::shallow_verify_memo (11.04%)
atomic_load<usize> (0.34%) (new)
le (0.08%) (new)
salsa::revision::AtomicRevision::load (6.93%)
atomic_load<usize> (6.93%) (new)
eq (0.17%) (new)
yes (0.17%) (new)
atomic_load<u8> (0.17%) (new)
salsa::zalsa_local::QueryRevisions::accumulated (1.27%)
as_ref<alloc::boxed::Box<salsa::zalsa_local::QueryRevisionsExtraInner, alloc::alloc::Global>> (0.25%) (new)
is_empty<salsa::zalsa::IngredientIndex, alloc::boxed::Box<dyn salsa::accumulator::accumulated::AnyAccumulated, alloc::alloc::Global>, rustc_hash::FxBuildHasher, allocator_api2::stable::alloc::global::Global> (0.69%) (new)
filter<&salsa::accumulator::accumulated_map::AccumulatedMap, salsa::zalsa_local::{impl#6}::accumulated::{closure_env#1}> (0.09%) (new)
salsa::accumulator::accumulated_map::AtomicInputAccumulatedValues::load (0.50%)
load (0.17%) (new)
salsa::accumulator::accumulated_map::AccumulatedMap::extend_with_accumulated (3.77%)
get_inner<salsa::zalsa::IngredientIndex, alloc::boxed::Box<dyn salsa::accumulator::accumulated::AnyAccumulated, alloc::alloc::Global>, rustc_hash::FxBuildHasher, allocator_api2::stable::alloc::global::Global, salsa::zalsa::IngredientIndex> (0.09%) (new)
write_u32 (0.02%) (new)
add_to_hash (0.01%) (new)
finish (0.01%) (new)
full (0.25%) (new)
probe_seq (0.08%) (new)
_mm_loadu_si128 (0.89%) (new)
match_tag (0.03%) (new)
lowest_set_bit (0.02%) (new)
nonzero_trailing_zeros (0.01%) (new)
find_inner (0.02%) (new)
bucket<(salsa::zalsa::IngredientIndex, alloc::boxed::Box<dyn salsa::accumulator::accumulated::AnyAccumulated, alloc::alloc::Global>), allocator_api2::stable::alloc::global::Global> (0.02%) (new)
likely (0.01%) (new)
salsa::accumulator::accumulated::Accumulated<A>::as_dyn_any (0.03%)
T::type_id (0.06%)
of<salsa::accumulator::IngredientImpl<accumulator::Diagnostic>> (0.00%) (new)
of<salsa::accumulator::accumulated::Accumulated<accumulator::Diagnostic>> (0.05%) (new)
is<salsa::accumulator::accumulated::Accumulated<accumulator::Diagnostic>> (0.03%) (new)
eq (0.50%) (new)
downcast_ref<salsa::accumulator::accumulated::Accumulated<accumulator::Diagnostic>> (0.01%) (new)
ptr<alloc::alloc::Global, accumulator::Diagnostic> (0.89%) (new)
as_slice<accumulator::Diagnostic, alloc::alloc::Global> (0.24%) (new)
extend_with_accumulated<accumulator::Diagnostic> (0.01%) (new)
alloc::raw_vec::RawVecInner<A>::reserve::do_reserve_and_handle (0.05%)
min_non_zero_cap (0.00%) (new)
max<usize> (0.00%) (new)
pad_to_align (0.00%) (new)
checked_mul (0.00%) (new)
current_memory<alloc::alloc::Global> (0.00%) (new)
alloc::raw_vec::finish_grow (0.04%)
__rustc::__rust_realloc (0.04%)
__rustc::__rdl_realloc (0.04%)
realloc (0.04%) (new)
musable (0.00%) (new)
checked_request2size (0.00%) (new)
_int_realloc (0.03%) (new)
_int_malloc (0.02%) (new)
get_max_fast (0.00%) (new)
alloc_perturb (0.00%) (new)
__glibc_morecore (0.00%) (new)
sbrk (0.00%) (new)
brk (0.00%) (new)
__brk_call (0.00%) (new)
sysmalloc (0.01%) (new)
__memcpy_avx_unaligned_erms (0.00%) (new)
memcpy (0.00%) (new)
_int_free (0.00%) (new)
get_max_fast (0.00%) (new)
_int_free_merge_chunk (0.00%) (new)
free_perturb (0.00%) (new)
unlink_chunk.isra.0 (0.00%) (new)
_int_free_maybe_consolidate (0.00%) (new)
_int_free_create_chunk (0.00%) (new)
tcache_put (0.00%) (new)
arena_for_chunk (0.00%) (new)
realloc (0.00%) (new)
grow (0.00%) (new)
__rustc::__rust_alloc (0.00%)
__rustc::__rdl_alloc (0.00%)
malloc (0.00%) (new)
checked_request2size (0.00%) (new)
tcache_get (0.00%) (new)
alloc (0.00%) (new)
map_err<core::ptr::non_null::NonNull<[u8]>, core::alloc::AllocError, alloc::collections::TryReserveError, alloc::raw_vec::finish_grow::{closure_env#0}<alloc::alloc::Global>> (0.00%) (new)
grow_amortized<alloc::alloc::Global> (0.00%) (new)
branch<core::ptr::non_null::NonNull<[u8]>, alloc::collections::TryReserveError> (0.00%) (new)
set_ptr_and_cap<alloc::alloc::Global> (0.00%) (new)
Unknown inlined function (0.35%) (new)
salsa::function::IngredientImpl<C>::origin (0.38%)
get_memo_from_table_for<accumulator::root::root_Configuration_> (0.00%) (new)
{closure#0}<accumulator::root::root_Configuration_> (0.00%) (new)
origin<accumulator::root::root_Configuration_> (0.00%) (new)
memo_table (0.01%) (new)
get<salsa::table::memo::MemoEntry> (0.01%) (new)
Unknown inlined function (0.14%) (new)
get<salsa::function::memo::Memo<accumulator::infer_expression::infer_expression_Configuration_>> (0.04%) (new)
map<&salsa::function::memo::Memo<accumulator::infer_expression::infer_expression_Configuration_>, salsa::zalsa_local::QueryOriginRef, salsa::function::inputs::{impl#0}::origin::{closure_env#0}<accumulator::infer_expression::infer_expression_Configuration_>> (0.01%) (new)
{closure#0}<accumulator::infer_expression::infer_expression_Configuration_> (0.05%) (new)
origin<accumulator::infer_expression::infer_expression_Configuration_> (0.09%) (new)
alloc::raw_vec::RawVecInner<A>::reserve::do_reserve_and_handle (0.09%)
min_non_zero_cap (0.00%) (new)
max<usize> (0.00%) (new)
pad_to_align (0.00%) (new)
checked_mul (0.00%) (new)
current_memory<alloc::alloc::Global> (0.00%) (new)
alloc::raw_vec::finish_grow (0.08%)
__rustc::__rust_realloc (0.08%)
__rustc::__rdl_realloc (0.08%)
realloc (0.07%) (new)
musable (0.00%) (new)
checked_request2size (0.00%) (new)
_int_realloc (0.06%) (new)
_int_malloc (0.04%) (new)
get_max_fast (0.00%) (new)
alloc_perturb (0.00%) (new)
__glibc_morecore (0.01%) (new)
sbrk (0.01%) (new)
brk (0.00%) (new)
__brk_call (0.00%) (new)
sysmalloc (0.01%) (new)
__memcpy_avx_unaligned_erms (0.00%) (new)
memcpy (0.00%) (new)
_int_free (0.00%) (new)
get_max_fast (0.00%) (new)
_int_free_merge_chunk (0.00%) (new)
free_perturb (0.00%) (new)
unlink_chunk.isra.0 (0.00%) (new)
_int_free_maybe_consolidate (0.00%) (new)
_int_free_create_chunk (0.00%) (new)
tcache_put (0.00%) (new)
arena_for_chunk (0.00%) (new)
realloc (0.00%) (new)
grow (0.00%) (new)
__rustc::__rust_alloc (0.00%)
__rustc::__rdl_alloc (0.00%)
malloc (0.00%) (new)
checked_request2size (0.00%) (new)
tcache_get (0.00%) (new)
alloc (0.00%) (new)
map_err<core::ptr::non_null::NonNull<[u8]>, core::alloc::AllocError, alloc::collections::TryReserveError, alloc::raw_vec::finish_grow::{closure_env#0}<alloc::alloc::Global>> (0.00%) (new)
grow_amortized<alloc::alloc::Global> (0.00%) (new)
branch<core::ptr::non_null::NonNull<[u8]>, alloc::collections::TryReserveError> (0.00%) (new)
set_ptr_and_cap<alloc::alloc::Global> (0.00%) (new)
alloc::vec::Vec<T,A>::spec_extend (14.35%)
eq<salsa::zalsa_local::QueryEdge> (0.18%) (new)
next_back<salsa::zalsa_local::QueryEdge> (0.18%) (new)
kind (1.61%) (new)
{closure#0}<&salsa::zalsa_local::QueryEdge, salsa::key::DatabaseKeyIndex, (), core::ops::control_flow::ControlFlow<salsa::key::DatabaseKeyIndex, ()>, salsa::zalsa_local::input_edges::{closure_env#0}, &mut core::iter::adapters::filter_map::{impl#3}::next_back::find::{closure_env#0}<salsa::key::DatabaseKeyIndex, salsa::key::DatabaseKeyIndex, salsa::function::accumulated::{impl#0}::accumulated_by::{closure_env#0}<accumulator::root::root_Configuration_, accumulator::Diagnostic>>> (0.34%) (new)
try_rfold<core::iter::adapters::map::Map<core::option::IntoIter<&[salsa::zalsa_local::QueryEdge]>, fn(&[salsa::zalsa_local::QueryEdge]) -> core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>>, (), core::iter::adapters::flatten::{impl#19}::iter_try_rfold::flatten::{closure_env#0}<core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>, (), core::ops::control_flow::ControlFlow<salsa::key::DatabaseKeyIndex, ()>, core::iter::adapters::flatten::{impl#21}::try_rfold::flatten::{closure_env#0}<core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>, (), core::ops::control_flow::ControlFlow<salsa::key::DatabaseKeyIndex, ()>, core::iter::adapters::filter_map::{impl#3}::next_back::find::{closure_env#0}<salsa::key::DatabaseKeyIndex, salsa::key::DatabaseKeyIndex, salsa::function::accumulated::{impl#0}::accumulated_by::{closure_env#0}<accumulator::root::root_Configuration_, accumulator::Diagnostic>>>>, core::ops::control_flow::ControlFlow<salsa::key::DatabaseKeyIndex, ()>> (0.02%) (new)
next_back_unchecked<salsa::zalsa_local::QueryEdge> (0.86%) (new)
Unknown inlined function (0.36%) (new)
len<salsa::key::DatabaseKeyIndex, alloc::alloc::Global> (0.35%) (new)
as_mut_ptr<salsa::key::DatabaseKeyIndex, alloc::alloc::Global> (0.35%) (new)
add<salsa::key::DatabaseKeyIndex> (0.53%) (new)
write<salsa::key::DatabaseKeyIndex> (4.00%) (new)
extend_desugared<salsa::key::DatabaseKeyIndex, alloc::alloc::Global, core::iter::adapters::rev::Rev<core::iter::adapters::filter_map::FilterMap<core::iter::adapters::flatten::FlatMap<core::option::IntoIter<&[salsa::zalsa_local::QueryEdge]>, core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>, fn(&[salsa::zalsa_local::QueryEdge]) -> core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>>, salsa::function::accumulated::{impl#0}::accumulated_by::{closure_env#0}<accumulator::root::root_Configuration_, accumulator::Diagnostic>>>> (1.08%) (new)
set_len<salsa::key::DatabaseKeyIndex, alloc::alloc::Global> (0.53%) (new)
iter_try_rfold<core::iter::adapters::map::Map<core::option::IntoIter<&[salsa::zalsa_local::QueryEdge]>, fn(&[salsa::zalsa_local::QueryEdge]) -> core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>>, core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>, (), core::iter::adapters::flatten::{impl#21}::try_rfold::flatten::{closure_env#0}<core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>, (), core::ops::control_flow::ControlFlow<salsa::key::DatabaseKeyIndex, ()>, core::iter::adapters::filter_map::{impl#3}::next_back::find::{closure_env#0}<salsa::key::DatabaseKeyIndex, salsa::key::DatabaseKeyIndex, salsa::function::accumulated::{impl#0}::accumulated_by::{closure_env#0}<accumulator::root::root_Configuration_, accumulator::Diagnostic>>>, core::ops::control_flow::ControlFlow<salsa::key::DatabaseKeyIndex, ()>> (0.35%) (new)
next<core::iter::adapters::filter_map::FilterMap<core::iter::adapters::flatten::FlatMap<core::option::IntoIter<&[salsa::zalsa_local::QueryEdge]>, core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>, fn(&[salsa::zalsa_local::QueryEdge]) -> core::iter::adapters::filter_map::FilterMap<core::slice::iter::Iter<salsa::zalsa_local::QueryEdge>, salsa::zalsa_local::input_edges::{closure_env#0}>>, salsa::function::accumulated::{impl#0}::accumulated_by::{closure_env#0}<accumulator::root::root_Configuration_, accumulator::Diagnostic>>> (3.51%) (new)
hashbrown::raw::RawTable<T,A>::reserve_rehash (1.02%)
reserve_rehash_inner<alloc::alloc::Global> (0.00%) (new)
__rustc::__rust_alloc (0.01%)
__rustc::__rdl_alloc (0.01%)
malloc (0.01%) (new)
checked_request2size (0.00%) (new)
tcache_get (0.00%) (new)
_int_malloc (0.00%) (new)
get_max_fast (0.00%) (new)
alloc_perturb (0.00%) (new)
__glibc_morecore (0.00%) (new)
sbrk (0.00%) (new)
brk (0.00%) (new)
__brk_call (0.00%) (new)
sysmalloc (0.00%) (new)
alloc (0.00%) (new)
fallible_with_capacity<alloc::alloc::Global> (0.00%) (new)
__memset_avx2_unaligned_erms (0.99%) (new)
bucket_mask_to_capacity (0.00%) (new)
copy_nonoverlapping<u8> (0.00%) (new)
__rustc::__rust_dealloc (0.00%)
__rustc::__rdl_dealloc (0.00%)
free (0.00%) (new)
arena_for_chunk (0.00%) (new)
_int_free (0.00%) (new)
get_max_fast (0.00%) (new)
_int_free_merge_chunk (0.00%) (new)
free_perturb (0.00%) (new)
unlink_chunk.isra.0 (0.00%) (new)
_int_free_maybe_consolidate (0.00%) (new)
_int_free_create_chunk (0.00%) (new)
tcache_put (0.00%) (new)
dealloc (0.00%) (new)
User
Library
System
Unknown
Check out the complete run
When you hover over a frame in the flamegraph, you'll see whether the frame was inlined, giving you complete visibility into both your code and compiler optimizations.
Get started
This feature is available with
CodSpeedHQ/action v4.3.1
and later. If you're using the @v4 shorthand, you're already good to go!
Otherwise, update your workflow to use @v4.3.1 or later.