diff options
| -rw-r--r-- | components/compositing/compositor.rs | 12 | ||||
| -rw-r--r-- | components/layout/layout_task.rs | 13 | ||||
| -rw-r--r-- | components/profile/mem.rs | 79 | ||||
| -rw-r--r-- | components/profile_traits/mem.rs | 36 | ||||
| -rw-r--r-- | components/script/script_task.rs | 46 |
5 files changed, 161 insertions, 25 deletions
diff --git a/components/compositing/compositor.rs b/components/compositing/compositor.rs index 468301317b8..e0caae3f98d 100644 --- a/components/compositing/compositor.rs +++ b/components/compositing/compositor.rs @@ -39,7 +39,7 @@ use msg::constellation_msg::{ConstellationChan, NavigationDirection}; use msg::constellation_msg::{Key, KeyModifiers, KeyState, LoadData}; use msg::constellation_msg::{PipelineId, WindowSizeData}; use png; -use profile_traits::mem::{self, Reporter, ReporterRequest}; +use profile_traits::mem::{self, Reporter, ReporterRequest, ReportKind}; use profile_traits::time::{self, ProfilerCategory, profile}; use script_traits::{ConstellationControlMsg, LayoutControlMsg, ScriptControlChan}; use std::collections::HashMap; @@ -501,11 +501,17 @@ impl<Window: WindowMethods> IOCompositor<Window> { (Msg::CollectMemoryReports(reports_chan), ShutdownState::NotShuttingDown) => { let mut reports = vec![]; let name = "compositor-task"; + // These are both `ExplicitUnknownLocationSize` because the memory might be in the + // GPU or on the heap. reports.push(mem::Report { - path: path![name, "surface-map"], size: self.surface_map.mem(), + path: path![name, "surface-map"], + kind: ReportKind::ExplicitUnknownLocationSize, + size: self.surface_map.mem(), }); reports.push(mem::Report { - path: path![name, "layer-tree"], size: self.scene.get_memory_usage(), + path: path![name, "layer-tree"], + kind: ReportKind::ExplicitUnknownLocationSize, + size: self.scene.get_memory_usage(), }); reports_chan.send(reports); } diff --git a/components/layout/layout_task.rs b/components/layout/layout_task.rs index 363e35cc72e..803b8fb1a0a 100644 --- a/components/layout/layout_task.rs +++ b/components/layout/layout_task.rs @@ -46,7 +46,7 @@ use log; use msg::compositor_msg::{Epoch, ScrollPolicy, LayerId}; use msg::constellation_msg::Msg as ConstellationMsg; use msg::constellation_msg::{ConstellationChan, Failure, PipelineExitType, PipelineId}; -use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportsChan}; +use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportKind, ReportsChan}; use profile_traits::time::{self, ProfilerMetadata, profile}; use profile_traits::time::{TimerMetadataFrameType, TimerMetadataReflowType}; use net_traits::{load_bytes_iter, PendingAsyncLoad}; @@ -609,13 +609,15 @@ impl LayoutTask { let rw_data = self.lock_rw_data(possibly_locked_rw_data); let stacking_context = rw_data.stacking_context.as_ref(); reports.push(Report { - path: path!["pages", format!("url({})", self.url), "layout-task", "display-list"], + path: path![format!("url({})", self.url), "layout-task", "display-list"], + kind: ReportKind::ExplicitJemallocHeapSize, size: stacking_context.map_or(0, |sc| sc.heap_size_of_children()), }); // The LayoutTask has a context in TLS... reports.push(Report { - path: path!["pages", format!("url({})", self.url), "layout-task", "local-context"], + path: path![format!("url({})", self.url), "layout-task", "local-context"], + kind: ReportKind::ExplicitJemallocHeapSize, size: heap_size_of_local_context(), }); @@ -624,9 +626,10 @@ impl LayoutTask { let sizes = traversal.heap_size_of_tls(heap_size_of_local_context); for (i, size) in sizes.iter().enumerate() { reports.push(Report { - path: path!["pages", format!("url({})", self.url), + path: path![format!("url({})", self.url), format!("layout-worker-{}-local-context", i)], - size: *size + kind: ReportKind::ExplicitJemallocHeapSize, + size: *size, }); } } diff --git a/components/profile/mem.rs b/components/profile/mem.rs index 9c4a5ac710f..ba0d9e168e6 100644 --- a/components/profile/mem.rs +++ b/components/profile/mem.rs @@ -6,7 +6,8 @@ use ipc_channel::ipc::{self, IpcReceiver}; use ipc_channel::router::ROUTER; -use profile_traits::mem::{ProfilerChan, ProfilerMsg, Reporter, ReporterRequest, ReportsChan}; +use profile_traits::mem::{ProfilerChan, ProfilerMsg, Reporter, ReporterRequest, ReportKind}; +use profile_traits::mem::ReportsChan; use std::borrow::ToOwned; use std::cmp::Ordering; use std::collections::HashMap; @@ -21,6 +22,9 @@ pub struct Profiler { reporters: HashMap<String, Reporter>, } +const JEMALLOC_HEAP_ALLOCATED_STR: &'static str = "jemalloc-heap-allocated"; +const SYSTEM_HEAP_ALLOCATED_STR: &'static str = "system-heap-allocated"; + impl Profiler { pub fn create(period: Option<f64>) -> ProfilerChan { let (chan, port) = ipc::channel().unwrap(); @@ -122,16 +126,72 @@ impl Profiler { // each reporter once we have enough of them. // // If anything goes wrong with a reporter, we just skip it. + // + // We also track the total memory reported on the jemalloc heap and the system heap, and + // use that to compute the special "jemalloc-heap-unclassified" and + // "system-heap-unclassified" values. + let mut forest = ReportsForest::new(); + + let mut jemalloc_heap_reported_size = 0; + let mut system_heap_reported_size = 0; + + let mut jemalloc_heap_allocated_size: Option<usize> = None; + let mut system_heap_allocated_size: Option<usize> = None; + for reporter in self.reporters.values() { let (chan, port) = ipc::channel().unwrap(); reporter.collect_reports(ReportsChan(chan)); - if let Ok(reports) = port.recv() { - for report in reports.iter() { + if let Ok(mut reports) = port.recv() { + + for report in reports.iter_mut() { + + // Add "explicit" to the start of the path, when appropriate. + match report.kind { + ReportKind::ExplicitJemallocHeapSize | + ReportKind::ExplicitSystemHeapSize | + ReportKind::ExplicitNonHeapSize | + ReportKind::ExplicitUnknownLocationSize => + report.path.insert(0, String::from("explicit")), + ReportKind::NonExplicitSize => {}, + } + + // Update the reported fractions of the heaps, when appropriate. + match report.kind { + ReportKind::ExplicitJemallocHeapSize => + jemalloc_heap_reported_size += report.size, + ReportKind::ExplicitSystemHeapSize => + system_heap_reported_size += report.size, + _ => {}, + } + + // Record total size of the heaps, when we see them. + if report.path.len() == 1 { + if report.path[0] == JEMALLOC_HEAP_ALLOCATED_STR { + assert!(jemalloc_heap_allocated_size.is_none()); + jemalloc_heap_allocated_size = Some(report.size); + } else if report.path[0] == SYSTEM_HEAP_ALLOCATED_STR { + assert!(system_heap_allocated_size.is_none()); + system_heap_allocated_size = Some(report.size); + } + } + + // Insert the report. forest.insert(&report.path, report.size); } } } + + // Compute and insert the heap-unclassified values. + if let Some(jemalloc_heap_allocated_size) = jemalloc_heap_allocated_size { + forest.insert(&path!["explicit", "jemalloc-heap-unclassified"], + jemalloc_heap_allocated_size - jemalloc_heap_reported_size); + } + if let Some(system_heap_allocated_size) = system_heap_allocated_size { + forest.insert(&path!["explicit", "system-heap-unclassified"], + system_heap_allocated_size - system_heap_reported_size); + } + forest.print(); println!("|"); @@ -301,11 +361,12 @@ impl ReportsForest { mod system_reporter { use libc::{c_char, c_int, c_void, size_t}; - use profile_traits::mem::{Report, ReporterRequest}; + use profile_traits::mem::{Report, ReporterRequest, ReportKind}; use std::borrow::ToOwned; use std::ffi::CString; use std::mem::size_of; use std::ptr::null_mut; + use super::{JEMALLOC_HEAP_ALLOCATED_STR, SYSTEM_HEAP_ALLOCATED_STR}; #[cfg(target_os="macos")] use task_info::task_basic_info::{virtual_size, resident_size}; @@ -315,7 +376,11 @@ mod system_reporter { { let mut report = |path, size| { if let Some(size) = size { - reports.push(Report { path: path, size: size }); + reports.push(Report { + path: path, + kind: ReportKind::NonExplicitSize, + size: size, + }); } }; @@ -330,13 +395,13 @@ mod system_reporter { // Total number of bytes allocated by the application on the system // heap. - report(path!["system-heap-allocated"], get_system_heap_allocated()); + report(path![SYSTEM_HEAP_ALLOCATED_STR], get_system_heap_allocated()); // The descriptions of the following jemalloc measurements are taken // directly from the jemalloc documentation. // "Total number of bytes allocated by the application." - report(path!["jemalloc-heap-allocated"], get_jemalloc_stat("stats.allocated")); + report(path![JEMALLOC_HEAP_ALLOCATED_STR], get_jemalloc_stat("stats.allocated")); // "Total number of bytes in active pages allocated by the application. // This is a multiple of the page size, and greater than or equal to diff --git a/components/profile_traits/mem.rs b/components/profile_traits/mem.rs index 0269e8f29d7..19470576aad 100644 --- a/components/profile_traits/mem.rs +++ b/components/profile_traits/mem.rs @@ -23,12 +23,48 @@ impl ProfilerChan { } } +/// The various kinds of memory measurement. +/// +/// Here "explicit" means explicit memory allocations done by the application. It includes +/// allocations made at the OS level (via functions such as VirtualAlloc, vm_allocate, and mmap), +/// allocations made at the heap allocation level (via functions such as malloc, calloc, realloc, +/// memalign, operator new, and operator new[]) and where possible, the overhead of the heap +/// allocator itself. It excludes memory that is mapped implicitly such as code and data segments, +/// and thread stacks. "explicit" is not guaranteed to cover every explicit allocation, but it does +/// cover most (including the entire heap), and therefore it is the single best number to focus on +/// when trying to reduce memory usage. +#[derive(Deserialize, Serialize)] +pub enum ReportKind { + /// A size measurement for an explicit allocation on the jemalloc heap. This should be used + /// for any measurements done via the `HeapSizeOf` trait. + ExplicitJemallocHeapSize, + + /// A size measurement for an explicit allocation on the system heap. Only likely to be used + /// for external C or C++ libraries that don't use jemalloc. + ExplicitSystemHeapSize, + + /// A size measurement for an explicit allocation not on the heap, e.g. via mmap(). + ExplicitNonHeapSize, + + /// A size measurement for an explicit allocation whose location is unknown or uncertain. + ExplicitUnknownLocationSize, + + /// A size measurement for a non-explicit allocation. This kind is used for global + /// measurements such as "resident" and "vsize", and also for measurements that cross-cut the + /// measurements grouped under "explicit", e.g. by grouping those measurements in a way that's + /// different to how they are grouped under "explicit". + NonExplicitSize, +} + /// A single memory-related measurement. #[derive(Deserialize, Serialize)] pub struct Report { /// The identifying path for this report. pub path: Vec<String>, + /// The report kind. + pub kind: ReportKind, + /// The size, in bytes. pub size: usize, } diff --git a/components/script/script_task.rs b/components/script/script_task.rs index d9560e40911..e34e93008d2 100644 --- a/components/script/script_task.rs +++ b/components/script/script_task.rs @@ -70,7 +70,7 @@ use net_traits::{ResourceTask, LoadConsumer, ControlMsg, Metadata}; use net_traits::LoadData as NetLoadData; use net_traits::image_cache_task::{ImageCacheChan, ImageCacheTask, ImageCacheResult}; use net_traits::storage_task::StorageTask; -use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportsChan}; +use profile_traits::mem::{self, Report, Reporter, ReporterRequest, ReportKind, ReportsChan}; use string_cache::Atom; use util::str::DOMString; use util::task::spawn_named_with_send_on_failure; @@ -1042,18 +1042,44 @@ impl ScriptTask { let rt = JS_GetRuntime(cx); let mut stats = ::std::mem::zeroed(); if CollectServoSizes(rt, &mut stats) { - let mut report = |mut path_suffix, size| { - let mut path = path!["pages", path_seg, "js"]; + let mut report = |mut path_suffix, kind, size| { + let mut path = path![path_seg, "js"]; path.append(&mut path_suffix); - reports.push(Report { path: path, size: size as usize }) + reports.push(Report { + path: path, + kind: kind, + size: size as usize, + }) }; - report(path!["gc-heap", "used"], stats.gcHeapUsed); - report(path!["gc-heap", "unused"], stats.gcHeapUnused); - report(path!["gc-heap", "admin"], stats.gcHeapAdmin); - report(path!["gc-heap", "decommitted"], stats.gcHeapDecommitted); - report(path!["malloc-heap"], stats.mallocHeap); - report(path!["non-heap"], stats.nonHeap); + // A note about possibly confusing terminology: the JS GC "heap" is allocated via + // mmap/VirtualAlloc, which means it's not on the malloc "heap", so we use + // `ExplicitNonHeapSize` as its kind. + + report(path!["gc-heap", "used"], + ReportKind::ExplicitNonHeapSize, + stats.gcHeapUsed); + + report(path!["gc-heap", "unused"], + ReportKind::ExplicitNonHeapSize, + stats.gcHeapUnused); + + report(path!["gc-heap", "admin"], + ReportKind::ExplicitNonHeapSize, + stats.gcHeapAdmin); + + report(path!["gc-heap", "decommitted"], + ReportKind::ExplicitNonHeapSize, + stats.gcHeapDecommitted); + + // SpiderMonkey uses the system heap, not jemalloc. + report(path!["malloc-heap"], + ReportKind::ExplicitSystemHeapSize, + stats.mallocHeap); + + report(path!["non-heap"], + ReportKind::ExplicitNonHeapSize, + stats.nonHeap); } } reports |