From 0a24bdb45cefd21ccae2d476abd7ec99be956886 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 19 Aug 2026 17:19:55 +0200 Subject: [PATCH 1/3] fix: count DRAM in RAPL energy total Aggregate the dram domain into the reported processor energy alongside package/psys, with tests covering aggregation and the non-power-domain fallback. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/cpu.py | 5 ++- tests/test_rapl_parameters.py | 80 +++++++++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index e21c39fd8..75c0d8f5b 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -770,8 +770,9 @@ def _create_rapl_files(self, domain_map: dict, found_main_readable: bool): domain_name, ) in domain_map.values(): try: - if domain_name and ( - "package" in domain_name.lower() or "psys" in domain_name.lower() + domain_lower = (domain_name or "").lower() + if any( + keyword in domain_lower for keyword in ("package", "psys", "dram") ): display_name = f"Processor Energy Delta_{domain_index}(kWh)" domain_index += 1 diff --git a/tests/test_rapl_parameters.py b/tests/test_rapl_parameters.py index 4fafe7a65..ac600ccfc 100644 --- a/tests/test_rapl_parameters.py +++ b/tests/test_rapl_parameters.py @@ -3,11 +3,13 @@ """ import logging +import re import sys import pytest from codecarbon.core.cpu import IntelRAPL +from codecarbon.core.units import Time @pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") @@ -112,8 +114,50 @@ def test_rapl_include_dram_true_explicit(tmp_path): # Verify both package and dram are present names = [f.name for f in rapl._rapl_files] - assert any("Processor Energy" in name for name in names), "Missing package domain" - assert any("dram" in name.lower() for name in names), "Missing DRAM domain" + # Both domains must be named so that they are summed in the reported energy + assert ( + len([name for name in names if "Processor Energy" in name]) == 2 + ), f"Expected package and DRAM to both be aggregated, got {names}" + + +@pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") +@pytest.mark.parametrize("include_dram, expected_uj", [(False, 100000), (True, 125000)]) +def test_rapl_include_dram_energy_is_aggregated(tmp_path, include_dram, expected_uj): + """ + Verify that DRAM energy actually contributes to the total reported by + get_cpu_details() when rapl_include_dram=True. + """ + base = tmp_path + rapl_provider = base / "intel-rapl" + rapl_provider.mkdir() + + d_package = rapl_provider / "intel-rapl:0" + d_package.mkdir() + (d_package / "name").write_text("package-0") + (d_package / "energy_uj").write_text("1000000") + (d_package / "max_energy_range_uj").write_text("262143328850") + + d_dram = rapl_provider / "intel-rapl:1" + d_dram.mkdir() + (d_dram / "name").write_text("dram") + (d_dram / "energy_uj").write_text("500000") + (d_dram / "max_energy_range_uj").write_text("262143328850") + + rapl = IntelRAPL(rapl_dir=str(base), rapl_include_dram=include_dram) + + # Simulate one second of consumption + (d_package / "energy_uj").write_text(str(1000000 + 100000)) + (d_dram / "energy_uj").write_text(str(500000 + 25000)) + + details = rapl.get_cpu_details(Time.from_seconds(1)) + + energy = sum( + value + for metric, value in details.items() + if re.match(r"^Processor Energy Delta_\d", metric) + ) + # micro joules -> kWh + assert energy == pytest.approx(expected_uj / (1000 * 3600 * 1e6)) @pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") @@ -295,9 +339,14 @@ def test_rapl_both_parameters_together(tmp_path): assert ( len(rapl2._rapl_files) == 2 ), f"Expected 2 files (package + dram), got {len(rapl2._rapl_files)}" + # Both domains get an aggregated name so their energy is summed in the total names = [f.name for f in rapl2._rapl_files] - assert any("Processor Energy" in name for name in names), "Missing package domain" - assert any("dram" in name.lower() for name in names), "Missing DRAM domain" + assert all( + "Processor Energy" in name for name in names + ), f"All selected domains should be aggregated, got: {names}" + paths = [f.path for f in rapl2._rapl_files] + assert any("intel-rapl:0" in path for path in paths), "Missing package domain" + assert any("intel-rapl:1" in path for path in paths), "Missing DRAM domain" # Test 3: rapl_prefer_psys=False with rapl_include_dram=False (should use only package) rapl3 = IntelRAPL( @@ -475,3 +524,26 @@ def test_rapl_parameters_stored_correctly(tmp_path): ) assert rapl_true.rapl_include_dram is True assert rapl_true.rapl_prefer_psys is True + + +@pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") +def test_rapl_non_power_domain_keeps_its_own_name(tmp_path): + """ + When no package/psys/dram domain exists, the remaining domains are used as a + fallback but must not be renamed into the aggregated "Processor Energy" total. + """ + base = tmp_path + rapl_provider = base / "intel-rapl" + rapl_provider.mkdir() + + d_core = rapl_provider / "intel-rapl:0" + d_core.mkdir() + (d_core / "name").write_text("core") + (d_core / "energy_uj").write_text("1000000") + (d_core / "max_energy_range_uj").write_text("262143328850") + + rapl = IntelRAPL(rapl_dir=str(base)) + + names = [f.name for f in rapl._rapl_files] + assert names, "Fallback should still expose the available domain" + assert not any("Processor Energy" in name for name in names), names From c2cc54b89c69159244ce47851262ce2f649a602b Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 23 Sep 2026 16:52:09 +0900 Subject: [PATCH 2/3] fix(rapl): exclude DRAM from mirror detection by type and honour rapl_include_dram in fallback Co-Authored-By: Claude Opus 5.5 (1M context) --- codecarbon/core/cpu.py | 15 ++++++++++++--- codecarbon/core/rapl.py | 2 ++ tests/test_rapl_mmio_scanning.py | 4 +++- tests/test_rapl_parameters.py | 21 +++++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 5ad13237c..d25d0ca71 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -744,7 +744,11 @@ def _select_domains_to_use( logger.warning( "\tRAPL - No package or psys domains found, using all available domains" ) - domains_to_use = readable_domains + domains_to_use = [ + domain + for domain in readable_domains + if self.rapl_include_dram or "dram" not in (domain[5] or "").lower() + ] return domains_to_use @@ -790,7 +794,12 @@ def _create_rapl_files(self, domain_map: dict, found_main_readable: bool): interface_type = "MMIO" if is_mmio else "MSR" self._rapl_files.append( - RAPLFile(name=display_name, path=rapl_file, max_path=rapl_file_max) + RAPLFile( + name=display_name, + path=rapl_file, + max_path=rapl_file_max, + is_dram="dram" in domain_lower, + ) ) logger.info( "\tRAPL - Monitoring domain '%s' (displayed as '%s') via %s at %s", @@ -893,7 +902,7 @@ def start(self) -> None: counters = [ (rapl_file.path, float(rapl_file.last_energy)) for rapl_file in self._rapl_files - if "dram" not in rapl_file.name.lower() + if not rapl_file.is_dram ] self._mirrored_candidates = find_mirrored_counters( counters, SEQUENTIAL_READ_TOLERANCE_KWH diff --git a/codecarbon/core/rapl.py b/codecarbon/core/rapl.py index 46bd6e864..ae2d78889 100644 --- a/codecarbon/core/rapl.py +++ b/codecarbon/core/rapl.py @@ -68,6 +68,8 @@ class RAPLFile: last_energy: Energy = field(default_factory=lambda: Energy(0)) # Max value energy can hold before it wraps max_energy_reading: Energy = field(default_factory=lambda: Energy(0)) + # DRAM domain: a separate measurement, never a mirror of a package counter + is_dram: bool = False def __post_init__(self): self.last_energy = self._get_value() diff --git a/tests/test_rapl_mmio_scanning.py b/tests/test_rapl_mmio_scanning.py index 5175e43d8..4b1cec888 100644 --- a/tests/test_rapl_mmio_scanning.py +++ b/tests/test_rapl_mmio_scanning.py @@ -73,7 +73,9 @@ def test_rapl_start_keeps_dram_when_it_matches_a_package_counter(tmp_path, monke assert len(rapl._mirrored_candidates) == 1 details = rapl.get_cpu_details(Time.from_seconds(1)) assert len(_counted_domains(details)) == 2 - assert "dram" in details + dram = next(f for f in rapl._rapl_files if f.is_dram) + assert dram.path not in rapl._mirrored_candidates + assert dram.name in details def test_rapl_start_reevaluates_mirror_candidates(tmp_path, monkeypatch): diff --git a/tests/test_rapl_parameters.py b/tests/test_rapl_parameters.py index ac600ccfc..354255d12 100644 --- a/tests/test_rapl_parameters.py +++ b/tests/test_rapl_parameters.py @@ -526,6 +526,27 @@ def test_rapl_parameters_stored_correctly(tmp_path): assert rapl_true.rapl_prefer_psys is True +@pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") +@pytest.mark.parametrize("include_dram", [False, True]) +def test_rapl_fallback_respects_include_dram(tmp_path, include_dram): + """ + Without package/psys domains every readable domain is used as a fallback, but + DRAM must still only be counted when rapl_include_dram is set. + """ + rapl_provider = tmp_path / "intel-rapl" + rapl_provider.mkdir() + for index, name in enumerate(("core", "dram")): + domain = rapl_provider / f"intel-rapl:{index}" + domain.mkdir() + (domain / "name").write_text(name) + (domain / "energy_uj").write_text("1000000") + (domain / "max_energy_range_uj").write_text("262143328850") + + rapl = IntelRAPL(rapl_dir=str(tmp_path), rapl_include_dram=include_dram) + + assert any(f.is_dram for f in rapl._rapl_files) is include_dram + + @pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") def test_rapl_non_power_domain_keeps_its_own_name(tmp_path): """ From 944063989cdf5fbb678fe68ad0b88e514616e6f1 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Mon, 28 Sep 2026 15:38:03 +0900 Subject: [PATCH 3/3] fix: report RAPL DRAM as RAM energy instead of adding it to CPU energy --- codecarbon/core/cpu.py | 89 ++++++++++++----------- codecarbon/core/rapl.py | 2 - codecarbon/core/resource_tracker.py | 25 +++++++ codecarbon/emissions_tracker.py | 17 ++--- codecarbon/external/ram.py | 36 +++++++++- docs/explanation/methodology.md | 8 ++- docs/explanation/power-estimation.md | 2 +- docs/explanation/rapl.md | 20 +++--- docs/how-to/configuration.md | 32 ++++++--- tests/test_ram.py | 80 +++++++++++++++++++++ tests/test_rapl_mmio_scanning.py | 11 +-- tests/test_rapl_parameters.py | 102 +++++++++++++++++++++++---- tests/test_resource_tracker.py | 35 +++++++++ 13 files changed, 367 insertions(+), 92 deletions(-) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index d25d0ca71..bc2b3d9f7 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -12,7 +12,7 @@ import subprocess import sys from functools import lru_cache -from typing import TYPE_CHECKING, Dict, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple import psutil from rapidfuzz import fuzz, process, utils @@ -23,7 +23,7 @@ counters_match, find_mirrored_counters, ) -from codecarbon.core.units import Time +from codecarbon.core.units import Energy, Power, Time from codecarbon.core.util import count_cpus, detect_cpu_model from codecarbon.external.logger import logger @@ -421,15 +421,15 @@ class IntelRAPL: _rapl_files (List[RAPLFile]): A list of RAPLFile objects representing the files to read energy data from. _cpu_details (Dict): A dictionary storing the latest CPU energy details. _last_mesure (int): Placeholder for storing the last measurement time. - rapl_include_dram (bool): Whether to include DRAM power in measurements (default: False for complete hardware measurement). + rapl_include_dram (bool): Whether to read the DRAM domains, reported by get_dram_energy() and never in the CPU details (default: False). rapl_prefer_psys (bool): Whether to prefer psys domain over package domains (default: False). When True, uses psys (platform/system) domain which includes CPU + platform components. When False (default), uses package domains which are more reliable and match CPU TDP specs. Args: rapl_dir (str): Path to RAPL directory (default: "/sys/class/powercap/intel-rapl/subsystem") - rapl_include_dram (bool): Include DRAM domain for complete hardware measurement (default: False). - Set to False to measure only CPU package power. + rapl_include_dram (bool): Read the DRAM domains alongside the package ones, for the + RAM energy (default: False). Ignored with psys domains. rapl_prefer_psys (bool): Prefer psys (platform) domain over package domains (default: False). Set to True to measure total platform power (CPU + chipset + PCIe). Note: psys can report higher values than CPU TDP and may be less reliable on older systems. @@ -455,6 +455,8 @@ def __init__( self._lin_rapl_dir = rapl_dir self._system = sys.platform.lower() self._rapl_files = [] + # DRAM domains, read apart to report the RAM energy (never the CPU's) + self._dram_files: List[RAPLFile] = [] # Files that look like a duplicate of another counter, but whose # energy deltas have not confirmed it yet. They are left out of the # measurement while pending. Maps file path -> mirrored file path. @@ -664,24 +666,14 @@ def _classify_domains(self, readable_domains: list): domain_dir, ) elif "dram" in domain_lower: - parent_dir = os.path.dirname(domain_dir) - if ( - parent_dir.endswith(("intel-rapl", "intel-rapl-mmio")) - or os.path.basename(domain_dir).count(":") == 1 - ): - dram_domains.append(domain_tuple) - logger.debug( - "\tRAPL - Found top-level DRAM domain '%s' at %s", - domain_name, - domain_dir, - ) - else: - subdomain_of_package.append(domain_tuple) - logger.debug( - "\tRAPL - Found DRAM subdomain '%s' at %s (will be skipped to avoid double-counting)", - domain_name, - domain_dir, - ) + # Top-level or a child zone of its package (the usual layout on + # servers): the package energy never includes the DRAM's + dram_domains.append(domain_tuple) + logger.debug( + "\tRAPL - Found DRAM domain '%s' at %s", + domain_name, + domain_dir, + ) elif any(sub in domain_lower for sub in ["core", "uncore"]): subdomain_of_package.append(domain_tuple) logger.debug( @@ -716,14 +708,14 @@ def _select_domains_to_use( if self.rapl_include_dram and dram_domains: logger.info( - "\tRAPL - Including %d DRAM domain(s) for complete hardware power measurement (CPU+DRAM)", + "\tRAPL - Reading %d DRAM domain(s) as the RAM energy", len(dram_domains), ) domains_to_use.extend(dram_domains) elif dram_domains and not self.rapl_include_dram: logger.info( "\tRAPL - Found %d DRAM domain(s) but not including (rapl_include_dram=False). " - "Set rapl_include_dram=True for complete hardware measurement.", + "Set rapl_include_dram=True to measure the RAM energy with it.", len(dram_domains), ) @@ -759,7 +751,12 @@ def _deduplicate_domains(self, domains_to_use: list): name, domain_dir, is_mmio, rapl_file, rapl_file_max, domain_name = ( domain_tuple ) - base_name = domain_name if domain_name else os.path.basename(domain_dir) + zone_id = os.path.basename(domain_dir) + base_name = domain_name if domain_name else zone_id + if "dram" in base_name.lower(): + # Every socket's DRAM zone is named "dram": key on the zone id + # (intel-rapl:N:M), the same whatever path it was found through + base_name = zone_id if base_name not in domain_map or ( is_mmio and not domain_map[base_name][2] ): @@ -784,22 +781,19 @@ def _create_rapl_files(self, domain_map: dict, found_main_readable: bool): ) in domain_map.values(): try: domain_lower = (domain_name or "").lower() - if any( - keyword in domain_lower for keyword in ("package", "psys", "dram") - ): + is_dram = "dram" in domain_lower + if is_dram: + display_name = f"DRAM Energy Delta_{len(self._dram_files)}(kWh)" + elif "package" in domain_lower or "psys" in domain_lower: display_name = f"Processor Energy Delta_{domain_index}(kWh)" domain_index += 1 else: display_name = name interface_type = "MMIO" if is_mmio else "MSR" - self._rapl_files.append( - RAPLFile( - name=display_name, - path=rapl_file, - max_path=rapl_file_max, - is_dram="dram" in domain_lower, - ) + files = self._dram_files if is_dram else self._rapl_files + files.append( + RAPLFile(name=display_name, path=rapl_file, max_path=rapl_file_max) ) logger.info( "\tRAPL - Monitoring domain '%s' (displayed as '%s') via %s at %s", @@ -831,7 +825,8 @@ def _fetch_rapl_files(self) -> None: Fetches RAPL files from the RAPL directory. By default, reads CPU package only - Set rapl_include_dram=True to measure CPU package + DRAM domains + Set rapl_include_dram=True to also read the DRAM domains, which are + reported as the RAM energy by get_dram_energy() """ candidate_bases = self._get_rapl_candidate_bases() domain_dirs = self._collect_domain_dirs(candidate_bases) @@ -892,17 +887,31 @@ def get_static_cpu_details(self) -> Dict: """ return self._cpu_details + def get_dram_energy(self, duration: Time) -> Optional[Tuple[Power, Energy]]: + """ + Power and energy of the DRAM domains since the previous call, or None + when no DRAM domain is read (rapl_include_dram=False or none found). + """ + if not self._dram_files: + return None + for rapl_file in self._dram_files: + rapl_file.delta(duration) + return ( + Power.from_watts(sum(rapl_file.power.W for rapl_file in self._dram_files)), + Energy.from_energy( + sum(rapl_file.energy_delta.kWh for rapl_file in self._dram_files) + ), + ) + def start(self) -> None: """ Starts monitoring CPU energy consumption. """ - for rapl_file in self._rapl_files: + for rapl_file in self._rapl_files + self._dram_files: rapl_file.start() - # DRAM domains are a different measurement, never a duplicate counters = [ (rapl_file.path, float(rapl_file.last_energy)) for rapl_file in self._rapl_files - if not rapl_file.is_dram ] self._mirrored_candidates = find_mirrored_counters( counters, SEQUENTIAL_READ_TOLERANCE_KWH diff --git a/codecarbon/core/rapl.py b/codecarbon/core/rapl.py index ae2d78889..46bd6e864 100644 --- a/codecarbon/core/rapl.py +++ b/codecarbon/core/rapl.py @@ -68,8 +68,6 @@ class RAPLFile: last_energy: Energy = field(default_factory=lambda: Energy(0)) # Max value energy can hold before it wraps max_energy_reading: Energy = field(default_factory=lambda: Energy(0)) - # DRAM domain: a separate measurement, never a mirror of a package counter - is_dram: bool = False def __post_init__(self): self.last_energy = self._get_value() diff --git a/codecarbon/core/resource_tracker.py b/codecarbon/core/resource_tracker.py index e20838718..648e9be32 100644 --- a/codecarbon/core/resource_tracker.py +++ b/codecarbon/core/resource_tracker.py @@ -329,3 +329,28 @@ def set_CPU_GPU_ram_tracking(self): param tracker: BaseEmissionsTracker object """ get_or_run_setup(self, self._run_full_hardware_setup) + self._use_measured_dram() + + def _use_measured_dram(self) -> None: + """ + Report the RAPL DRAM energy (rapl_include_dram=True) as the RAM energy + rather than the estimate. Not cached with the hardware plan, as the + RAM has to read the DRAM files of this very CPU instance. + """ + ram = next((hw for hw in self.tracker._hardware if isinstance(hw, RAM)), None) + cpu_hw = next( + (hw for hw in self.tracker._hardware if isinstance(hw, CPU)), None + ) + if ( + ram is None + or cpu_hw is None + or ram._force_ram_power is not None + # The DRAM counter is machine-wide: keep the per-process estimate + or ram._tracking_mode != "machine" + or cpu_hw._mode != "intel_rapl" + or not cpu_hw._intel_interface._dram_files + ): + return + ram._dram_source = cpu_hw._intel_interface + self.ram_tracker = "RAPL DRAM measurement (estimation model as fallback)" + logger.info(f"RAM Tracking Method: {self.ram_tracker}") diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 4bd0ea99d..addec3143 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -512,11 +512,12 @@ def __init__( :param force_mode_cpu_load: Force the addition of a CPU in MODE_CPU_LOAD :param allow_multiple_runs: Allow multiple CodeCarbon instances on the same machine. Defaults to True since v3 (was False in v2). - :param rapl_include_dram: Include DRAM (memory) power in the counter-based CPU - measurements, defaults to False. When True, measures - CPU package + DRAM. Applies to the Linux RAPL interface - and to the Windows Energy Meter Interface, on systems - exposing separate DRAM domains/channels. + :param rapl_include_dram: Read the DRAM (memory) energy counters, defaults + to False. On Linux RAPL, the DRAM domain energy + is reported as the RAM energy instead of its + estimate (machine tracking mode, package domains + only). On the Windows Energy Meter Interface, + the DRAM channels are added to the CPU energy. :param rapl_prefer_psys: Prefer psys (platform) RAPL domain over package domains on Linux, defaults to False. When True, uses total platform power (CPU + chipset + PCIe). When False, uses package domains which @@ -1621,9 +1622,9 @@ def track_emissions( litres of water consumed per kilowatt-hour of electricity consumed. :param force_carbon_intensity_g_co2e_kwh: Override grid carbon intensity in gCO2e/kWh for emissions calculations. - :param rapl_include_dram: Include DRAM in the counter-based CPU measurements - (Linux RAPL and Windows EMI, default: False). - When True, measures CPU package + DRAM. + :param rapl_include_dram: Read the DRAM energy counters (default: False). + On Linux RAPL, reported as the RAM energy instead + of its estimate; on Windows EMI, added to the CPU. :param rapl_prefer_psys: Prefer psys over package domains for RAPL on Linux (default: False). When True, uses total platform power. diff --git a/codecarbon/external/ram.py b/codecarbon/external/ram.py index b417e1c5b..57b26a558 100644 --- a/codecarbon/external/ram.py +++ b/codecarbon/external/ram.py @@ -2,16 +2,19 @@ import re import subprocess from dataclasses import dataclass -from typing import Optional +from typing import Optional, Tuple import psutil -from codecarbon.core.units import Power +from codecarbon.core.units import Energy, Power, Time from codecarbon.core.util import SLURM_JOB_ID from codecarbon.external.hardware import B_TO_GB, BaseHardware from codecarbon.external.logger import logger RAM_SLOT_POWER_X86 = 5 # Watts +# Measurement intervals a DRAM counter may stay still before it is deemed +# unimplemented: client CPUs often expose a DRAM domain stuck at 0 +DRAM_DEAD_AFTER_INTERVALS = 3 @dataclass @@ -31,6 +34,11 @@ class RAM(BaseHardware): memory_size = None is_arm_cpu = False + # Interface measuring the DRAM energy (IntelRAPL with rapl_include_dram) + _dram_source = None + # Whether the DRAM counter was seen moving, and intervals it has not + _dram_alive = False + _dram_still_intervals = 0 def __init__( self, @@ -339,3 +347,27 @@ def total_power(self) -> Power: ram_power = Power.from_watts(0) return ram_power + + def measure_power_and_energy(self, last_duration: float) -> Tuple[Power, Energy]: + """ + Use the measured DRAM energy when a DRAM domain is read, otherwise + estimate it. Until the DRAM counter is seen moving, the estimate is + used, and a counter that never moves is dropped for good. + """ + if self._dram_source is not None: + measured = self._dram_source.get_dram_energy(Time(seconds=last_duration)) + # A negative delta is a counter wrap we could not correct: estimate + # this interval instead of reporting negative energy. + if measured is not None and measured[1].kWh >= 0: + if self._dram_alive or measured[1].kWh > 0: + self._dram_alive = True + return measured + self._dram_still_intervals += 1 + if self._dram_still_intervals >= DRAM_DEAD_AFTER_INTERVALS: + logger.warning( + "The DRAM energy counter did not move over %d measurements, " + "falling back to the RAM power estimation model", + self._dram_still_intervals, + ) + self._dram_source = None + return super().measure_power_and_energy(last_duration=last_duration) diff --git a/docs/explanation/methodology.md b/docs/explanation/methodology.md index 5437ee158..0fbf4f7b9 100644 --- a/docs/explanation/methodology.md +++ b/docs/explanation/methodology.md @@ -117,6 +117,12 @@ Tracks Nvidia GPUs energy consumption using `nvidia-ml-py` library ### RAM +On Linux, with +[`rapl_include_dram`](../how-to/configuration.md#measuring-ram-with-the-dram-energy-counter) +enabled, the RAM energy is measured by the RAPL `dram` domain when it +exists and its counter increases. Otherwise, it is estimated as described +below. + CodeCarbon v2 uses a 3 Watts for 8 GB ratio [source](https://www.crucial.com/support/articles-faq-memory/how-much-power-does-memory-use) . @@ -225,7 +231,7 @@ twice, so CodeCarbon keeps the package channels only. On multi-die CPUs where every die mirrors the same socket-wide counter, the duplicates are detected and dropped as well. The `DRAM` channels are excluded too, unless the -[`rapl_include_dram`](../how-to/configuration.md#including-dram-in-the-cpu-measurement) +[`rapl_include_dram`](../how-to/configuration.md#measuring-ram-with-the-dram-energy-counter) option is enabled. Legacy support for `Intel Power Gadget` is kept for machines where it is diff --git a/docs/explanation/power-estimation.md b/docs/explanation/power-estimation.md index b56880429..dc919f217 100644 --- a/docs/explanation/power-estimation.md +++ b/docs/explanation/power-estimation.md @@ -11,7 +11,7 @@ The most accurate tracking methods rely on built-in hardware energy counters rat - **NVIDIA GPUs** using `nvmlDeviceGetTotalEnergyConsumption` return accumulated energy in millijoules. - **AMD GPUs** using `amdsmi_get_energy_count` yield a counter that is multiplied by its resolution and converted into millijoules. - **CPUs** using the RAPL interface read from files like `energy_uj` to get accumulated microjoules. -- **RAM** using the RAPL interface read from files like `energy_uj` to get accumulated microjoules. See `rapl_include_dram` option. Not used by default. +- **RAM** using the RAPL `dram` domain read from files like `energy_uj` to get accumulated microjoules, on Linux. See `rapl_include_dram` option. Not used by default. At every measurement interval, CodeCarbon calculates the `energy_delta` by subtracting the previously tracked `last_energy` from the current total energy reading. diff --git a/docs/explanation/rapl.md b/docs/explanation/rapl.md index d37ff5359..6d7f16235 100644 --- a/docs/explanation/rapl.md +++ b/docs/explanation/rapl.md @@ -96,8 +96,8 @@ and consistent measurements: - Match CPU TDP specifications - Provide consistent measurements across different Intel generations - - Can be supplemented with `dram` domains for complete hardware - measurement (package + DRAM) + - `dram` domains are never added to them: with + `rapl_include_dram=True` they are reported as the RAM energy 2. **Optional psys mode**: Set `prefer_psys=True` to use `psys` (platform/system) domain instead: - Provides total platform power (CPU + chipset + PCIe + some other @@ -113,9 +113,13 @@ and consistent measurements: - Falls back to MSR if MMIO is unreadable 4. **Subdomain filtering**: Excludes `core` and `uncore` subdomains when `package` is available to avoid double-counting -5. **DRAM exclusion**: By default (`include_dram=False`), don't add - DRAM domain to package. As DRAM is supposed to be in RAM power, not - CPU in a future version of CodeCarbon. +5. **DRAM as RAM**: By default (`rapl_include_dram=False`), the DRAM + domains are not read and the RAM power is estimated. With + `rapl_include_dram=True` and `package` domains, the DRAM domains + (top-level, or children of their package as on most servers) are + reported as the RAM energy, never added to the CPU package. With + `psys`, which usually includes the memory, the DRAM domains are not + read and the RAM keeps its estimate. ## Platform-Specific Behavior @@ -370,9 +374,9 @@ Analysis: 5. **Interface deduplication**: The same domain may appear in both `intel-rapl` (MSR) and `intel-rapl-mmio` interfaces. CodeCarbon automatically deduplicates, preferring MMIO. -6. **DRAM measurement**: CodeCarbon does not include DRAM domains by - default (`include_dram=False`) for CPU hardware measurement. Set - `include_dram=True` to measure CPU package + DRAM domains. +6. **DRAM measurement**: CodeCarbon does not read DRAM domains by + default (`rapl_include_dram=False`). Set `rapl_include_dram=True` to + report the DRAM domains as the RAM energy instead of its estimate. 7. **Platform-specific behavior**: - Intel modern: package or psys (with prefer_psys=True) - Intel older: package-0 for CPU only diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 9075470e9..ae271bb98 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -162,13 +162,13 @@ EmissionsTracker(tracking_mode="process") `"process"` mode gives a lower-bound estimate of your code's footprint. `"machine"` mode is more conservative and accounts for all activity on the system. -## Including DRAM in the CPU Measurement +## Measuring RAM with the DRAM Energy Counter When CodeCarbon reads the CPU energy counters, the hardware also exposes a -`DRAM` domain measuring the memory controller. It is **excluded by default**, so -that memory power is reported by the RAM tracker only and is not counted twice. +`DRAM` domain measuring the memory. It is **not read by default**: the RAM +power comes from the [estimation model](../explanation/methodology.md#ram). -Set `rapl_include_dram` to add it to the CPU measurement: +Set `rapl_include_dram` to measure the RAM with it instead: ``` ini [codecarbon] @@ -181,15 +181,25 @@ Or in code: EmissionsTracker(rapl_include_dram=True) ``` -Despite its name, this option applies to every counter-based CPU interface: - -- **Linux**: the `dram` domains of the [RAPL](../explanation/rapl.md) powercap - interface. +What it does depends on the interface: + +- **Linux**: the energy of the `dram` domains of the + [RAPL](../explanation/rapl.md) powercap interface is reported as the RAM + energy and power, and is not part of the CPU energy. When no `dram` domain + exists, the RAM keeps its estimate. Until the counter is seen moving, the + estimate is used, and a counter that never moves over the first few + measurements (client CPUs often expose a `dram` domain stuck at 0) is + dropped for the estimate. When the CPU is measured with the `psys` domain + (`rapl_prefer_psys`, or no `package` domain), which usually includes the + memory already, the `dram` domain is not read and the RAM keeps its + estimate. - **Windows 11**: the `DRAM` channels of the Energy Meter Interface, which - exposes the very same RAPL counters. + exposes the very same RAPL counters, are still added to the CPU energy, + while the RAM keeps its estimate. The memory power is then counted twice. -It has no effect when CodeCarbon falls back to TDP/CPU-load estimation, since -that mode models the CPU package only. +The option is ignored for the RAM when `force_ram_power` is set or when +`tracking_mode="process"`, since the DRAM counter measures the whole machine. +It has no effect when CodeCarbon falls back to TDP/CPU-load estimation. ## Access internet through proxy server diff --git a/tests/test_ram.py b/tests/test_ram.py index 6b553dd3c..dfc09d1ce 100644 --- a/tests/test_ram.py +++ b/tests/test_ram.py @@ -4,7 +4,9 @@ from unittest import mock import numpy as np +import pytest +from codecarbon.core.units import Energy, Power from codecarbon.external.ram import RAM, RAM_SLOT_POWER_X86 # TODO: need help: test multiprocess case @@ -437,3 +439,81 @@ def test_force_ram_power(self): ram_power = ram.total_power() # Verify the calculation method was not called mock_calc.assert_not_called() + + +def test_ram_reports_measured_dram_energy(): + """A working RAPL DRAM domain replaces the estimate (issue #1268).""" + source = mock.Mock() + source.get_dram_energy.return_value = ( + Power.from_watts(3.0), + Energy.from_energy(kWh=1e-6), + ) + ram = RAM(tracking_mode="machine") + ram._dram_source = source + + with mock.patch.object(RAM, "total_power") as estimate: + power, energy = ram.measure_power_and_energy(last_duration=2.0) + + estimate.assert_not_called() + assert power.W == 3.0 + assert energy.kWh == 1e-6 + assert source.get_dram_energy.call_args.args[0].seconds == 2.0 + + +def _ram_with_dram_source(deltas_kwh): + source = mock.Mock() + source.get_dram_energy.side_effect = [ + (Power.from_watts(kwh * 3.6e6), Energy.from_energy(kWh=kwh)) + for kwh in deltas_kwh + ] + ram = RAM(tracking_mode="machine") + ram._dram_source = source + return ram, source + + +def test_ram_live_dram_counter_reports_a_zero_delta_as_measured(): + ram, _ = _ram_with_dram_source([1e-6, 0.0]) + with mock.patch.object(RAM, "total_power", return_value=Power.from_watts(10)): + ram.measure_power_and_energy(last_duration=1) + power, energy = ram.measure_power_and_energy(last_duration=0.01) + assert power.W == 0 + assert energy.kWh == 0 + + +def test_ram_negative_dram_delta_falls_back_to_estimate(): + """An uncorrected counter wrap must not be reported as negative energy.""" + ram, _ = _ram_with_dram_source([1e-6, -5e-6]) + with mock.patch.object(RAM, "total_power", return_value=Power.from_watts(10)): + ram.measure_power_and_energy(last_duration=1) + power, energy = ram.measure_power_and_energy(last_duration=1) + assert power.W == 10 + assert energy.kWh > 0 + + +def test_ram_dead_dram_counter_is_dropped_once(): + """A DRAM counter that never moves is abandoned for the estimate.""" + ram, source = _ram_with_dram_source([0.0] * 10) + with mock.patch.object(RAM, "total_power", return_value=Power.from_watts(10)): + for _ in range(6): + power, _ = ram.measure_power_and_energy(last_duration=1) + assert power.W == 10 + assert ram._dram_source is None + assert source.get_dram_energy.call_count < 6 + + +@pytest.mark.parametrize( + "measured", + # No DRAM domain, or a dead counter reading a flat 0 (client CPUs) + [None, (Power.from_watts(0), Energy.from_energy(kWh=0))], +) +def test_ram_falls_back_to_estimate_without_live_dram(measured): + source = mock.Mock() + source.get_dram_energy.return_value = measured + ram = RAM(tracking_mode="machine") + ram._dram_source = source + + with mock.patch.object(RAM, "total_power", return_value=Power.from_watts(10)): + power, energy = ram.measure_power_and_energy(last_duration=3600) + + assert power.W == 10 + assert energy.kWh == pytest.approx(0.01) diff --git a/tests/test_rapl_mmio_scanning.py b/tests/test_rapl_mmio_scanning.py index 4b1cec888..5ba0d0973 100644 --- a/tests/test_rapl_mmio_scanning.py +++ b/tests/test_rapl_mmio_scanning.py @@ -72,10 +72,13 @@ def test_rapl_start_keeps_dram_when_it_matches_a_package_counter(tmp_path, monke assert len(rapl._mirrored_candidates) == 1 details = rapl.get_cpu_details(Time.from_seconds(1)) - assert len(_counted_domains(details)) == 2 - dram = next(f for f in rapl._rapl_files if f.is_dram) - assert dram.path not in rapl._mirrored_candidates - assert dram.name in details + assert len(_counted_domains(details)) == 1 + # The DRAM domain is read apart for the RAM, never checked as a mirror + assert [f.path for f in rapl._dram_files] == [ + str(tmp_path / "intel-rapl" / "intel-rapl:2" / "energy_uj") + ] + assert rapl._dram_files[0].path not in rapl._mirrored_candidates + assert rapl.get_dram_energy(Time.from_seconds(1)) is not None def test_rapl_start_reevaluates_mirror_candidates(tmp_path, monkeypatch): diff --git a/tests/test_rapl_parameters.py b/tests/test_rapl_parameters.py index 354255d12..6a1bdc8f2 100644 --- a/tests/test_rapl_parameters.py +++ b/tests/test_rapl_parameters.py @@ -109,24 +109,20 @@ def test_rapl_include_dram_true_explicit(tmp_path): # Create IntelRAPL with rapl_include_dram=True explicitly rapl = IntelRAPL(rapl_dir=str(base), rapl_include_dram=True) - # Should have 2 RAPL files: package-0 + dram - assert len(rapl._rapl_files) == 2, f"Expected 2 files, got {len(rapl._rapl_files)}" + # The package is a CPU counter, the DRAM domain is read apart for the RAM + assert [f.name for f in rapl._rapl_files] == ["Processor Energy Delta_0(kWh)"] + assert len(rapl._dram_files) == 1 + assert "intel-rapl:1" in rapl._dram_files[0].path - # Verify both package and dram are present - names = [f.name for f in rapl._rapl_files] - # Both domains must be named so that they are summed in the reported energy - assert ( - len([name for name in names if "Processor Energy" in name]) == 2 - ), f"Expected package and DRAM to both be aggregated, got {names}" - -@pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") -@pytest.mark.parametrize("include_dram, expected_uj", [(False, 100000), (True, 125000)]) -def test_rapl_include_dram_energy_is_aggregated(tmp_path, include_dram, expected_uj): +@pytest.mark.parametrize("include_dram", [False, True]) +def test_rapl_cpu_energy_excludes_dram(tmp_path, monkeypatch, include_dram): """ - Verify that DRAM energy actually contributes to the total reported by - get_cpu_details() when rapl_include_dram=True. + The DRAM energy is never added to the CPU energy: with + rapl_include_dram=True it is reported by the RAM component instead + (https://github.com/mlco2/codecarbon/issues/1268). """ + monkeypatch.setattr(sys, "platform", "linux") base = tmp_path rapl_provider = base / "intel-rapl" rapl_provider.mkdir() @@ -144,12 +140,14 @@ def test_rapl_include_dram_energy_is_aggregated(tmp_path, include_dram, expected (d_dram / "max_energy_range_uj").write_text("262143328850") rapl = IntelRAPL(rapl_dir=str(base), rapl_include_dram=include_dram) + rapl.start() # Simulate one second of consumption (d_package / "energy_uj").write_text(str(1000000 + 100000)) (d_dram / "energy_uj").write_text(str(500000 + 25000)) details = rapl.get_cpu_details(Time.from_seconds(1)) + dram = rapl.get_dram_energy(Time.from_seconds(1)) energy = sum( value @@ -157,7 +155,13 @@ def test_rapl_include_dram_energy_is_aggregated(tmp_path, include_dram, expected if re.match(r"^Processor Energy Delta_\d", metric) ) # micro joules -> kWh - assert energy == pytest.approx(expected_uj / (1000 * 3600 * 1e6)) + assert energy == pytest.approx(100000 / (1000 * 3600 * 1e6)) + if include_dram: + power, dram_energy = dram + assert dram_energy.kWh == pytest.approx(25000 / (1000 * 3600 * 1e6)) + assert power.W == pytest.approx(0.025) + else: + assert dram is None @pytest.mark.skipif(not sys.platform.lower().startswith("lin"), reason="requires Linux") @@ -568,3 +572,71 @@ def test_rapl_non_power_domain_keeps_its_own_name(tmp_path): names = [f.name for f in rapl._rapl_files] assert names, "Fallback should still expose the available domain" assert not any("Processor Energy" in name for name in names), names + + +def _make_nested_dram_tree(base, sockets): + """ + Mimic real Xeon sysfs: each DRAM zone is a child of its package + (intel-rapl/intel-rapl:N/intel-rapl:N:0), with flat symlinks + intel-rapl:N and intel-rapl:N:0 next to the control type directory. + """ + control = base / "intel-rapl" + control.mkdir() + dram_files = [] + for socket in range(sockets): + package = control / f"intel-rapl:{socket}" + package.mkdir() + (package / "name").write_text(f"package-{socket}") + (package / "energy_uj").write_text("1000000") + (package / "max_energy_range_uj").write_text("262143328850") + dram = package / f"intel-rapl:{socket}:0" + dram.mkdir() + (dram / "name").write_text("dram") + (dram / "energy_uj").write_text("500000") + (dram / "max_energy_range_uj").write_text("262143328850") + (base / f"intel-rapl:{socket}").symlink_to(package) + (base / f"intel-rapl:{socket}:0").symlink_to(dram) + dram_files.append(dram / "energy_uj") + return dram_files + + +@pytest.mark.parametrize("sockets", [1, 2]) +def test_rapl_reads_nested_dram_zones_once_per_socket(tmp_path, monkeypatch, sockets): + monkeypatch.setattr(sys, "platform", "linux") + dram_files = _make_nested_dram_tree(tmp_path, sockets) + + # Scanning /intel-rapl and sees each DRAM zone twice + rapl = IntelRAPL(rapl_dir=str(tmp_path / "intel-rapl"), rapl_include_dram=True) + rapl.start() + + assert len(rapl._dram_files) == sockets + assert len(rapl._rapl_files) == sockets + for dram_file in dram_files: + dram_file.write_text(str(500000 + 25000)) + + _, energy = rapl.get_dram_energy(Time.from_seconds(1)) + assert energy.kWh == pytest.approx(sockets * 25000 / (1000 * 3600 * 1e6)) + + +@pytest.mark.parametrize("prefer_psys", [True, False]) +def test_rapl_psys_does_not_read_dram(tmp_path, monkeypatch, prefer_psys): + """psys usually includes the DRAM already: it is never read apart.""" + monkeypatch.setattr(sys, "platform", "linux") + provider = tmp_path / "intel-rapl" + provider.mkdir() + domains = [("psys", "intel-rapl:1"), ("dram", "intel-rapl:2")] + if prefer_psys: + domains.append(("package-0", "intel-rapl:0")) + for name, zone in domains: + d = provider / zone + d.mkdir() + (d / "name").write_text(name) + (d / "energy_uj").write_text("1000000") + (d / "max_energy_range_uj").write_text("262143328850") + + rapl = IntelRAPL( + rapl_dir=str(tmp_path), rapl_include_dram=True, rapl_prefer_psys=prefer_psys + ) + + assert rapl._dram_files == [] + assert rapl.get_dram_energy(Time.from_seconds(1)) is None diff --git a/tests/test_resource_tracker.py b/tests/test_resource_tracker.py index 27a7d1b4d..2c06e5352 100644 --- a/tests/test_resource_tracker.py +++ b/tests/test_resource_tracker.py @@ -645,3 +645,38 @@ def __bool__(self): ) assert resource_tracker.cpu_tracker == MODE_CPU_LOAD assert tracker._hardware == [hardware_cpu] + + +@pytest.mark.parametrize( + "mode, force_ram_power, dram_files, tracking_mode, wired", + [ + ("intel_rapl", None, ["dram"], "machine", True), + # rapl_include_dram=False: no DRAM domain is read + ("intel_rapl", None, [], "machine", False), + # The user-provided RAM power wins + ("intel_rapl", 12.5, ["dram"], "machine", False), + ("windows_emi", None, ["dram"], "machine", False), + # The DRAM counter is machine-wide, keep the per-process estimate + ("intel_rapl", None, ["dram"], "process", False), + ], +) +def test_ram_uses_rapl_dram_when_available( + mode, force_ram_power, dram_files, tracking_mode, wired +): + # The classes resource_tracker checks against: another test drops + # codecarbon.external.hardware from sys.modules, so a fresh import would + # give a different CPU class. + from codecarbon.core.resource_tracker import CPU, RAM + + ram = RAM(tracking_mode=tracking_mode, force_ram_power=force_ram_power) + cpu = CPU.__new__(CPU) + cpu._mode = mode + cpu._intel_interface = SimpleNamespace(_dram_files=dram_files) + tracker = make_tracker(_hardware=[ram, cpu], _force_ram_power=force_ram_power) + resource_tracker = ResourceTracker(tracker) + + with patch("codecarbon.core.resource_tracker.get_or_run_setup"): + resource_tracker.set_CPU_GPU_ram_tracking() + + assert (getattr(ram, "_dram_source", None) is cpu._intel_interface) == wired + assert ("RAPL DRAM" in resource_tracker.ram_tracker) == wired