From 6a76321951cd49ff5740280db67bda11606324d8 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Thu, 10 Sep 2026 23:36:31 +0000 Subject: [PATCH 1/9] Add ESXi support to platform in-band collectors Enable os, bios, dimm, kernel, storage, and device_enumeration collectors on ESXi via esxcli/smbiosDump. kernel reuses the existing `uname -a` path (ESXi reports the release in the same field). dimm shares a _parse_dmi_sizes helper across dmidecode (Linux) and smbiosDump (ESXi). device_enumeration counts GPU PF/VF by device ID, adding devid_ep/devid_ep_vf to SystemInfo. Validated on ESXi 9.1.0 and Linux; Linux/Windows paths unchanged. --- nodescraper/models/systeminfo.py | 2 + .../plugins/inband/bios/bios_collector.py | 8 ++++ .../device_enumeration_collector.py | 28 +++++++++++ .../plugins/inband/dimm/dimm_collector.py | 47 ++++++++++--------- .../plugins/inband/kernel/kernel_collector.py | 4 ++ nodescraper/plugins/inband/os/os_collector.py | 29 ++++++++++++ .../inband/storage/storage_collector.py | 20 ++++++++ 7 files changed, 115 insertions(+), 23 deletions(-) diff --git a/nodescraper/models/systeminfo.py b/nodescraper/models/systeminfo.py index d91a68cf..d593a9a0 100644 --- a/nodescraper/models/systeminfo.py +++ b/nodescraper/models/systeminfo.py @@ -44,3 +44,5 @@ class SystemInfo(BaseModel): metadata: Optional[dict] = Field(default_factory=dict) location: Optional[SystemLocation] = SystemLocation.LOCAL vendorid_ep: int = 0x1002 + devid_ep: Optional[int] = None + devid_ep_vf: Optional[int] = None diff --git a/nodescraper/plugins/inband/bios/bios_collector.py b/nodescraper/plugins/inband/bios/bios_collector.py index e0ab1011..e94242ef 100644 --- a/nodescraper/plugins/inband/bios/bios_collector.py +++ b/nodescraper/plugins/inband/bios/bios_collector.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.base import InBandDataCollector @@ -35,9 +36,11 @@ class BiosCollector(InBandDataCollector[BiosDataModel, None]): """Collect BIOS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = BiosDataModel CMD_WINDOWS = "wmic bios get SMBIOSBIOSVersion /Value" CMD = "sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'" + CMD_ESXI = "smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1" def collect_data( self, @@ -57,6 +60,11 @@ def collect_data( bios = [line for line in res.stdout.splitlines() if "SMBIOSBIOSVersion=" in line][ 0 ].split("=")[1] + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + match = re.search(r'Version:\s*"?([^"]+)"?', res.stdout) + bios = match.group(1).strip() if match else res.stdout.strip() else: res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 9b0dc295..9f579672 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -36,6 +36,7 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, None]): """Collect CPU and GPU count""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DeviceEnumerationDataModel CMD_GPU_COUNT_LINUX = ( @@ -55,6 +56,12 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, 'powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count"' ) + # ESXi busybox `lspci -d` dumps hex instead of filtering, so use esxcli. GPUs are + # counted by exact device ID (PF vs VF), anchored on "Device ID:" to avoid also + # matching "SubDevice ID:". + CMD_CPU_COUNT_ESXI = "esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'" + CMD_PCI_COUNT_ESXI = "esxcli hardware pci list | grep -E '^ *Device ID: 0x{device_id}' | wc -l" + def _warning( self, description: str, @@ -72,10 +79,20 @@ def _warning( priority=EventPriority.WARNING, ) + def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: + """Count PCI devices on ESXi whose Device ID matches ``device_id`` (as hex). + + A None id produces an unmatched pattern (count 0) so the caller still gets a + valid CommandArtifact to parse. + """ + hex_id = format(device_id, "x") if device_id is not None else "__unset__" + return self._run_sut_cmd(self.CMD_PCI_COUNT_ESXI.format(device_id=hex_id)) + def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: """ Read CPU and GPU count On Linux, use lscpu and lspci + On ESXi, use esxcli On Windows, use WMI and hyper-v cmdlets """ if self.system_info.os_family == OSFamily.LINUX: @@ -92,6 +109,17 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio # Collect lshw output lshw_res = self._run_sut_cmd(self.CMD_LSHW_LINUX, sudo=True, log_artifact=False) + elif self.system_info.os_family == OSFamily.ESXI: + cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_ESXI) + if self.system_info.devid_ep is None: + self._log_event( + category=EventCategory.PLATFORM, + description="devid_ep not set; cannot count GPUs/VFs on ESXi by device ID", + priority=EventPriority.WARNING, + ) + # PFs and (SR-IOV) VFs are distinguished by device ID on ESXi. + gpu_count_res = self._esxi_device_count(self.system_info.devid_ep) + vf_count_res = self._esxi_device_count(self.system_info.devid_ep_vf) else: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_WINDOWS) gpu_count_res = self._run_sut_cmd(self.CMD_GPU_COUNT_WINDOWS) diff --git a/nodescraper/plugins/inband/dimm/dimm_collector.py b/nodescraper/plugins/inband/dimm/dimm_collector.py index b6b91987..c9b4dd8f 100644 --- a/nodescraper/plugins/inband/dimm/dimm_collector.py +++ b/nodescraper/plugins/inband/dimm/dimm_collector.py @@ -38,12 +38,31 @@ class DimmCollector(InBandDataCollector[DimmDataModel, DimmCollectorArgs]): """Collect data on installed DIMMs""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DimmDataModel CMD_WINDOWS = "wmic memorychip get Capacity" CMD = """sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\\|None\\|Module" | grep Size' 2>/dev/null""" + CMD_ESXI = "smbiosDump | grep -A15 'Memory Device (Type 17)' | grep 'Size:'" CMD_DMIDECODE_FULL = "dmidecode" + def _parse_dmi_sizes(self, stdout: str) -> str: + """Build the DIMM summary from 'Size: ' lines (dmidecode on Linux, + smbiosDump on ESXi — both emit the same field format).""" + total = 0 + topology: dict[str, int] = {} + size = "" + dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") + for num, unit in dimm_size_pattern.findall(stdout): + size = unit + total += int(num) + key = num + unit + topology[key] = topology.get(key, 0) + 1 + if total == 0: + return "0 GB" + dimm_entries = [f"{v} x {k}" for k, v in topology.items()] + return f"{total}{size} @ {' '.join(dimm_entries)}" + def collect_data( self, args: Optional[DimmCollectorArgs] = None, @@ -70,6 +89,10 @@ def collect_data( dimm_str = f"{total / 1024 / 1024:.2f}GB @ " for capacity, count in capacities.items(): dimm_str += f"{count} x {capacity / 1024 / 1024:.2f}GB " + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + dimm_str = self._parse_dmi_sizes(res.stdout) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" @@ -96,29 +119,7 @@ def collect_data( res = self._run_sut_cmd(self.CMD, sudo=True) if res.exit_code == 0: - total = 0 - topology = {} - size = "" - dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") - matches = dimm_size_pattern.findall(res.stdout) - if matches: - for match in matches: - size = match[1] - total += int(match[0]) - key = match[0] + match[1] - if not topology.get(key, None): - topology[key] = 1 - else: - topology[key] += 1 - topology["total"] = total - topology["size"] = size - total_gb = topology.pop("total") - size = topology.pop("size") - if total_gb == 0: - dimm_str = "0 GB" - else: - dimm_entries = [f"{v} x {k}" for k, v in topology.items()] - dimm_str = f"{total_gb}{size} @ {' '.join(dimm_entries)}" + dimm_str = self._parse_dmi_sizes(res.stdout) if res.exit_code != 0: self._log_event( category=EventCategory.OS, diff --git a/nodescraper/plugins/inband/kernel/kernel_collector.py b/nodescraper/plugins/inband/kernel/kernel_collector.py index 6b188940..d93fc215 100644 --- a/nodescraper/plugins/inband/kernel/kernel_collector.py +++ b/nodescraper/plugins/inband/kernel/kernel_collector.py @@ -36,6 +36,7 @@ class KernelCollector(InBandDataCollector[KernelDataModel, None]): """Read kernel version""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = KernelDataModel CMD_WINDOWS = "wmic os get Version /Value" CMD = "sh -c 'uname -a'" @@ -88,6 +89,9 @@ def collect_data( "=" )[1] else: + # Non-Windows (Linux and ESXi). ESXi `uname -a` yields the release in the + # same field the Linux parser reads (verified: "9.1.0"); numa_balancing has + # no ESXi equivalent and its command fails gracefully, leaving None. res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: kernel_info = res.stdout diff --git a/nodescraper/plugins/inband/os/os_collector.py b/nodescraper/plugins/inband/os/os_collector.py index 42e435a9..2fc46ab5 100644 --- a/nodescraper/plugins/inband/os/os_collector.py +++ b/nodescraper/plugins/inband/os/os_collector.py @@ -36,10 +36,13 @@ class OsCollector(InBandDataCollector[OsDataModel, None]): """Collect OS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = OsDataModel CMD_VERSION_WINDOWS = "wmic os get Version /value" CMD_VERSION = "cat /etc/*release | grep VERSION_ID" + CMD_VERSION_ESXI = "esxcli system version get" CMD_WINDOWS = "wmic os get Caption /Value" + CMD_ESXI = "vmware -v" PRETTY_STR = "PRETTY_NAME" # noqa: N806 CMD = f"sh -c '( lsb_release -ds || (cat /etc/*release | grep {PRETTY_STR}) || uname -om ) 2>/dev/null | head -n1'" @@ -60,6 +63,22 @@ def collect_version(self) -> str: priority=EventPriority.ERROR, ) os_version = "" + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_VERSION_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + if "Version:" in line: + os_version = line.split(":", 1)[1].strip() + break + else: + os_version = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS version not found", + priority=EventPriority.ERROR, + ) + os_version = "" else: res = self._run_sut_cmd(self.CMD_VERSION) if res.exit_code == 0: @@ -86,6 +105,16 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[OsDataModel]]: res = self._run_sut_cmd(self.CMD_WINDOWS) if res.exit_code == 0: os_name = re.search(r"Caption=([\w\s]+)", res.stdout).group(1) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + os_name = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS name not found", + priority=EventPriority.ERROR, + ) else: res = self._run_sut_cmd(self.CMD) # search for PRETTY_NAME in res diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index e5373ebc..a97aefac 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -37,9 +37,11 @@ class StorageCollector(InBandDataCollector[StorageDataModel, None]): """Collect disk usage details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = StorageDataModel CMD_WINDOWS = """wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace""" CMD = """sh -c 'df -lH -B1 | grep -v 'boot''""" + CMD_ESXI = "esxcli storage filesystem list" def collect_data( self, args: Optional[StorageCollectorArgs] = None @@ -61,6 +63,24 @@ def collect_data( used=int(size) - int(free_space), percent=round((int(size) - int(free_space)) / int(size) * 100, 2), ) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + # esxcli columns (fixed order): [0] Mount Point [1] Volume Name + # [2] UUID [3] Mounted [4] Type [5] Size [6] Free + fields = re.split(r"\s{2,}", line.strip()) + if len(fields) >= 7 and fields[5].isdigit() and fields[6].isdigit(): + device_id = fields[0] + total_bytes = int(fields[5]) + free_bytes = int(fields[6]) + used_bytes = total_bytes - free_bytes + storage_data[device_id] = DeviceStorageData( + total=total_bytes, + free=free_bytes, + used=used_bytes, + percent=round(used_bytes / total_bytes * 100, 2) if total_bytes else 0.0, + ) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" From 83bcf0680469a386f06358ac6c2e1430e2ef7f70 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 03:58:15 +0000 Subject: [PATCH 2/9] Add ESXi support to PcieCollector ESXi busybox lspci lacks per-device (-s) and bus-path (-PP) options, so dump all extended config space once via `lspci -e`, split by BDF, and select the GPU/VF BDFs resolved from `esxcli hardware pci list` by SKU device ID (system_info.devid_ep/_vf). Extract a shared _cfg_space_from_hex parser (used by the Linux per-BDF path too). Upstream-bridge traversal is skipped on ESXi (GPU + VF only). Depends on the SystemInfo devid_ep/devid_ep_vf fields. --- .../plugins/inband/pcie/pcie_collector.py | 157 ++++++++++++++++-- 1 file changed, 145 insertions(+), 12 deletions(-) diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 624122ec..7259d085 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -80,7 +80,10 @@ class PcieCollector(InBandDataCollector[PcieDataModel, None]): """ - SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX, OSFamily.ESXI} + + # A bare BDF line that begins an esxcli/lspci device block, e.g. "0000:05:00.0". + _BDF_LINE = re.compile(r"^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]+", re.IGNORECASE) DATA_MODEL = PcieDataModel @@ -518,17 +521,8 @@ def get_cap_cfg( return cap_structure # type: ignore[return-value] - def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: - """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" - hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) - if hex_data_raw is None: - self._log_event( - category=EventCategory.IO, - description="Failed to get hex data for BDF.", - data={"bdf": bdf}, - priority=EventPriority.ERROR, - ) - return PcieCfgSpace() + def _cfg_space_from_hex(self, hex_data_raw: str, bdf: str) -> PcieCfgSpace: + """Parse a raw lspci hex dump (Linux ``-xxxx`` or ESXi ``-e``) into a PcieCfgSpace.""" hex_data: List[int] = self.parse_hex_dump(hex_data_raw) if len(hex_data) < 64: # Expect at least 256 bytes of data, for the first 256 bytes of the PCIe config space @@ -542,6 +536,19 @@ def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: cap_data, ecap_data = self.discover_capability_structure(hex_data) return self.get_pcie_cfg(hex_data, cap_data, ecap_data) + def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: + """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" + hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) + if hex_data_raw is None: + self._log_event( + category=EventCategory.IO, + description="Failed to get hex data for BDF.", + data={"bdf": bdf}, + priority=EventPriority.ERROR, + ) + return PcieCfgSpace() + return self._cfg_space_from_hex(hex_data_raw, bdf) + def get_pcie_cfg( self, config_data: List[int], @@ -595,6 +602,129 @@ def _log_pcie_artifacts( if data is not None: self.result.artifacts.append(TextFileArtifact(filename=name, contents=data)) + def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: + """Return (pf_bdfs, vf_bdfs) for the GPUs on an ESXi host via esxcli. + + ESXi busybox lspci has no device filter, so GPU/VF BDFs are resolved from + ``esxcli hardware pci list`` by matching the SKU's PF/VF device IDs + (system_info.devid_ep / devid_ep_vf). Each device block starts with a bare + BDF line followed by indented fields incl. "Device ID". + """ + pf_bdfs: List[str] = [] + vf_bdfs: List[str] = [] + pf_devid = ( + format(self.system_info.devid_ep, "x") + if self.system_info.devid_ep is not None + else "" + ) + vf_devid = ( + format(self.system_info.devid_ep_vf, "x") + if self.system_info.devid_ep_vf is not None + else "" + ) + if not pf_devid and not vf_devid: + return pf_bdfs, vf_bdfs + + out = self._run_os_cmd("esxcli hardware pci list", sudo=False) + if not out: + return pf_bdfs, vf_bdfs + + current_bdf: Optional[str] = None + for line in out.splitlines(): + stripped = line.strip() + if self._BDF_LINE.match(stripped) and ":" in stripped and " " not in stripped: + # Bare BDF header line (anchors the block). + current_bdf = stripped + elif current_bdf and stripped.lower().startswith("device id:"): + devid = stripped.split(":", 1)[1].strip().lower().removeprefix("0x") + if pf_devid and devid == pf_devid: + pf_bdfs.append(current_bdf) + elif vf_devid and devid == vf_devid: + vf_bdfs.append(current_bdf) + return pf_bdfs, vf_bdfs + + def _get_all_cfg_space_esxi(self) -> Dict[str, str]: + """Return {bdf: hex_dump_text} for every device from a single ``lspci -e``. + + ESXi has no per-device dump; ``lspci -e`` emits the full extended (4096-byte) + config space for all devices in one blob. Each device section starts with a + header line " " followed by "NN: .." hex lines. + """ + blob = self._run_os_cmd("lspci -e", sudo=False) + if not blob: + return {} + self.result.artifacts.append(TextFileArtifact(filename="lspci_e.txt", contents=blob)) + sections: Dict[str, List[str]] = {} + current_bdf: Optional[str] = None + for line in blob.splitlines(): + header = self._BDF_LINE.match(line) + if header and " " in line: + # Device header line: " ". + current_bdf = line.split(" ", 1)[0] + sections[current_bdf] = [] + elif current_bdf is not None: + sections[current_bdf].append(line) + return {bdf: "\n".join(lines) for bdf, lines in sections.items()} + + def _get_pcie_data_esxi(self) -> Optional[PcieDataModel]: + """Collect GPU + VF PCIe config space on ESXi. + + ESXi busybox lspci lacks ``-s`` (per-device) and ``-PP`` (bus-path), so dump + all extended config space once via ``lspci -e``, split it by BDF, and select the + GPU/VF BDFs resolved from esxcli. Upstream-bridge traversal is not available on + ESXi and is intentionally skipped (GPU + VF only). + """ + pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi() + if not pf_bdfs and not vf_bdfs: + self._log_event( + category=EventCategory.IO, + description="No GPU/VF BDFs found on ESXi host for this SKU.", + data={ + "devid_ep": self.system_info.devid_ep, + "devid_ep_vf": self.system_info.devid_ep_vf, + }, + priority=EventPriority.WARNING, + ) + return None + + cfg_by_bdf = self._get_all_cfg_space_esxi() + if not cfg_by_bdf: + self.result.status = ExecutionStatus.ERROR + return None + + self._log_event( + category=EventCategory.IO, + description=( + "Upstream-bridge PCIe collection is not supported on ESXi; " + "collecting GPU + VF only." + ), + priority=EventPriority.INFO, + ) + + try: + pcie_cfg_dict: Dict[str, PcieCfgSpace] = {} + for bdf in pf_bdfs: + if bdf in cfg_by_bdf: + pcie_cfg_dict[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + vf_pcie_cfg_data: Dict[str, PcieCfgSpace] = {} + for bdf in vf_bdfs: + if bdf in cfg_by_bdf: + vf_pcie_cfg_data[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + pcie_data = PcieDataModel( + pcie_cfg_space=pcie_cfg_dict, + vf_pcie_cfg_space=vf_pcie_cfg_data, + ) + except ValidationError as e: + self._log_event( + category=EventCategory.OS, + description="Failed to build model for PCIe data", + data=get_exception_details(e), + priority=EventPriority.ERROR, + ) + self.result.status = ExecutionStatus.ERROR + return None + return pcie_data + def _get_pcie_data( self, upstream_steps_to_collect: Optional[int] = None ) -> Optional[PcieDataModel]: @@ -605,6 +735,9 @@ def _get_pcie_data( Optional[PcieDataModel] The data in a PcieDataModel object or None on failure """ + if self.system_info.os_family == OSFamily.ESXI: + return self._get_pcie_data_esxi() + minimum_system_interaction_level_required_for_sudo = SystemInteractionLevel.INTERACTIVE try: From 5bd26fe3dc0d0fb968dae83da6c9eab2609c55ee Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:08:00 +0000 Subject: [PATCH 3/9] Add ESXi support to DmesgCollector ESXi has no dmesg ring buffer; read the kernel log from /var/log/vmkernel.log (and vmkernel.[.gz] rotations) instead of `dmesg`. Validated on ESXi: reads vmkernel.log; Linux path unchanged. --- .../plugins/inband/dmesg/dmesg_collector.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index c280d7d2..0d2894c6 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -38,24 +38,33 @@ class DmesgCollector(InBandDataCollector[DmesgData, DmesgCollectorArgs]): """Read dmesg log""" - SUPPORTED_OS_FAMILY = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY = {OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DmesgData CMD = "dmesg --time-format iso -x" + # ESXi has no dmesg ring buffer; the kernel log is the vmkernel.log file. + CMD_ESXI = "cat /var/log/vmkernel.log" CMD_LOGS = ( r"ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true" ) + # ESXi rotates vmkernel.log to vmkernel. / vmkernel..gz. + CMD_LOGS_ESXI = ( + r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" + ) def _collect_dmesg_rotations(self): - """Collect dmesg logs""" - list_res = self._run_sut_cmd(self.CMD_LOGS, sudo=True) + """Collect dmesg (Linux) / vmkernel.log (ESXi) rotated logs""" + is_esxi = self.system_info.os_family == OSFamily.ESXI + log_label = "vmkernel" if is_esxi else "dmesg" + cmd_logs = self.CMD_LOGS_ESXI if is_esxi else self.CMD_LOGS + list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: self._log_event( category=EventCategory.OS, - description="No /var/log/dmesg files found (including rotations).", + description=f"No rotated {log_label} log files found.", data={"list_exit_code": list_res.exit_code}, priority=EventPriority.WARNING, ) @@ -68,7 +77,7 @@ def _collect_dmesg_rotations(self): cmd = f"gzip -dc {qp} 2>/dev/null || zcat {qp} 2>/dev/null" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -84,7 +93,7 @@ def _collect_dmesg_rotations(self): cmd = f"cat {qp}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -121,8 +130,10 @@ def _get_dmesg_content(self) -> str: str: dmesg output """ - self.logger.info("Running dmesg command on system") - res = self._run_sut_cmd(self.CMD, sudo=True, log_artifact=False) + is_esxi = self.system_info.os_family == OSFamily.ESXI + cmd = self.CMD_ESXI if is_esxi else self.CMD + self.logger.info("Reading kernel log from system") + res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code != 0: self._log_event( category=EventCategory.OS, From fb75f5b30b616fb7deff79966f8bad1df47dd0e8 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:25:22 +0000 Subject: [PATCH 4/9] Add ESXi support to DmesgAnalyzer Make the dmesg analyzer format-aware so it handles ESXi vmkernel.log as well as Linux dmesg: - Extract ESXi ISO8601 dot-ms/Z timestamps (e.g. 2026-08-20T09:35:58.380Z) via ESXI_TIMESTAMP_PATTERN, set on __init__ so event grouping and date-range filtering both use it; Linux keeps the base comma-form pattern. - filter_dmesg is now an instance method and reuses the base timestamp extractor, so a single code path honors whichever pattern is active. - Add ESXi mxGPU (gim/amdgpuv) RAS ERROR_REGEX entries (Block-capitalized correctable/uncorrectable, ECC Fatal Error, Whole GPU reset); these are inert on Linux logs. - Unknown-error detection keys off the driver-internal severity in the message body ("gim/amdgpuv error/warn") on ESXi, where the vmkernel -ALERT/-INFO tokens are unreliable; Linux keeps the "kern :err:" form. Validated on real ESXi vmkernel.log (7.2 MB) and Linux dmesg (no regression), plus synthetic RAS lines confirming per-OS phrasing is discriminated correctly. --- .../plugins/inband/dmesg/dmesg_analyzer.py | 79 ++++++++++++++++--- 1 file changed, 68 insertions(+), 11 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 5ae53f77..40e8cdef 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -30,7 +30,7 @@ from nodescraper.base.match_ignore import parse_ignore_match_rules from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer from nodescraper.connection.inband import TextFileArtifact -from nodescraper.enums import EventCategory, EventPriority +from nodescraper.enums import EventCategory, EventPriority, OSFamily from nodescraper.models import Event, TaskResult from .analyzer_args import DmesgAnalyzerArgs @@ -47,10 +47,26 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): - """Check dmesg for errors""" + """Check dmesg (Linux) or vmkernel.log (ESXi) for errors""" DATA_MODEL = DmesgData + # ESXi vmkernel.log timestamp, e.g. "2026-08-05T19:53:35.178Z" (ISO8601 dot-ms + Z). + # Linux uses the base RegexAnalyzer.TIMESTAMP_PATTERN (comma-form). + ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile( + r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" + ) + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # On ESXi, extract vmkernel.log timestamps so event grouping and date-range + # filtering both work; Linux keeps the base comma-form pattern. + if self._is_esxi(): + self.TIMESTAMP_PATTERN = self.ESXI_TIMESTAMP_PATTERN + + def _is_esxi(self) -> bool: + return self.system_info.os_family == OSFamily.ESXI + ERROR_REGEX: list[ErrorRegex] = [ ErrorRegex( regex=re.compile(r"(?:oom_kill_process.*)|(?:Out of memory.*)"), @@ -273,6 +289,30 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): message="RAS Deferred Error", event_category=EventCategory.RAS, ), + # ESXi mxGPU (gim/amdgpuv) RAS phrasing differs from Linux: the block name is + # capitalized ("... detected in MMHUB Block."), there is no "in total", and no + # "kern :err:" prefix. These match the ESXi host-driver forms and are inert on + # Linux logs (which use the lowercase "in total in block" phrasing above). + ErrorRegex( + regex=re.compile(r"(\d+ new uncorrectable hardware errors detected in \w+ Block.*)"), + message="RAS Uncorrectable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(\d+ new correctable hardware errors detected in \w+ Block.*)"), + message="RAS Correctable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(GPU detected ECC Fatal Error\.)"), + message="RAS ECC Fatal Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(Issuing Whole GPU reset\.)"), + message="GPU Reset", + event_category=EventCategory.RAS, + ), ErrorRegex( regex=re.compile( r"((?:\[Hardware Error\]:\s+)?event severity: corrected.*)" @@ -463,14 +503,13 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): ), ] - @classmethod def filter_dmesg( - cls, + self, dmesg_content: str, analysis_range_start: Optional[datetime.datetime] = None, analysis_range_end: Optional[datetime.datetime] = None, ) -> str: - """Filter a dmesg log by date + """Filter a dmesg (Linux) or vmkernel.log (ESXi) log by date Args: dmesg_content (str): unfiltered dmesg log @@ -482,9 +521,16 @@ def filter_dmesg( filtered_dmesg = "" found_start = False if analysis_range_start else True for line in dmesg_content.splitlines(): - date = re.search(r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)", line) - if date is not None: - date = datetime.datetime.fromisoformat(f"{date.group(1)}.{date.group(2)}") + # Reuse the base extractor so the active TIMESTAMP_PATTERN (ESXi dot-Z form + # when on ESXi, else Linux comma-form) is honored in exactly one place. + date_str = self._extract_timestamp_from_match_position(line, 0) + if date_str is not None: + # Linux uses a comma before fractional seconds; normalize to "." so + # fromisoformat() accepts it (no-op for the ESXi "...Z" form). + try: + date = datetime.datetime.fromisoformat(date_str.replace(",", ".")) + except ValueError: + continue # show date in UTC now date = date.astimezone(datetime.timezone.utc) if analysis_range_start and not found_start and date >= analysis_range_start: @@ -743,11 +789,22 @@ def analyze_data( self.result.events += known_err_events if args.check_unknown_dmesg_errors: + if self._is_esxi(): + # ESXi vmkernel severity tokens are unreliable (-ALERT is used for benign + # boot notices; -ERROR/-CRIT are never emitted). The reliable error signal + # is the driver-internal severity in the message body: "gim error/warning", + # "amdgpuv error/warning", or the bracket form "[amdgpuv warn]". + unknown_error_regex = re.compile( + r"(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning) [^:]*:\s*(.*)" + r"|\[(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning)\]:?\s*(.*)" + ) + else: + unknown_error_regex = re.compile( + r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" + ) unknown_dmesg_error_regexes = [ ErrorRegex( - regex=re.compile( - r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" - ), + regex=unknown_error_regex, message="Unknown dmesg error", event_category=EventCategory.UNKNOWN, event_priority=EventPriority.WARNING, From 3416c9eb54383670dd22b896a15222a99b481ff0 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:34:55 +0000 Subject: [PATCH 5/9] Fix dmesg ESXi unit-test regressions - filter_dmesg: keep it a classmethod (public API used as DmesgAnalyzer.filter_dmesg(content, ...) in tests). Recognize both Linux comma-form and ESXi dot-ms/Z timestamps via a combined pattern instead of the instance TIMESTAMP_PATTERN; normalize the trailing Z so fromisoformat accepts it on Python < 3.11. - Collector: restore the exact Linux "No /var/log/dmesg files found (including rotations)." wording for the no-rotations event and add an ESXi-specific vmkernel.log variant, rather than a generic reword. Restores test_dmesg_filter and test_collect_rotations_no_files; full dmesg collector+analyzer suite (57 tests) green. --- .../plugins/inband/dmesg/dmesg_analyzer.py | 32 ++++++++++++------- .../plugins/inband/dmesg/dmesg_collector.py | 6 +++- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 40e8cdef..56bf6934 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -503,8 +503,18 @@ def _is_esxi(self) -> bool: ), ] + # Date-range filtering must recognize both Linux dmesg comma-form timestamps + # (2024-10-01T05:00:00,000000-05:00) and ESXi vmkernel.log dot-ms/Z timestamps + # (2026-08-20T09:35:58.380Z). filter_dmesg stays a classmethod (public API), so it + # carries its own combined pattern rather than the instance TIMESTAMP_PATTERN. + _FILTER_TIMESTAMP_PATTERN: re.Pattern = re.compile( + r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)" + r"|(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" + ) + + @classmethod def filter_dmesg( - self, + cls, dmesg_content: str, analysis_range_start: Optional[datetime.datetime] = None, analysis_range_end: Optional[datetime.datetime] = None, @@ -521,16 +531,16 @@ def filter_dmesg( filtered_dmesg = "" found_start = False if analysis_range_start else True for line in dmesg_content.splitlines(): - # Reuse the base extractor so the active TIMESTAMP_PATTERN (ESXi dot-Z form - # when on ESXi, else Linux comma-form) is honored in exactly one place. - date_str = self._extract_timestamp_from_match_position(line, 0) - if date_str is not None: - # Linux uses a comma before fractional seconds; normalize to "." so - # fromisoformat() accepts it (no-op for the ESXi "...Z" form). - try: - date = datetime.datetime.fromisoformat(date_str.replace(",", ".")) - except ValueError: - continue + match = cls._FILTER_TIMESTAMP_PATTERN.search(line) + if match is not None: + if match.group(1) is not None: + # Linux comma-form: swap the comma for a dot so fromisoformat accepts it + iso = f"{match.group(1)}.{match.group(2)}" + else: + # ESXi dot-Z form: normalize the trailing Z so fromisoformat accepts it + # on Python < 3.11 as well + iso = match.group(3).replace("Z", "+00:00") + date = datetime.datetime.fromisoformat(iso) # show date in UTC now date = date.astimezone(datetime.timezone.utc) if analysis_range_start and not found_start and date >= analysis_range_start: diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index 0d2894c6..4c20420a 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -62,9 +62,13 @@ def _collect_dmesg_rotations(self): list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: + if is_esxi: + description = "No /var/log/vmkernel.log files found (including rotations)." + else: + description = "No /var/log/dmesg files found (including rotations)." self._log_event( category=EventCategory.OS, - description=f"No rotated {log_label} log files found.", + description=description, data={"list_exit_code": list_res.exit_code}, priority=EventPriority.WARNING, ) From ed0bdec8adf1b7f74834a62aff4e1cb9b35a6ce1 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:42:58 +0000 Subject: [PATCH 6/9] Satisfy black/ruff pre-commit on ESXi collectors Pre-commit black (line-length 100) flagged formatting in the ESXi branches. Expand the branch-selection ternaries to explicit if/else (pcie devid resolve, storage percent, dmesg cmd/log-label selection) and let black normalize the two long single-line constants (dmesg CMD_LOGS_ESXI, analyzer ESXI_TIMESTAMP_PATTERN). No behavior change; black --check and ruff clean, dmesg+storage suites green. --- .../plugins/inband/dmesg/dmesg_analyzer.py | 4 +--- .../plugins/inband/dmesg/dmesg_collector.py | 17 +++++++++++------ .../plugins/inband/pcie/pcie_collector.py | 18 ++++++++---------- .../inband/storage/storage_collector.py | 6 +++++- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 56bf6934..5bf93c36 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -53,9 +53,7 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): # ESXi vmkernel.log timestamp, e.g. "2026-08-05T19:53:35.178Z" (ISO8601 dot-ms + Z). # Linux uses the base RegexAnalyzer.TIMESTAMP_PATTERN (comma-form). - ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile( - r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" - ) + ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)") def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index 4c20420a..4fcbc4b7 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -50,15 +50,17 @@ class DmesgCollector(InBandDataCollector[DmesgData, DmesgCollectorArgs]): r"ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true" ) # ESXi rotates vmkernel.log to vmkernel. / vmkernel..gz. - CMD_LOGS_ESXI = ( - r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" - ) + CMD_LOGS_ESXI = r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" def _collect_dmesg_rotations(self): """Collect dmesg (Linux) / vmkernel.log (ESXi) rotated logs""" is_esxi = self.system_info.os_family == OSFamily.ESXI - log_label = "vmkernel" if is_esxi else "dmesg" - cmd_logs = self.CMD_LOGS_ESXI if is_esxi else self.CMD_LOGS + if is_esxi: + log_label = "vmkernel" + cmd_logs = self.CMD_LOGS_ESXI + else: + log_label = "dmesg" + cmd_logs = self.CMD_LOGS list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: @@ -135,7 +137,10 @@ def _get_dmesg_content(self) -> str: """ is_esxi = self.system_info.os_family == OSFamily.ESXI - cmd = self.CMD_ESXI if is_esxi else self.CMD + if is_esxi: + cmd = self.CMD_ESXI + else: + cmd = self.CMD self.logger.info("Reading kernel log from system") res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code != 0: diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 7259d085..2f46c19f 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -612,16 +612,14 @@ def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: """ pf_bdfs: List[str] = [] vf_bdfs: List[str] = [] - pf_devid = ( - format(self.system_info.devid_ep, "x") - if self.system_info.devid_ep is not None - else "" - ) - vf_devid = ( - format(self.system_info.devid_ep_vf, "x") - if self.system_info.devid_ep_vf is not None - else "" - ) + if self.system_info.devid_ep is not None: + pf_devid = format(self.system_info.devid_ep, "x") + else: + pf_devid = "" + if self.system_info.devid_ep_vf is not None: + vf_devid = format(self.system_info.devid_ep_vf, "x") + else: + vf_devid = "" if not pf_devid and not vf_devid: return pf_bdfs, vf_bdfs diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index a97aefac..bb9d4c32 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -75,11 +75,15 @@ def collect_data( total_bytes = int(fields[5]) free_bytes = int(fields[6]) used_bytes = total_bytes - free_bytes + if total_bytes: + percent = round(used_bytes / total_bytes * 100, 2) + else: + percent = 0.0 storage_data[device_id] = DeviceStorageData( total=total_bytes, free=free_bytes, used=used_bytes, - percent=round(used_bytes / total_bytes * 100, 2) if total_bytes else 0.0, + percent=percent, ) else: if args.skip_sudo: From 6e9d3d26be0b3eebe450b3a86ab718e53df9bbac Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Fri, 11 Sep 2026 04:49:48 +0000 Subject: [PATCH 7/9] Fix mypy name collision in storage ESXi branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hoisting the percent computation reused the name "percent", which the Linux branch later binds to a str from split() before re.sub()/float() — mypy flagged the float-vs-str conflict. Rename the ESXi-branch value to usage_percent. --- nodescraper/plugins/inband/storage/storage_collector.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index bb9d4c32..7b096b52 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -76,14 +76,14 @@ def collect_data( free_bytes = int(fields[6]) used_bytes = total_bytes - free_bytes if total_bytes: - percent = round(used_bytes / total_bytes * 100, 2) + usage_percent = round(used_bytes / total_bytes * 100, 2) else: - percent = 0.0 + usage_percent = 0.0 storage_data[device_id] = DeviceStorageData( total=total_bytes, free=free_bytes, used=used_bytes, - percent=percent, + percent=usage_percent, ) else: if args.skip_sudo: From 545661e195d62a3500879c51f59b94a9cb84ed31 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Mon, 14 Sep 2026 18:26:24 +0000 Subject: [PATCH 8/9] Address review: robust device-ID matching + safe count parsing (ESXi) Per review feedback on the ESXi device-ID handling: - pcie: compare the esxcli "Device ID" to the expected PF/VF id by integer value instead of an exact lowercase-string match, so uppercase ("0x744C") and zero-padded ("0x0000744c") ids are matched. - device_enumeration: make the PCI-count grep case-insensitive and zero-pad tolerant ("0x0*"), with a trailing [^0-9a-f]/$ guard so a shorter id does not match a longer one (744c vs 744cd). - device_enumeration: parse the CPU/GPU/VF counts defensively (guard non-zero exit and non-numeric stdout) instead of int()-ing command output directly, so an unexpected esxcli/awk result warns rather than raising. Validated on ESXi 9.1 (8 GPUs matched; counts parsed) and Linux (no regression); padded/uppercase ids confirmed against busybox grep and the int compare. --- .../device_enumeration_collector.py | 59 +++++++++++++------ .../plugins/inband/pcie/pcie_collector.py | 24 ++++---- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 9f579672..48853473 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -57,10 +57,15 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, ) # ESXi busybox `lspci -d` dumps hex instead of filtering, so use esxcli. GPUs are - # counted by exact device ID (PF vs VF), anchored on "Device ID:" to avoid also - # matching "SubDevice ID:". + # counted by device ID (PF vs VF), anchored on "Device ID:" to avoid also matching + # "SubDevice ID:". The match is case-insensitive and tolerates zero-padding + # ("0x744C" / "0x0000744c"); the trailing [^0-9a-f]/$ guard stops a shorter ID from + # matching a longer one (e.g. 744c vs 744cd). CMD_CPU_COUNT_ESXI = "esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'" - CMD_PCI_COUNT_ESXI = "esxcli hardware pci list | grep -E '^ *Device ID: 0x{device_id}' | wc -l" + CMD_PCI_COUNT_ESXI = ( + "esxcli hardware pci list | " + "grep -iE '^ *Device ID: 0x0*{device_id}([^0-9a-f]|$)' | wc -l" + ) def _warning( self, @@ -79,6 +84,27 @@ def _warning( priority=EventPriority.WARNING, ) + def _parse_count( + self, + res: CommandArtifact, + description: str, + category: EventCategory = EventCategory.PLATFORM, + ) -> Optional[int]: + """Parse a numeric count from command stdout, warning (not raising) on a + non-zero exit or non-numeric output (e.g. an unexpected esxcli/awk result).""" + if res.exit_code != 0: + self._warning(description=description, command=res, category=category) + return None + text = (res.stdout or "").strip() + if not text.isdigit(): + self._warning( + description=f"{description} (non-numeric output: {text!r})", + command=res, + category=category, + ) + return None + return int(text) + def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: """Count PCI devices on ESXi whose Device ID matches ``device_id`` (as hex). @@ -149,24 +175,19 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio else: self._warning(description="Cannot collect lscpu output", command=lscpu_res) else: - if cpu_count_res.exit_code == 0: - device_enum.cpu_count = int(cpu_count_res.stdout) - else: - self._warning(description="Cannot determine CPU count", command=cpu_count_res) + cpu_count = self._parse_count(cpu_count_res, "Cannot determine CPU count") + if cpu_count is not None: + device_enum.cpu_count = cpu_count - if gpu_count_res.exit_code == 0: - device_enum.gpu_count = int(gpu_count_res.stdout) - else: - self._warning(description="Cannot determine GPU count", command=gpu_count_res) + gpu_count = self._parse_count(gpu_count_res, "Cannot determine GPU count") + if gpu_count is not None: + device_enum.gpu_count = gpu_count - if vf_count_res.exit_code == 0: - device_enum.vf_count = int(vf_count_res.stdout) - else: - self._warning( - description="Cannot determine VF count", - command=vf_count_res, - category=EventCategory.SW_DRIVER, - ) + vf_count = self._parse_count( + vf_count_res, "Cannot determine VF count", category=EventCategory.SW_DRIVER + ) + if vf_count is not None: + device_enum.vf_count = vf_count # Collect lshw output on Linux if self.system_info.os_family == OSFamily.LINUX: diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 2f46c19f..f690aa55 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -612,15 +612,9 @@ def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: """ pf_bdfs: List[str] = [] vf_bdfs: List[str] = [] - if self.system_info.devid_ep is not None: - pf_devid = format(self.system_info.devid_ep, "x") - else: - pf_devid = "" - if self.system_info.devid_ep_vf is not None: - vf_devid = format(self.system_info.devid_ep_vf, "x") - else: - vf_devid = "" - if not pf_devid and not vf_devid: + pf_devid = self.system_info.devid_ep + vf_devid = self.system_info.devid_ep_vf + if pf_devid is None and vf_devid is None: return pf_bdfs, vf_bdfs out = self._run_os_cmd("esxcli hardware pci list", sudo=False) @@ -634,10 +628,16 @@ def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: # Bare BDF header line (anchors the block). current_bdf = stripped elif current_bdf and stripped.lower().startswith("device id:"): - devid = stripped.split(":", 1)[1].strip().lower().removeprefix("0x") - if pf_devid and devid == pf_devid: + # Compare by integer value so case ("0x744C") and zero-padding + # ("0x0000744c") both match the expected device ID. + raw = stripped.split(":", 1)[1].strip() + try: + devid = int(raw, 16) + except ValueError: + continue + if pf_devid is not None and devid == pf_devid: pf_bdfs.append(current_bdf) - elif vf_devid and devid == vf_devid: + elif vf_devid is not None and devid == vf_devid: vf_bdfs.append(current_bdf) return pf_bdfs, vf_bdfs From 053b734ee24dde63abd6038b139fe05682bd6517 Mon Sep 17 00:00:00 2001 From: Shamee Mahmud Date: Tue, 15 Sep 2026 03:24:56 +0000 Subject: [PATCH 9/9] Address review: move devid_ep/devid_ep_vf to collector args The expected GPU PF/VF PCI device IDs were SystemInfo fields that nothing populated upstream, so the ESXi GPU/VF resolution in device_enumeration and pcie was always inert. Move them to per-collector args (DeviceEnumerationCollectorArgs / PcieCollectorArgs), user-populated, matching how amd-smi takes them via args; read from args instead of SystemInfo and drop the unused SystemInfo fields. Validated on ESXi (8 GPU PF BDFs / gpu_count 8 via args) and Linux (no regression). --- nodescraper/models/systeminfo.py | 2 - .../device_enumeration/collector_args.py | 23 +++++++++ .../device_enumeration_collector.py | 19 ++++--- .../device_enumeration_plugin.py | 9 +++- .../plugins/inband/pcie/collector_args.py | 24 +++++++++ .../plugins/inband/pcie/pcie_collector.py | 49 +++++++++++-------- .../plugins/inband/pcie/pcie_plugin.py | 5 +- 7 files changed, 101 insertions(+), 30 deletions(-) create mode 100644 nodescraper/plugins/inband/device_enumeration/collector_args.py create mode 100644 nodescraper/plugins/inband/pcie/collector_args.py diff --git a/nodescraper/models/systeminfo.py b/nodescraper/models/systeminfo.py index d593a9a0..d91a68cf 100644 --- a/nodescraper/models/systeminfo.py +++ b/nodescraper/models/systeminfo.py @@ -44,5 +44,3 @@ class SystemInfo(BaseModel): metadata: Optional[dict] = Field(default_factory=dict) location: Optional[SystemLocation] = SystemLocation.LOCAL vendorid_ep: int = 0x1002 - devid_ep: Optional[int] = None - devid_ep_vf: Optional[int] = None diff --git a/nodescraper/plugins/inband/device_enumeration/collector_args.py b/nodescraper/plugins/inband/device_enumeration/collector_args.py new file mode 100644 index 00000000..c2d7b35c --- /dev/null +++ b/nodescraper/plugins/inband/device_enumeration/collector_args.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class DeviceEnumerationCollectorArgs(CollectorArgs): + """Collector args for device enumeration. + + On ESXi, GPUs and their SR-IOV VFs are counted by PCI device ID (esxcli has no + device filter). Provide the expected PF/VF device IDs here; when unset the ESXi + GPU/VF counts are skipped. The caller populates these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi device counting.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF counting.", + ) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 48853473..78c54ad0 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -30,10 +30,13 @@ from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult +from .collector_args import DeviceEnumerationCollectorArgs from .deviceenumdata import DeviceEnumerationDataModel -class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, None]): +class DeviceEnumerationCollector( + InBandDataCollector[DeviceEnumerationDataModel, DeviceEnumerationCollectorArgs] +): """Collect CPU and GPU count""" SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} @@ -114,13 +117,17 @@ def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: hex_id = format(device_id, "x") if device_id is not None else "__unset__" return self._run_sut_cmd(self.CMD_PCI_COUNT_ESXI.format(device_id=hex_id)) - def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: + def collect_data( + self, args: Optional[DeviceEnumerationCollectorArgs] = None + ) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: """ Read CPU and GPU count On Linux, use lscpu and lspci - On ESXi, use esxcli + On ESXi, use esxcli (GPU/VF counts need args.devid_ep / devid_ep_vf) On Windows, use WMI and hyper-v cmdlets """ + if args is None: + args = DeviceEnumerationCollectorArgs() if self.system_info.os_family == OSFamily.LINUX: lscpu_res = self._run_sut_cmd(self.CMD_LSCPU_LINUX, log_artifact=False) @@ -137,15 +144,15 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio lshw_res = self._run_sut_cmd(self.CMD_LSHW_LINUX, sudo=True, log_artifact=False) elif self.system_info.os_family == OSFamily.ESXI: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_ESXI) - if self.system_info.devid_ep is None: + if args.devid_ep is None: self._log_event( category=EventCategory.PLATFORM, description="devid_ep not set; cannot count GPUs/VFs on ESXi by device ID", priority=EventPriority.WARNING, ) # PFs and (SR-IOV) VFs are distinguished by device ID on ESXi. - gpu_count_res = self._esxi_device_count(self.system_info.devid_ep) - vf_count_res = self._esxi_device_count(self.system_info.devid_ep_vf) + gpu_count_res = self._esxi_device_count(args.devid_ep) + vf_count_res = self._esxi_device_count(args.devid_ep_vf) else: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_WINDOWS) gpu_count_res = self._run_sut_cmd(self.CMD_GPU_COUNT_WINDOWS) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py index baf2aa2d..cff51210 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py @@ -26,13 +26,18 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import DeviceEnumerationAnalyzerArgs +from .collector_args import DeviceEnumerationCollectorArgs from .device_enumeration_analyzer import DeviceEnumerationAnalyzer from .device_enumeration_collector import DeviceEnumerationCollector from .deviceenumdata import DeviceEnumerationDataModel class DeviceEnumerationPlugin( - InBandDataPlugin[DeviceEnumerationDataModel, None, DeviceEnumerationAnalyzerArgs] + InBandDataPlugin[ + DeviceEnumerationDataModel, + DeviceEnumerationCollectorArgs, + DeviceEnumerationAnalyzerArgs, + ] ): """Plugin for collection and analysis of BIOS data""" @@ -40,6 +45,8 @@ class DeviceEnumerationPlugin( COLLECTOR = DeviceEnumerationCollector + COLLECTOR_ARGS = DeviceEnumerationCollectorArgs + ANALYZER = DeviceEnumerationAnalyzer ANALYZER_ARGS = DeviceEnumerationAnalyzerArgs diff --git a/nodescraper/plugins/inband/pcie/collector_args.py b/nodescraper/plugins/inband/pcie/collector_args.py new file mode 100644 index 00000000..21260fe2 --- /dev/null +++ b/nodescraper/plugins/inband/pcie/collector_args.py @@ -0,0 +1,24 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class PcieCollectorArgs(CollectorArgs): + """Collector args for PCIe data. + + On ESXi, GPU/VF BDFs are resolved from ``esxcli hardware pci list`` by matching + the expected PF/VF PCI device IDs (esxcli has no device filter). Provide them + here; when both are unset no ESXi GPU BDFs are resolved. The caller populates + these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi BDF resolution.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF BDF resolution.", + ) diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index f690aa55..1085bf2a 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -41,6 +41,7 @@ from nodescraper.models import TaskResult from nodescraper.utils import get_all_subclasses, get_exception_details +from .collector_args import PcieCollectorArgs from .pcie_data import ( MAX_CAP_ID, MAX_ECAP_ID, @@ -54,7 +55,7 @@ ) -class PcieCollector(InBandDataCollector[PcieDataModel, None]): +class PcieCollector(InBandDataCollector[PcieDataModel, PcieCollectorArgs]): """class for collection of PCIe data only supports Linux OS type. This class collects the PCIE config space using the lspci hex dump and then parses the hex dump to get the @@ -602,18 +603,18 @@ def _log_pcie_artifacts( if data is not None: self.result.artifacts.append(TextFileArtifact(filename=name, contents=data)) - def _get_gpu_vf_bdfs_esxi(self) -> Tuple[List[str], List[str]]: + def _get_gpu_vf_bdfs_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Tuple[List[str], List[str]]: """Return (pf_bdfs, vf_bdfs) for the GPUs on an ESXi host via esxcli. ESXi busybox lspci has no device filter, so GPU/VF BDFs are resolved from - ``esxcli hardware pci list`` by matching the SKU's PF/VF device IDs - (system_info.devid_ep / devid_ep_vf). Each device block starts with a bare - BDF line followed by indented fields incl. "Device ID". + ``esxcli hardware pci list`` by matching the expected PF/VF device IDs + (``pf_devid`` / ``vf_devid``, from the collector args). Each device block + starts with a bare BDF line followed by indented fields incl. "Device ID". """ pf_bdfs: List[str] = [] vf_bdfs: List[str] = [] - pf_devid = self.system_info.devid_ep - vf_devid = self.system_info.devid_ep_vf if pf_devid is None and vf_devid is None: return pf_bdfs, vf_bdfs @@ -664,23 +665,23 @@ def _get_all_cfg_space_esxi(self) -> Dict[str, str]: sections[current_bdf].append(line) return {bdf: "\n".join(lines) for bdf, lines in sections.items()} - def _get_pcie_data_esxi(self) -> Optional[PcieDataModel]: + def _get_pcie_data_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Optional[PcieDataModel]: """Collect GPU + VF PCIe config space on ESXi. ESXi busybox lspci lacks ``-s`` (per-device) and ``-PP`` (bus-path), so dump all extended config space once via ``lspci -e``, split it by BDF, and select the - GPU/VF BDFs resolved from esxcli. Upstream-bridge traversal is not available on - ESXi and is intentionally skipped (GPU + VF only). + GPU/VF BDFs resolved from esxcli (matching ``pf_devid`` / ``vf_devid`` from the + collector args). Upstream-bridge traversal is not available on ESXi and is + intentionally skipped (GPU + VF only). """ - pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi() + pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi(pf_devid, vf_devid) if not pf_bdfs and not vf_bdfs: self._log_event( category=EventCategory.IO, description="No GPU/VF BDFs found on ESXi host for this SKU.", - data={ - "devid_ep": self.system_info.devid_ep, - "devid_ep_vf": self.system_info.devid_ep_vf, - }, + data={"devid_ep": pf_devid, "devid_ep_vf": vf_devid}, priority=EventPriority.WARNING, ) return None @@ -724,7 +725,10 @@ def _get_pcie_data_esxi(self) -> Optional[PcieDataModel]: return pcie_data def _get_pcie_data( - self, upstream_steps_to_collect: Optional[int] = None + self, + upstream_steps_to_collect: Optional[int] = None, + pf_devid: Optional[int] = None, + vf_devid: Optional[int] = None, ) -> Optional[PcieDataModel]: """Will return all PCIe data in a PcieDataModel object. @@ -734,7 +738,7 @@ def _get_pcie_data( The data in a PcieDataModel object or None on failure """ if self.system_info.os_family == OSFamily.ESXI: - return self._get_pcie_data_esxi() + return self._get_pcie_data_esxi(pf_devid, vf_devid) minimum_system_interaction_level_required_for_sudo = SystemInteractionLevel.INTERACTIVE @@ -833,19 +837,24 @@ def discover_capability_structure( return cap, ecap def collect_data( - self, args=None, upstream_steps_to_collect: Optional[int] = None, **kwargs + self, + args: Optional[PcieCollectorArgs] = None, + upstream_steps_to_collect: Optional[int] = None, + **kwargs, ) -> Tuple[TaskResult, Optional[PcieDataModel]]: """Read PCIe data. Args: - args: Optional collector arguments (not used) + args: Optional collector arguments (devid_ep / devid_ep_vf for ESXi GPU BDF resolution) upstream_steps_to_collect: Number of upstream devices to collect **kwargs: Additional keyword arguments Returns: Tuple[TaskResult, Optional[PcieDataModel]]: tuple containing the result of the task and the PCIe data if available """ - pcie_data = self._get_pcie_data(upstream_steps_to_collect) + if args is None: + args = PcieCollectorArgs() + pcie_data = self._get_pcie_data(upstream_steps_to_collect, args.devid_ep, args.devid_ep_vf) if pcie_data: self._log_event( category=EventCategory.IO, diff --git a/nodescraper/plugins/inband/pcie/pcie_plugin.py b/nodescraper/plugins/inband/pcie/pcie_plugin.py index 0e4f3eb0..9d894ade 100644 --- a/nodescraper/plugins/inband/pcie/pcie_plugin.py +++ b/nodescraper/plugins/inband/pcie/pcie_plugin.py @@ -26,18 +26,21 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import PcieAnalyzerArgs +from .collector_args import PcieCollectorArgs from .pcie_analyzer import PcieAnalyzer from .pcie_collector import PcieCollector from .pcie_data import PcieDataModel -class PciePlugin(InBandDataPlugin[PcieDataModel, None, PcieAnalyzerArgs]): +class PciePlugin(InBandDataPlugin[PcieDataModel, PcieCollectorArgs, PcieAnalyzerArgs]): """Plugin for collection and analysis of PCIe data""" DATA_MODEL = PcieDataModel COLLECTOR = PcieCollector + COLLECTOR_ARGS = PcieCollectorArgs + ANALYZER = PcieAnalyzer ANALYZER_ARGS = PcieAnalyzerArgs