diff --git a/nodescraper/plugins/inband/bios/bios_collector.py b/nodescraper/plugins/inband/bios/bios_collector.py index e0ab1011..e94242ef 100644 --- a/nodescraper/plugins/inband/bios/bios_collector.py +++ b/nodescraper/plugins/inband/bios/bios_collector.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.base import InBandDataCollector @@ -35,9 +36,11 @@ class BiosCollector(InBandDataCollector[BiosDataModel, None]): """Collect BIOS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = BiosDataModel CMD_WINDOWS = "wmic bios get SMBIOSBIOSVersion /Value" CMD = "sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'" + CMD_ESXI = "smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1" def collect_data( self, @@ -57,6 +60,11 @@ def collect_data( bios = [line for line in res.stdout.splitlines() if "SMBIOSBIOSVersion=" in line][ 0 ].split("=")[1] + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + match = re.search(r'Version:\s*"?([^"]+)"?', res.stdout) + bios = match.group(1).strip() if match else res.stdout.strip() else: res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: diff --git a/nodescraper/plugins/inband/device_enumeration/collector_args.py b/nodescraper/plugins/inband/device_enumeration/collector_args.py new file mode 100644 index 00000000..c2d7b35c --- /dev/null +++ b/nodescraper/plugins/inband/device_enumeration/collector_args.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class DeviceEnumerationCollectorArgs(CollectorArgs): + """Collector args for device enumeration. + + On ESXi, GPUs and their SR-IOV VFs are counted by PCI device ID (esxcli has no + device filter). Provide the expected PF/VF device IDs here; when unset the ESXi + GPU/VF counts are skipped. The caller populates these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi device counting.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF counting.", + ) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 9b0dc295..78c54ad0 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -30,12 +30,16 @@ from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult +from .collector_args import DeviceEnumerationCollectorArgs from .deviceenumdata import DeviceEnumerationDataModel -class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, None]): +class DeviceEnumerationCollector( + InBandDataCollector[DeviceEnumerationDataModel, DeviceEnumerationCollectorArgs] +): """Collect CPU and GPU count""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DeviceEnumerationDataModel CMD_GPU_COUNT_LINUX = ( @@ -55,6 +59,17 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, 'powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count"' ) + # ESXi busybox `lspci -d` dumps hex instead of filtering, so use esxcli. GPUs are + # counted by device ID (PF vs VF), anchored on "Device ID:" to avoid also matching + # "SubDevice ID:". The match is case-insensitive and tolerates zero-padding + # ("0x744C" / "0x0000744c"); the trailing [^0-9a-f]/$ guard stops a shorter ID from + # matching a longer one (e.g. 744c vs 744cd). + CMD_CPU_COUNT_ESXI = "esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'" + CMD_PCI_COUNT_ESXI = ( + "esxcli hardware pci list | " + "grep -iE '^ *Device ID: 0x0*{device_id}([^0-9a-f]|$)' | wc -l" + ) + def _warning( self, description: str, @@ -72,12 +87,47 @@ def _warning( priority=EventPriority.WARNING, ) - def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: + def _parse_count( + self, + res: CommandArtifact, + description: str, + category: EventCategory = EventCategory.PLATFORM, + ) -> Optional[int]: + """Parse a numeric count from command stdout, warning (not raising) on a + non-zero exit or non-numeric output (e.g. an unexpected esxcli/awk result).""" + if res.exit_code != 0: + self._warning(description=description, command=res, category=category) + return None + text = (res.stdout or "").strip() + if not text.isdigit(): + self._warning( + description=f"{description} (non-numeric output: {text!r})", + command=res, + category=category, + ) + return None + return int(text) + + def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: + """Count PCI devices on ESXi whose Device ID matches ``device_id`` (as hex). + + A None id produces an unmatched pattern (count 0) so the caller still gets a + valid CommandArtifact to parse. + """ + hex_id = format(device_id, "x") if device_id is not None else "__unset__" + return self._run_sut_cmd(self.CMD_PCI_COUNT_ESXI.format(device_id=hex_id)) + + def collect_data( + self, args: Optional[DeviceEnumerationCollectorArgs] = None + ) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: """ Read CPU and GPU count On Linux, use lscpu and lspci + On ESXi, use esxcli (GPU/VF counts need args.devid_ep / devid_ep_vf) On Windows, use WMI and hyper-v cmdlets """ + if args is None: + args = DeviceEnumerationCollectorArgs() if self.system_info.os_family == OSFamily.LINUX: lscpu_res = self._run_sut_cmd(self.CMD_LSCPU_LINUX, log_artifact=False) @@ -92,6 +142,17 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio # Collect lshw output lshw_res = self._run_sut_cmd(self.CMD_LSHW_LINUX, sudo=True, log_artifact=False) + elif self.system_info.os_family == OSFamily.ESXI: + cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_ESXI) + if args.devid_ep is None: + self._log_event( + category=EventCategory.PLATFORM, + description="devid_ep not set; cannot count GPUs/VFs on ESXi by device ID", + priority=EventPriority.WARNING, + ) + # PFs and (SR-IOV) VFs are distinguished by device ID on ESXi. + gpu_count_res = self._esxi_device_count(args.devid_ep) + vf_count_res = self._esxi_device_count(args.devid_ep_vf) else: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_WINDOWS) gpu_count_res = self._run_sut_cmd(self.CMD_GPU_COUNT_WINDOWS) @@ -121,24 +182,19 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio else: self._warning(description="Cannot collect lscpu output", command=lscpu_res) else: - if cpu_count_res.exit_code == 0: - device_enum.cpu_count = int(cpu_count_res.stdout) - else: - self._warning(description="Cannot determine CPU count", command=cpu_count_res) + cpu_count = self._parse_count(cpu_count_res, "Cannot determine CPU count") + if cpu_count is not None: + device_enum.cpu_count = cpu_count - if gpu_count_res.exit_code == 0: - device_enum.gpu_count = int(gpu_count_res.stdout) - else: - self._warning(description="Cannot determine GPU count", command=gpu_count_res) + gpu_count = self._parse_count(gpu_count_res, "Cannot determine GPU count") + if gpu_count is not None: + device_enum.gpu_count = gpu_count - if vf_count_res.exit_code == 0: - device_enum.vf_count = int(vf_count_res.stdout) - else: - self._warning( - description="Cannot determine VF count", - command=vf_count_res, - category=EventCategory.SW_DRIVER, - ) + vf_count = self._parse_count( + vf_count_res, "Cannot determine VF count", category=EventCategory.SW_DRIVER + ) + if vf_count is not None: + device_enum.vf_count = vf_count # Collect lshw output on Linux if self.system_info.os_family == OSFamily.LINUX: diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py index baf2aa2d..cff51210 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py @@ -26,13 +26,18 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import DeviceEnumerationAnalyzerArgs +from .collector_args import DeviceEnumerationCollectorArgs from .device_enumeration_analyzer import DeviceEnumerationAnalyzer from .device_enumeration_collector import DeviceEnumerationCollector from .deviceenumdata import DeviceEnumerationDataModel class DeviceEnumerationPlugin( - InBandDataPlugin[DeviceEnumerationDataModel, None, DeviceEnumerationAnalyzerArgs] + InBandDataPlugin[ + DeviceEnumerationDataModel, + DeviceEnumerationCollectorArgs, + DeviceEnumerationAnalyzerArgs, + ] ): """Plugin for collection and analysis of BIOS data""" @@ -40,6 +45,8 @@ class DeviceEnumerationPlugin( COLLECTOR = DeviceEnumerationCollector + COLLECTOR_ARGS = DeviceEnumerationCollectorArgs + ANALYZER = DeviceEnumerationAnalyzer ANALYZER_ARGS = DeviceEnumerationAnalyzerArgs diff --git a/nodescraper/plugins/inband/dimm/dimm_collector.py b/nodescraper/plugins/inband/dimm/dimm_collector.py index b6b91987..c9b4dd8f 100644 --- a/nodescraper/plugins/inband/dimm/dimm_collector.py +++ b/nodescraper/plugins/inband/dimm/dimm_collector.py @@ -38,12 +38,31 @@ class DimmCollector(InBandDataCollector[DimmDataModel, DimmCollectorArgs]): """Collect data on installed DIMMs""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DimmDataModel CMD_WINDOWS = "wmic memorychip get Capacity" CMD = """sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\\|None\\|Module" | grep Size' 2>/dev/null""" + CMD_ESXI = "smbiosDump | grep -A15 'Memory Device (Type 17)' | grep 'Size:'" CMD_DMIDECODE_FULL = "dmidecode" + def _parse_dmi_sizes(self, stdout: str) -> str: + """Build the DIMM summary from 'Size: ' lines (dmidecode on Linux, + smbiosDump on ESXi — both emit the same field format).""" + total = 0 + topology: dict[str, int] = {} + size = "" + dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") + for num, unit in dimm_size_pattern.findall(stdout): + size = unit + total += int(num) + key = num + unit + topology[key] = topology.get(key, 0) + 1 + if total == 0: + return "0 GB" + dimm_entries = [f"{v} x {k}" for k, v in topology.items()] + return f"{total}{size} @ {' '.join(dimm_entries)}" + def collect_data( self, args: Optional[DimmCollectorArgs] = None, @@ -70,6 +89,10 @@ def collect_data( dimm_str = f"{total / 1024 / 1024:.2f}GB @ " for capacity, count in capacities.items(): dimm_str += f"{count} x {capacity / 1024 / 1024:.2f}GB " + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + dimm_str = self._parse_dmi_sizes(res.stdout) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" @@ -96,29 +119,7 @@ def collect_data( res = self._run_sut_cmd(self.CMD, sudo=True) if res.exit_code == 0: - total = 0 - topology = {} - size = "" - dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") - matches = dimm_size_pattern.findall(res.stdout) - if matches: - for match in matches: - size = match[1] - total += int(match[0]) - key = match[0] + match[1] - if not topology.get(key, None): - topology[key] = 1 - else: - topology[key] += 1 - topology["total"] = total - topology["size"] = size - total_gb = topology.pop("total") - size = topology.pop("size") - if total_gb == 0: - dimm_str = "0 GB" - else: - dimm_entries = [f"{v} x {k}" for k, v in topology.items()] - dimm_str = f"{total_gb}{size} @ {' '.join(dimm_entries)}" + dimm_str = self._parse_dmi_sizes(res.stdout) if res.exit_code != 0: self._log_event( category=EventCategory.OS, diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 5ae53f77..5bf93c36 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -30,7 +30,7 @@ from nodescraper.base.match_ignore import parse_ignore_match_rules from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer from nodescraper.connection.inband import TextFileArtifact -from nodescraper.enums import EventCategory, EventPriority +from nodescraper.enums import EventCategory, EventPriority, OSFamily from nodescraper.models import Event, TaskResult from .analyzer_args import DmesgAnalyzerArgs @@ -47,10 +47,24 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): - """Check dmesg for errors""" + """Check dmesg (Linux) or vmkernel.log (ESXi) for errors""" DATA_MODEL = DmesgData + # ESXi vmkernel.log timestamp, e.g. "2026-08-05T19:53:35.178Z" (ISO8601 dot-ms + Z). + # Linux uses the base RegexAnalyzer.TIMESTAMP_PATTERN (comma-form). + ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)") + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # On ESXi, extract vmkernel.log timestamps so event grouping and date-range + # filtering both work; Linux keeps the base comma-form pattern. + if self._is_esxi(): + self.TIMESTAMP_PATTERN = self.ESXI_TIMESTAMP_PATTERN + + def _is_esxi(self) -> bool: + return self.system_info.os_family == OSFamily.ESXI + ERROR_REGEX: list[ErrorRegex] = [ ErrorRegex( regex=re.compile(r"(?:oom_kill_process.*)|(?:Out of memory.*)"), @@ -273,6 +287,30 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): message="RAS Deferred Error", event_category=EventCategory.RAS, ), + # ESXi mxGPU (gim/amdgpuv) RAS phrasing differs from Linux: the block name is + # capitalized ("... detected in MMHUB Block."), there is no "in total", and no + # "kern :err:" prefix. These match the ESXi host-driver forms and are inert on + # Linux logs (which use the lowercase "in total in block" phrasing above). + ErrorRegex( + regex=re.compile(r"(\d+ new uncorrectable hardware errors detected in \w+ Block.*)"), + message="RAS Uncorrectable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(\d+ new correctable hardware errors detected in \w+ Block.*)"), + message="RAS Correctable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(GPU detected ECC Fatal Error\.)"), + message="RAS ECC Fatal Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(Issuing Whole GPU reset\.)"), + message="GPU Reset", + event_category=EventCategory.RAS, + ), ErrorRegex( regex=re.compile( r"((?:\[Hardware Error\]:\s+)?event severity: corrected.*)" @@ -463,6 +501,15 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): ), ] + # Date-range filtering must recognize both Linux dmesg comma-form timestamps + # (2024-10-01T05:00:00,000000-05:00) and ESXi vmkernel.log dot-ms/Z timestamps + # (2026-08-20T09:35:58.380Z). filter_dmesg stays a classmethod (public API), so it + # carries its own combined pattern rather than the instance TIMESTAMP_PATTERN. + _FILTER_TIMESTAMP_PATTERN: re.Pattern = re.compile( + r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)" + r"|(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" + ) + @classmethod def filter_dmesg( cls, @@ -470,7 +517,7 @@ def filter_dmesg( analysis_range_start: Optional[datetime.datetime] = None, analysis_range_end: Optional[datetime.datetime] = None, ) -> str: - """Filter a dmesg log by date + """Filter a dmesg (Linux) or vmkernel.log (ESXi) log by date Args: dmesg_content (str): unfiltered dmesg log @@ -482,9 +529,16 @@ def filter_dmesg( filtered_dmesg = "" found_start = False if analysis_range_start else True for line in dmesg_content.splitlines(): - date = re.search(r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)", line) - if date is not None: - date = datetime.datetime.fromisoformat(f"{date.group(1)}.{date.group(2)}") + match = cls._FILTER_TIMESTAMP_PATTERN.search(line) + if match is not None: + if match.group(1) is not None: + # Linux comma-form: swap the comma for a dot so fromisoformat accepts it + iso = f"{match.group(1)}.{match.group(2)}" + else: + # ESXi dot-Z form: normalize the trailing Z so fromisoformat accepts it + # on Python < 3.11 as well + iso = match.group(3).replace("Z", "+00:00") + date = datetime.datetime.fromisoformat(iso) # show date in UTC now date = date.astimezone(datetime.timezone.utc) if analysis_range_start and not found_start and date >= analysis_range_start: @@ -743,11 +797,22 @@ def analyze_data( self.result.events += known_err_events if args.check_unknown_dmesg_errors: + if self._is_esxi(): + # ESXi vmkernel severity tokens are unreliable (-ALERT is used for benign + # boot notices; -ERROR/-CRIT are never emitted). The reliable error signal + # is the driver-internal severity in the message body: "gim error/warning", + # "amdgpuv error/warning", or the bracket form "[amdgpuv warn]". + unknown_error_regex = re.compile( + r"(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning) [^:]*:\s*(.*)" + r"|\[(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning)\]:?\s*(.*)" + ) + else: + unknown_error_regex = re.compile( + r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" + ) unknown_dmesg_error_regexes = [ ErrorRegex( - regex=re.compile( - r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" - ), + regex=unknown_error_regex, message="Unknown dmesg error", event_category=EventCategory.UNKNOWN, event_priority=EventPriority.WARNING, diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index c280d7d2..4fcbc4b7 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -38,24 +38,39 @@ class DmesgCollector(InBandDataCollector[DmesgData, DmesgCollectorArgs]): """Read dmesg log""" - SUPPORTED_OS_FAMILY = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY = {OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DmesgData CMD = "dmesg --time-format iso -x" + # ESXi has no dmesg ring buffer; the kernel log is the vmkernel.log file. + CMD_ESXI = "cat /var/log/vmkernel.log" CMD_LOGS = ( r"ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true" ) + # ESXi rotates vmkernel.log to vmkernel. / vmkernel..gz. + CMD_LOGS_ESXI = r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" def _collect_dmesg_rotations(self): - """Collect dmesg logs""" - list_res = self._run_sut_cmd(self.CMD_LOGS, sudo=True) + """Collect dmesg (Linux) / vmkernel.log (ESXi) rotated logs""" + is_esxi = self.system_info.os_family == OSFamily.ESXI + if is_esxi: + log_label = "vmkernel" + cmd_logs = self.CMD_LOGS_ESXI + else: + log_label = "dmesg" + cmd_logs = self.CMD_LOGS + list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: + if is_esxi: + description = "No /var/log/vmkernel.log files found (including rotations)." + else: + description = "No /var/log/dmesg files found (including rotations)." self._log_event( category=EventCategory.OS, - description="No /var/log/dmesg files found (including rotations).", + description=description, data={"list_exit_code": list_res.exit_code}, priority=EventPriority.WARNING, ) @@ -68,7 +83,7 @@ def _collect_dmesg_rotations(self): cmd = f"gzip -dc {qp} 2>/dev/null || zcat {qp} 2>/dev/null" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -84,7 +99,7 @@ def _collect_dmesg_rotations(self): cmd = f"cat {qp}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -121,8 +136,13 @@ def _get_dmesg_content(self) -> str: str: dmesg output """ - self.logger.info("Running dmesg command on system") - res = self._run_sut_cmd(self.CMD, sudo=True, log_artifact=False) + is_esxi = self.system_info.os_family == OSFamily.ESXI + if is_esxi: + cmd = self.CMD_ESXI + else: + cmd = self.CMD + self.logger.info("Reading kernel log from system") + res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code != 0: self._log_event( category=EventCategory.OS, diff --git a/nodescraper/plugins/inband/kernel/kernel_collector.py b/nodescraper/plugins/inband/kernel/kernel_collector.py index 6b188940..d93fc215 100644 --- a/nodescraper/plugins/inband/kernel/kernel_collector.py +++ b/nodescraper/plugins/inband/kernel/kernel_collector.py @@ -36,6 +36,7 @@ class KernelCollector(InBandDataCollector[KernelDataModel, None]): """Read kernel version""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = KernelDataModel CMD_WINDOWS = "wmic os get Version /Value" CMD = "sh -c 'uname -a'" @@ -88,6 +89,9 @@ def collect_data( "=" )[1] else: + # Non-Windows (Linux and ESXi). ESXi `uname -a` yields the release in the + # same field the Linux parser reads (verified: "9.1.0"); numa_balancing has + # no ESXi equivalent and its command fails gracefully, leaving None. res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: kernel_info = res.stdout diff --git a/nodescraper/plugins/inband/os/os_collector.py b/nodescraper/plugins/inband/os/os_collector.py index 42e435a9..2fc46ab5 100644 --- a/nodescraper/plugins/inband/os/os_collector.py +++ b/nodescraper/plugins/inband/os/os_collector.py @@ -36,10 +36,13 @@ class OsCollector(InBandDataCollector[OsDataModel, None]): """Collect OS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = OsDataModel CMD_VERSION_WINDOWS = "wmic os get Version /value" CMD_VERSION = "cat /etc/*release | grep VERSION_ID" + CMD_VERSION_ESXI = "esxcli system version get" CMD_WINDOWS = "wmic os get Caption /Value" + CMD_ESXI = "vmware -v" PRETTY_STR = "PRETTY_NAME" # noqa: N806 CMD = f"sh -c '( lsb_release -ds || (cat /etc/*release | grep {PRETTY_STR}) || uname -om ) 2>/dev/null | head -n1'" @@ -60,6 +63,22 @@ def collect_version(self) -> str: priority=EventPriority.ERROR, ) os_version = "" + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_VERSION_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + if "Version:" in line: + os_version = line.split(":", 1)[1].strip() + break + else: + os_version = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS version not found", + priority=EventPriority.ERROR, + ) + os_version = "" else: res = self._run_sut_cmd(self.CMD_VERSION) if res.exit_code == 0: @@ -86,6 +105,16 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[OsDataModel]]: res = self._run_sut_cmd(self.CMD_WINDOWS) if res.exit_code == 0: os_name = re.search(r"Caption=([\w\s]+)", res.stdout).group(1) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + os_name = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS name not found", + priority=EventPriority.ERROR, + ) else: res = self._run_sut_cmd(self.CMD) # search for PRETTY_NAME in res diff --git a/nodescraper/plugins/inband/pcie/collector_args.py b/nodescraper/plugins/inband/pcie/collector_args.py new file mode 100644 index 00000000..21260fe2 --- /dev/null +++ b/nodescraper/plugins/inband/pcie/collector_args.py @@ -0,0 +1,24 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class PcieCollectorArgs(CollectorArgs): + """Collector args for PCIe data. + + On ESXi, GPU/VF BDFs are resolved from ``esxcli hardware pci list`` by matching + the expected PF/VF PCI device IDs (esxcli has no device filter). Provide them + here; when both are unset no ESXi GPU BDFs are resolved. The caller populates + these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi BDF resolution.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF BDF resolution.", + ) diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 624122ec..1085bf2a 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -41,6 +41,7 @@ from nodescraper.models import TaskResult from nodescraper.utils import get_all_subclasses, get_exception_details +from .collector_args import PcieCollectorArgs from .pcie_data import ( MAX_CAP_ID, MAX_ECAP_ID, @@ -54,7 +55,7 @@ ) -class PcieCollector(InBandDataCollector[PcieDataModel, None]): +class PcieCollector(InBandDataCollector[PcieDataModel, PcieCollectorArgs]): """class for collection of PCIe data only supports Linux OS type. This class collects the PCIE config space using the lspci hex dump and then parses the hex dump to get the @@ -80,7 +81,10 @@ class PcieCollector(InBandDataCollector[PcieDataModel, None]): """ - SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX, OSFamily.ESXI} + + # A bare BDF line that begins an esxcli/lspci device block, e.g. "0000:05:00.0". + _BDF_LINE = re.compile(r"^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]+", re.IGNORECASE) DATA_MODEL = PcieDataModel @@ -518,17 +522,8 @@ def get_cap_cfg( return cap_structure # type: ignore[return-value] - def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: - """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" - hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) - if hex_data_raw is None: - self._log_event( - category=EventCategory.IO, - description="Failed to get hex data for BDF.", - data={"bdf": bdf}, - priority=EventPriority.ERROR, - ) - return PcieCfgSpace() + def _cfg_space_from_hex(self, hex_data_raw: str, bdf: str) -> PcieCfgSpace: + """Parse a raw lspci hex dump (Linux ``-xxxx`` or ESXi ``-e``) into a PcieCfgSpace.""" hex_data: List[int] = self.parse_hex_dump(hex_data_raw) if len(hex_data) < 64: # Expect at least 256 bytes of data, for the first 256 bytes of the PCIe config space @@ -542,6 +537,19 @@ def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: cap_data, ecap_data = self.discover_capability_structure(hex_data) return self.get_pcie_cfg(hex_data, cap_data, ecap_data) + def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: + """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" + hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) + if hex_data_raw is None: + self._log_event( + category=EventCategory.IO, + description="Failed to get hex data for BDF.", + data={"bdf": bdf}, + priority=EventPriority.ERROR, + ) + return PcieCfgSpace() + return self._cfg_space_from_hex(hex_data_raw, bdf) + def get_pcie_cfg( self, config_data: List[int], @@ -595,8 +603,132 @@ def _log_pcie_artifacts( if data is not None: self.result.artifacts.append(TextFileArtifact(filename=name, contents=data)) + def _get_gpu_vf_bdfs_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Tuple[List[str], List[str]]: + """Return (pf_bdfs, vf_bdfs) for the GPUs on an ESXi host via esxcli. + + ESXi busybox lspci has no device filter, so GPU/VF BDFs are resolved from + ``esxcli hardware pci list`` by matching the expected PF/VF device IDs + (``pf_devid`` / ``vf_devid``, from the collector args). Each device block + starts with a bare BDF line followed by indented fields incl. "Device ID". + """ + pf_bdfs: List[str] = [] + vf_bdfs: List[str] = [] + if pf_devid is None and vf_devid is None: + return pf_bdfs, vf_bdfs + + out = self._run_os_cmd("esxcli hardware pci list", sudo=False) + if not out: + return pf_bdfs, vf_bdfs + + current_bdf: Optional[str] = None + for line in out.splitlines(): + stripped = line.strip() + if self._BDF_LINE.match(stripped) and ":" in stripped and " " not in stripped: + # Bare BDF header line (anchors the block). + current_bdf = stripped + elif current_bdf and stripped.lower().startswith("device id:"): + # Compare by integer value so case ("0x744C") and zero-padding + # ("0x0000744c") both match the expected device ID. + raw = stripped.split(":", 1)[1].strip() + try: + devid = int(raw, 16) + except ValueError: + continue + if pf_devid is not None and devid == pf_devid: + pf_bdfs.append(current_bdf) + elif vf_devid is not None and devid == vf_devid: + vf_bdfs.append(current_bdf) + return pf_bdfs, vf_bdfs + + def _get_all_cfg_space_esxi(self) -> Dict[str, str]: + """Return {bdf: hex_dump_text} for every device from a single ``lspci -e``. + + ESXi has no per-device dump; ``lspci -e`` emits the full extended (4096-byte) + config space for all devices in one blob. Each device section starts with a + header line " " followed by "NN: .." hex lines. + """ + blob = self._run_os_cmd("lspci -e", sudo=False) + if not blob: + return {} + self.result.artifacts.append(TextFileArtifact(filename="lspci_e.txt", contents=blob)) + sections: Dict[str, List[str]] = {} + current_bdf: Optional[str] = None + for line in blob.splitlines(): + header = self._BDF_LINE.match(line) + if header and " " in line: + # Device header line: " ". + current_bdf = line.split(" ", 1)[0] + sections[current_bdf] = [] + elif current_bdf is not None: + sections[current_bdf].append(line) + return {bdf: "\n".join(lines) for bdf, lines in sections.items()} + + def _get_pcie_data_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Optional[PcieDataModel]: + """Collect GPU + VF PCIe config space on ESXi. + + ESXi busybox lspci lacks ``-s`` (per-device) and ``-PP`` (bus-path), so dump + all extended config space once via ``lspci -e``, split it by BDF, and select the + GPU/VF BDFs resolved from esxcli (matching ``pf_devid`` / ``vf_devid`` from the + collector args). Upstream-bridge traversal is not available on ESXi and is + intentionally skipped (GPU + VF only). + """ + pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi(pf_devid, vf_devid) + if not pf_bdfs and not vf_bdfs: + self._log_event( + category=EventCategory.IO, + description="No GPU/VF BDFs found on ESXi host for this SKU.", + data={"devid_ep": pf_devid, "devid_ep_vf": vf_devid}, + priority=EventPriority.WARNING, + ) + return None + + cfg_by_bdf = self._get_all_cfg_space_esxi() + if not cfg_by_bdf: + self.result.status = ExecutionStatus.ERROR + return None + + self._log_event( + category=EventCategory.IO, + description=( + "Upstream-bridge PCIe collection is not supported on ESXi; " + "collecting GPU + VF only." + ), + priority=EventPriority.INFO, + ) + + try: + pcie_cfg_dict: Dict[str, PcieCfgSpace] = {} + for bdf in pf_bdfs: + if bdf in cfg_by_bdf: + pcie_cfg_dict[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + vf_pcie_cfg_data: Dict[str, PcieCfgSpace] = {} + for bdf in vf_bdfs: + if bdf in cfg_by_bdf: + vf_pcie_cfg_data[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + pcie_data = PcieDataModel( + pcie_cfg_space=pcie_cfg_dict, + vf_pcie_cfg_space=vf_pcie_cfg_data, + ) + except ValidationError as e: + self._log_event( + category=EventCategory.OS, + description="Failed to build model for PCIe data", + data=get_exception_details(e), + priority=EventPriority.ERROR, + ) + self.result.status = ExecutionStatus.ERROR + return None + return pcie_data + def _get_pcie_data( - self, upstream_steps_to_collect: Optional[int] = None + self, + upstream_steps_to_collect: Optional[int] = None, + pf_devid: Optional[int] = None, + vf_devid: Optional[int] = None, ) -> Optional[PcieDataModel]: """Will return all PCIe data in a PcieDataModel object. @@ -605,6 +737,9 @@ def _get_pcie_data( Optional[PcieDataModel] The data in a PcieDataModel object or None on failure """ + if self.system_info.os_family == OSFamily.ESXI: + return self._get_pcie_data_esxi(pf_devid, vf_devid) + minimum_system_interaction_level_required_for_sudo = SystemInteractionLevel.INTERACTIVE try: @@ -702,19 +837,24 @@ def discover_capability_structure( return cap, ecap def collect_data( - self, args=None, upstream_steps_to_collect: Optional[int] = None, **kwargs + self, + args: Optional[PcieCollectorArgs] = None, + upstream_steps_to_collect: Optional[int] = None, + **kwargs, ) -> Tuple[TaskResult, Optional[PcieDataModel]]: """Read PCIe data. Args: - args: Optional collector arguments (not used) + args: Optional collector arguments (devid_ep / devid_ep_vf for ESXi GPU BDF resolution) upstream_steps_to_collect: Number of upstream devices to collect **kwargs: Additional keyword arguments Returns: Tuple[TaskResult, Optional[PcieDataModel]]: tuple containing the result of the task and the PCIe data if available """ - pcie_data = self._get_pcie_data(upstream_steps_to_collect) + if args is None: + args = PcieCollectorArgs() + pcie_data = self._get_pcie_data(upstream_steps_to_collect, args.devid_ep, args.devid_ep_vf) if pcie_data: self._log_event( category=EventCategory.IO, diff --git a/nodescraper/plugins/inband/pcie/pcie_plugin.py b/nodescraper/plugins/inband/pcie/pcie_plugin.py index 0e4f3eb0..9d894ade 100644 --- a/nodescraper/plugins/inband/pcie/pcie_plugin.py +++ b/nodescraper/plugins/inband/pcie/pcie_plugin.py @@ -26,18 +26,21 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import PcieAnalyzerArgs +from .collector_args import PcieCollectorArgs from .pcie_analyzer import PcieAnalyzer from .pcie_collector import PcieCollector from .pcie_data import PcieDataModel -class PciePlugin(InBandDataPlugin[PcieDataModel, None, PcieAnalyzerArgs]): +class PciePlugin(InBandDataPlugin[PcieDataModel, PcieCollectorArgs, PcieAnalyzerArgs]): """Plugin for collection and analysis of PCIe data""" DATA_MODEL = PcieDataModel COLLECTOR = PcieCollector + COLLECTOR_ARGS = PcieCollectorArgs + ANALYZER = PcieAnalyzer ANALYZER_ARGS = PcieAnalyzerArgs diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index e5373ebc..7b096b52 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -37,9 +37,11 @@ class StorageCollector(InBandDataCollector[StorageDataModel, None]): """Collect disk usage details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = StorageDataModel CMD_WINDOWS = """wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace""" CMD = """sh -c 'df -lH -B1 | grep -v 'boot''""" + CMD_ESXI = "esxcli storage filesystem list" def collect_data( self, args: Optional[StorageCollectorArgs] = None @@ -61,6 +63,28 @@ def collect_data( used=int(size) - int(free_space), percent=round((int(size) - int(free_space)) / int(size) * 100, 2), ) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + # esxcli columns (fixed order): [0] Mount Point [1] Volume Name + # [2] UUID [3] Mounted [4] Type [5] Size [6] Free + fields = re.split(r"\s{2,}", line.strip()) + if len(fields) >= 7 and fields[5].isdigit() and fields[6].isdigit(): + device_id = fields[0] + total_bytes = int(fields[5]) + free_bytes = int(fields[6]) + used_bytes = total_bytes - free_bytes + if total_bytes: + usage_percent = round(used_bytes / total_bytes * 100, 2) + else: + usage_percent = 0.0 + storage_data[device_id] = DeviceStorageData( + total=total_bytes, + free=free_bytes, + used=used_bytes, + percent=usage_percent, + ) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin"