diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..0f0f7ca8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +# node-scraper line ending enforcement + +# Default: auto-detect text, normalize to LF on checkout for all text files. +* text=auto eol=lf + +# Explicit binary so git never applies EOL conversion or line-diffing. +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary +*.zip binary +*.gz binary +*.whl binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b77d429c..2b7c7033 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,2 @@ *@alexandraBara +*@amd-ansonyim diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 2ee95c0c..0f8773e5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,72 +1,72 @@ -name: Bug report -description: Report a reproducible problem with Node Scraper -title: "[Bug]: " -labels: ["bug"] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to report a bug. Please include enough detail to reproduce the issue. - - type: textarea - id: summary - attributes: - label: Summary - description: What happened and what did you expect to happen? - placeholder: Clear and concise description of the bug. - validations: - required: true - - type: textarea - id: steps - attributes: - label: Steps to reproduce - description: Minimal, concrete steps to reproduce the issue. - placeholder: | - 1. ... - 2. ... - 3. ... - validations: - required: true - - type: textarea - id: logs - attributes: - label: CLI output / logs - description: Paste relevant output (redact secrets/tokens/hosts as needed). - render: shell - validations: - required: false - - type: input - id: version - attributes: - label: Version - description: Output of `node-scraper --version` (or the git commit hash). - placeholder: "e.g. 0.6.1" - validations: - required: false - - type: dropdown - id: install_method - attributes: - label: Install method - options: - - PyPI (pip install amd-node-scraper) - - Editable install from source (pip install -e .) - - Other - validations: - required: false - - type: input - id: python - attributes: - label: Python version - placeholder: "e.g. 3.11.8" - validations: - required: false - - type: textarea - id: environment - attributes: - label: Environment - description: OS, target system type (LOCAL/REMOTE), and anything else relevant. - placeholder: | - OS: - Target: - Notes: - validations: - required: false +name: Bug report +description: Report a reproducible problem with Node Scraper +title: "[Bug]: " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report a bug. Please include enough detail to reproduce the issue. + - type: textarea + id: summary + attributes: + label: Summary + description: What happened and what did you expect to happen? + placeholder: Clear and concise description of the bug. + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: Minimal, concrete steps to reproduce the issue. + placeholder: | + 1. ... + 2. ... + 3. ... + validations: + required: true + - type: textarea + id: logs + attributes: + label: CLI output / logs + description: Paste relevant output (redact secrets/tokens/hosts as needed). + render: shell + validations: + required: false + - type: input + id: version + attributes: + label: Version + description: Output of `node-scraper --version` (or the git commit hash). + placeholder: "e.g. 0.6.1" + validations: + required: false + - type: dropdown + id: install_method + attributes: + label: Install method + options: + - PyPI (pip install amd-node-scraper) + - Editable install from source (pip install -e .) + - Other + validations: + required: false + - type: input + id: python + attributes: + label: Python version + placeholder: "e.g. 3.11.8" + validations: + required: false + - type: textarea + id: environment + attributes: + label: Environment + description: OS, target system type (LOCAL/REMOTE), and anything else relevant. + placeholder: | + OS: + Target: + Notes: + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 7a048e76..f83f075c 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,8 +1,8 @@ -blank_issues_enabled: false -contact_links: - - name: Questions - url: https://github.com/amd/node-scraper/issues - about: If you're not sure it's a bug/feature request yet, start with an issue and add details. - - name: Security reports - url: https://github.com/amd/node-scraper/security/policy - about: Please report security issues privately (see SECURITY.md). +blank_issues_enabled: false +contact_links: + - name: Questions + url: https://github.com/amd/node-scraper/issues + about: If you're not sure it's a bug/feature request yet, start with an issue and add details. + - name: Security reports + url: https://github.com/amd/node-scraper/security/policy + about: Please report security issues privately (see SECURITY.md). diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index b9249856..14d3e688 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,38 +1,38 @@ -name: Feature request -description: Suggest an enhancement or new capability -title: "[Feature]: " -labels: ["enhancement"] -body: - - type: markdown - attributes: - value: | - Thanks for the suggestion. Please describe the problem you're trying to solve and what success looks like. - - type: textarea - id: problem - attributes: - label: Problem / motivation - description: What problem are you trying to solve? - placeholder: "I want to..." - validations: - required: true - - type: textarea - id: proposal - attributes: - label: Proposed solution - description: What would you like Node Scraper to do? - validations: - required: true - - type: textarea - id: alternatives - attributes: - label: Alternatives considered - description: What else have you tried or considered? - validations: - required: false - - type: textarea - id: scope - attributes: - label: Scope / impact - description: What plugins/OSes/environments would this affect? - validations: - required: false +name: Feature request +description: Suggest an enhancement or new capability +title: "[Feature]: " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for the suggestion. Please describe the problem you're trying to solve and what success looks like. + - type: textarea + id: problem + attributes: + label: Problem / motivation + description: What problem are you trying to solve? + placeholder: "I want to..." + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed solution + description: What would you like Node Scraper to do? + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: What else have you tried or considered? + validations: + required: false + - type: textarea + id: scope + attributes: + label: Scope / impact + description: What plugins/OSes/environments would this affect? + validations: + required: false diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 269ee465..dabd6d0e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,12 @@ -## Summary -- - -## Test plan -- [ ] `pytest test/unit` -- [ ] `pytest test/functional` (if applicable) -- [ ] `pre-commit run --all-files` - -## Checklist -- [ ] Added/updated tests (or explained why not) -- [ ] Updated docs/README if behavior changed -- [ ] No secrets or credentials committed +## Summary +- + +## Test plan +- [ ] `pytest test/unit` +- [ ] `pytest test/functional` (if applicable) +- [ ] `pre-commit run --all-files` + +## Checklist +- [ ] Added/updated tests (or explained why not) +- [ ] Updated docs/README if behavior changed +- [ ] No secrets or credentials committed diff --git a/.github/workflows/code_quality_checks.yml b/.github/workflows/code_quality_checks.yml index d92e6bbf..59b8da2e 100644 --- a/.github/workflows/code_quality_checks.yml +++ b/.github/workflows/code_quality_checks.yml @@ -18,6 +18,9 @@ jobs: with: fetch-depth: 0 # Fetch all history for pre-commit to work + - name: Set up uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + # python:3.9 image has no git; pre-commit requires it. - name: Install git run: | diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index c8f15246..1de978b6 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -17,6 +17,9 @@ jobs: steps: - uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + - name: Install xmllint run: | apt-get update diff --git a/.github/workflows/release-trusted-publisher.yml b/.github/workflows/release-trusted-publisher.yml index e2a29680..8be52141 100644 --- a/.github/workflows/release-trusted-publisher.yml +++ b/.github/workflows/release-trusted-publisher.yml @@ -1,127 +1,124 @@ -name: Release (Trusted Publisher) - -permissions: - contents: write # Required for creating releases and pushing tags - id-token: write # Required for PyPI Trusted Publishing - -on: - workflow_dispatch: - inputs: - release_type: - description: 'Release type (major, minor, patch)' - required: true - type: choice - options: - - patch - - minor - - major - -jobs: - release: - runs-on: ubuntu-latest - # Only allow this workflow to run on main branch - if: github.ref == 'refs/heads/main' - - steps: - - name: Checkout code - uses: actions/checkout@v5 - with: - fetch-depth: 0 # Fetch all history and tags - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Fetch all tags - run: | - git fetch --tags --force - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - - name: Configure Git - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - - name: Calculate next version - id: next_version - shell: bash - run: | - # Get the highest semantic version tag (not most recent, but highest version number) - LATEST_TAG=$(git tag -l 'v*' | sort -V | tail -n1) - if [ -z "$LATEST_TAG" ]; then - LATEST_TAG="v0.0.0" - fi - echo "Latest tag: $LATEST_TAG" - - # Remove 'v' prefix and split into components - VERSION=${LATEST_TAG#v} - IFS='.' read -r -a VERSION_PARTS <<< "$VERSION" - MAJOR="${VERSION_PARTS[0]:-0}" - MINOR="${VERSION_PARTS[1]:-0}" - PATCH="${VERSION_PARTS[2]:-0}" - - echo "Current version: $MAJOR.$MINOR.$PATCH" - - # Increment based on release type - case "${{ github.event.inputs.release_type }}" in - major) - MAJOR=$((MAJOR + 1)) - MINOR=0 - PATCH=0 - ;; - minor) - MINOR=$((MINOR + 1)) - PATCH=0 - ;; - patch) - PATCH=$((PATCH + 1)) - ;; - esac - - NEW_VERSION="v${MAJOR}.${MINOR}.${PATCH}" - echo "New version: $NEW_VERSION" - echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT - - - name: Create and push tag - run: | - git tag ${{ steps.next_version.outputs.version }} - git push origin ${{ steps.next_version.outputs.version }} - - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - python -m pip install build - - - name: Build package - run: | - python -m build - - - name: Upload to PyPI using Trusted Publisher - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: dist/ - - - name: Create GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release create ${{ steps.next_version.outputs.version }} \ - --title "${{ steps.next_version.outputs.version }}" \ - --generate-notes \ - dist/* - - - name: Print summary - if: success() - run: | - echo "### :rocket: Release ${{ steps.next_version.outputs.version }} completed successfully!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Release Type:** ${{ github.event.inputs.release_type }}" >> $GITHUB_STEP_SUMMARY - echo "- **New Version:** ${{ steps.next_version.outputs.version }}" >> $GITHUB_STEP_SUMMARY - echo "- **PyPI Package:** Published via Trusted Publisher" >> $GITHUB_STEP_SUMMARY - echo "- **GitHub Release:** Created with auto-generated notes" >> $GITHUB_STEP_SUMMARY - - - name: Print failure message - if: failure() - run: | - echo "### :x: Release failed. Please check the logs above." >> $GITHUB_STEP_SUMMARY +name: Release (Trusted Publisher) + +permissions: + contents: write # Required for creating releases and pushing tags + id-token: write # Required for PyPI Trusted Publishing + +on: + workflow_dispatch: + inputs: + release_type: + description: 'Release type (major, minor, patch)' + required: true + type: choice + options: + - patch + - minor + - major + +jobs: + release: + runs-on: ubuntu-latest + # Only allow this workflow to run on main branch + if: github.ref == 'refs/heads/main' + + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 # Fetch all history and tags + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Fetch all tags + run: | + git fetch --tags --force + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Configure Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Calculate next version + id: next_version + shell: bash + run: | + # Get the highest semantic version tag (not most recent, but highest version number) + LATEST_TAG=$(git tag -l 'v*' | sort -V | tail -n1) + if [ -z "$LATEST_TAG" ]; then + LATEST_TAG="v0.0.0" + fi + echo "Latest tag: $LATEST_TAG" + + # Remove 'v' prefix and split into components + VERSION=${LATEST_TAG#v} + IFS='.' read -r -a VERSION_PARTS <<< "$VERSION" + MAJOR="${VERSION_PARTS[0]:-0}" + MINOR="${VERSION_PARTS[1]:-0}" + PATCH="${VERSION_PARTS[2]:-0}" + + echo "Current version: $MAJOR.$MINOR.$PATCH" + + # Increment based on release type + case "${{ github.event.inputs.release_type }}" in + major) + MAJOR=$((MAJOR + 1)) + MINOR=0 + PATCH=0 + ;; + minor) + MINOR=$((MINOR + 1)) + PATCH=0 + ;; + patch) + PATCH=$((PATCH + 1)) + ;; + esac + + NEW_VERSION="v${MAJOR}.${MINOR}.${PATCH}" + echo "New version: $NEW_VERSION" + echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT + + - name: Create and push tag + run: | + git tag ${{ steps.next_version.outputs.version }} + git push origin ${{ steps.next_version.outputs.version }} + + - name: Set up uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + - name: Build package + run: uv build + + - name: Upload to PyPI using Trusted Publisher + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # release/v1 + with: + packages-dir: dist/ + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create ${{ steps.next_version.outputs.version }} \ + --title "${{ steps.next_version.outputs.version }}" \ + --generate-notes \ + dist/* + + - name: Print summary + if: success() + run: | + echo "### :rocket: Release ${{ steps.next_version.outputs.version }} completed successfully!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Release Type:** ${{ github.event.inputs.release_type }}" >> $GITHUB_STEP_SUMMARY + echo "- **New Version:** ${{ steps.next_version.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "- **PyPI Package:** Published via Trusted Publisher" >> $GITHUB_STEP_SUMMARY + echo "- **GitHub Release:** Created with auto-generated notes" >> $GITHUB_STEP_SUMMARY + + - name: Print failure message + if: failure() + run: | + echo "### :x: Release failed. Please check the logs above." >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index e2fa1e76..1cf7b7df 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -17,6 +17,9 @@ jobs: steps: - uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + - name: Install xmllint run: | apt-get update diff --git a/.github/workflows/update-plugin-docs.yml b/.github/workflows/update-plugin-docs.yml index a5b7040a..9db6ffb3 100644 --- a/.github/workflows/update-plugin-docs.yml +++ b/.github/workflows/update-plugin-docs.yml @@ -30,8 +30,16 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies - run: ./dev-setup.sh + run: bash -euo pipefail ./dev-setup.sh - name: Run plugin documentation generator run: | @@ -53,7 +61,7 @@ jobs: pre-commit run --files docs/PLUGIN_DOC.md README.md || true - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v8 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "docs: Update plugin documentation [automated]" diff --git a/.mypy.ini b/.mypy.ini index cf6c2344..b9a990fe 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -1,6 +1,23 @@ [mypy] -# Global mypy configuration -mypy_path = test/unit +explicit_package_bases = True + +[mypy-pydantic] +ignore_missing_imports = True + +[mypy-pydantic.*] +ignore_missing_imports = True + +[mypy-pytest] +ignore_missing_imports = True + +[mypy-serviceability_dummy_data] +ignore_missing_imports = True + +[mypy-instinct_shaped_engine] +ignore_missing_imports = True + +[mypy-nodescraper.connection.redfish.redfish_params] +ignore_errors = True [mypy-nodescraper.base.regexanalyzer] ignore_errors = True @@ -121,3 +138,6 @@ ignore_errors = True [mypy-test.unit.framework.test_cli] ignore_errors = True + +[mypy-test.unit.framework.test_plugin_execution_target] +ignore_errors = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85a64e4f..103c4391 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,8 @@ repos: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace + - id: mixed-line-ending + args: [--fix=lf] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.11.6 hooks: diff --git a/README.md b/README.md index dce86aef..4cfaa806 100644 --- a/README.md +++ b/README.md @@ -3,26 +3,36 @@ Node Scraper is a tool which performs automated data collection and analysis for system debug. For details on what data is collected and analyzed, see the [plugin reference table](docs/PLUGIN_DOC.md). ## Table of Contents -- [Installation](#installation) - - [Install from PyPI](#install-from-pypi) - - [Install from Source](#install-from-source) -- [CLI Usage](#cli-usage) - - [Execution Methods](#execution-methods) - - [Example: Remote Execution](#example-remote-execution) - - [Example: connection_config.json](#example-connection_configjson) - - [Subcommands](#subcommands) - - ['describe' subcommand](#describe-subcommand) - - ['run-plugins' sub command](#run-plugins-sub-command) - - ['gen-plugin-config' sub command](#gen-plugin-config-sub-command) - - ['compare-runs' subcommand](#compare-runs-subcommand) - - ['summary' sub command](#summary-sub-command) -- [Configs](#configs) - - [Global args](#global-args) - - [Plugin config: `--plugin-configs` command](#plugin-config---plugin-configs-command) - - [Reference config: `gen-reference-config` command](#reference-config-gen-reference-config-command) -- **Extending Node Scraper (integration & external plugins)** → See [EXTENDING.md](EXTENDING.md) -- **Full view of the plugins with the associated collectors & analyzers as well as the commands -invoked by collectors** -> See [docs/PLUGIN_DOC.md](docs/PLUGIN_DOC.md) +- [Node Scraper](#node-scraper) + - [Table of Contents](#table-of-contents) + - [Installation](#installation) + - [Install from PyPI](#install-from-pypi) + - [Install from Source](#install-from-source) + - [1. Install uv](#1-install-uv) + - [2. Virtual Environment (Optional)](#2-virtual-environment-optional) + - [3. Install from Source (Required)](#3-install-from-source-required) + - [4. Git Hooks (Optional)](#4-git-hooks-optional) + - [CLI Usage](#cli-usage) + - [Lazy plugin loading](#lazy-plugin-loading) + - [Execution Methods](#execution-methods) + - [Example: Remote Execution](#example-remote-execution) + - [Example: connection\_config.json](#example-connection_configjson) + - [Subcommands](#subcommands) + - [**'describe' subcommand**](#describe-subcommand) + - [**'run-plugins' sub command**](#run-plugins-sub-command) + - [**'gen-plugin-config' sub command**](#gen-plugin-config-sub-command) + - [**'compare-runs' subcommand**](#compare-runs-subcommand) + - [**'show-redfish-oem-allowable' subcommand**](#show-redfish-oem-allowable-subcommand) + - [**RedfishEndpointPlugin**](#redfishendpointplugin) + - [**'summary' sub command**](#summary-sub-command) + - [Configs](#configs) + - [Global args](#global-args) + - [Plugin config: **'--plugin-configs' command**](#plugin-config---plugin-configs-command) + - [Post-action plugins](#post-action-plugins) + - [Config structure](#config-structure) + - [Condition fields](#condition-fields) + - [Example: run OsPlugin if DmesgPlugin finds error-level events](#example-run-osplugin-if-dmesgplugin-finds-error-level-events) + - [Reference config: **'gen-reference-config' command**](#reference-config-gen-reference-config-command) ## Installation ### Install from PyPI @@ -32,6 +42,12 @@ Node Scraper is published on [PyPI](https://pypi.org/project/amd-node-scraper/) pip install amd-node-scraper ``` +With [uv](https://docs.astral.sh/uv/): + +```sh +uv pip install amd-node-scraper +``` + Use a virtual environment if you prefer. After installation, confirm the CLI is available: ```sh @@ -40,8 +56,8 @@ node-scraper --help ### Install from Source Node Scraper requires Python 3.9+ for installation. After cloning this repository, -call dev-setup.sh script with 'source'. This script creates an editable install of Node Scraper in -a python virtual environment and also configures the pre-commit hooks for the project. +run `dev-setup.sh` with `source`. This script uses [uv](https://docs.astral.sh/uv/) to create a +Python 3.9+ virtual environment, perform an editable install, and configure pre-commit hooks. ```sh source dev-setup.sh @@ -49,20 +65,31 @@ source dev-setup.sh Alternatively, follow these manual steps: -### 1. Virtual Environment (Optional) +### 1. Install uv ```sh -python3 -m venv venv +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +### 2. Virtual Environment (Optional) +```sh +uv venv venv --python 3.9 source venv/bin/activate ``` -On Debian/Ubuntu, you may need: `sudo apt install python3-venv` +On Debian/Ubuntu without uv, you can use `python3.9 -m venv venv` instead. -### 2. Install from Source (Required) +### 3. Install from Source (Required) ```sh -python3 -m pip install --editable .[dev] --upgrade +uv pip install -e ".[dev]" ``` This installs Node Scraper in editable mode with development dependencies. To verify: `node-scraper --help` -### 3. Git Hooks (Optional) +Equivalent using pip: + +```sh +python3 -m pip install --editable .[dev] --upgrade +``` + +### 4. Git Hooks (Optional) ```sh pre-commit install ``` @@ -82,6 +109,7 @@ usage: cli.py [-h] [--version] [--sys-name STRING] [--connection-config STRING] [--log-path STRING] [--log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}] [--no-console-log] [--gen-reference-config] [--skip-sudo] + [--lazy-load] {summary,run-plugins,describe,gen-plugin-config,compare-runs,show-redfish-oem-allowable} ... @@ -116,7 +144,7 @@ options: Comma-separated built-in names and/or plugin config JSON paths (e.g. --plugin- configs=NodeStatus,/path/c.json). Built-ins: - AllPlugins, NodeStatus (default: None) + AllIbPlugins, NodeStatus (default: None) --system-config STRING Path to system config json (default: None) --connection-config STRING @@ -135,9 +163,27 @@ options: ./reference_config.json. (default: False) --skip-sudo Skip plugins that require sudo permissions (default: False) + --lazy-load Import plugin modules only when they are needed + instead of at startup (default: False) ``` +### Lazy plugin loading + +By default node-scraper imports every plugin at startup this can be inefficient to run a single plugin run. +Lazy loading imports a plugin only when it is first needed, so runs that use a few plugins start faster and use less memory. + +```sh +node-scraper --lazy-load run-plugins OsPlugin +``` + +What to expect in lazy mode: +- Plugin names are read from the `nodescraper.plugins` entry points in `pyproject.toml` without importing them. Built-in plugins are listed there, and a plugin must be registered there to be available in lazy mode (see [external plugins](docs/node-scraper-external/README.md) for the format). After adding or renaming a plugin, reinstall the package (`pip install -e .`) to refresh the entry-point metadata. +- Only the plugins you name on the command line (or in a config) are imported. +- `describe plugin` lists every plugin from the entry points without importing any; `describe plugin ` imports just that plugin to show its description. `--help` still imports every plugin because it documents each one's arguments. +- A plugin that fails to import is logged and skipped instead of stopping node-scraper. +- Named configs from `--plugin-configs` are evaluated only when you select them, so recipes such as `AllIbPlugins` (which inspects every plugin) do not import everything at startup. + ### Execution Methods Node Scraper can operate in two modes: LOCAL and REMOTE, determined by the `--sys-location` argument. @@ -169,6 +215,8 @@ In-band (SSH) connection: } ``` +A sample is in `config/connection-config_inband.example.json`. Use `password` or `key_filename`. + Redfish (BMC) connection for Redfish-only plugins: ```json @@ -187,6 +235,55 @@ Redfish (BMC) connection for Redfish-only plugins: - `api_root` (optional): Redfish API path (e.g. `redfish/v1`). If omitted, the default `redfish/v1` is used. Override this when your BMC uses a different API version path. +OOB SSH plugins use this same single-host `RedfishConnectionManager` block and open SSH to that BMC. A sample is in `config/connection-config_oob.example.json`. + +#### Redfish multi-target + +Redfish plugins can collect from multiple BMCs concurrently. In-band plugins and OOB SSH plugins stay single-host. If this config has no top-level `host`, OOB SSH plugins are skipped. Add a top-level `host` when those plugins should still run against one BMC. + +```json +{ + "RedfishConnectionManager": { + "targets": [ + { + "target_key": "node-a", + "host": "bmc-node-a.example.com", + "username": "admin", + "password": "secret", + "use_https": true, + "verify_ssl": false, + "timeout_seconds": 30 + }, + { + "target_key": "node-b", + "host": "bmc-node-b.example.com", + "username": "admin", + "password": "secret", + "use_https": true, + "verify_ssl": false, + "timeout_seconds": 30 + } + ], + "max_workers": 32 + } +} +``` + +List each Redfish plugin once. Put the hosts on that plugin's connection manager `targets` list. `RedfishConnectionManager` is the direct BMC session used by `RedfishEndpointPlugin`, `RedfishInventoryPlugin`, and `RedfishOemDiagPlugin`. `RedfishSshProxyConnectionManager` is the SSH proxy used by `AmcRedfishInventoryPlugin`, `AmcRedfishEndpointPlugin`, and `AmcRedfishDiagPlugin`. Those two managers are separate lists, so a plugin compares and collects only the targets it connected. + +Targets are collected concurrently. Connect failures stay on the connection result. The plugin result shows collection and analysis for the targets that connected, and the run fails when every target fails to connect or analysis reports an error. + +- `targets`: list of per-target connection parameters. Each entry accepts the same fields as the single-target config plus an optional `target_key` (used as the result key; defaults to the host string). +- `max_workers` (optional): maximum concurrent collection threads. Defaults to `min(len(targets), 32)` and is capped at 32. + +Per-target results are written to `///`. Single-target results stay in `//`. The same layout is used for the analyzer directory and for the AMC SSH-proxy plugin. Inventory collection writes `redfishinventorydatamodel.json` in the collector directory. + +A ready-to-edit sample is in `config/connection-config_redfish_multi_target.example.json`. Replace the example hosts and password, then pass it with `--connection-config`. + +**Firmware comparison:** `RedfishInventoryPlugin` and `AmcRedfishInventoryPlugin` compare `FirmwareInventory` across the targets that plugin connected. `analysis_args.compare_firmware` defaults to `true`. Each member is compared by its published `Id`. A missing or empty `Version` is recorded as `N/A`. The warning also records `Status.Health`, `Status.State`, `SoftwareId`, `Updateable`, and `Oem.SVN`. A different version, or a member present on one connected target and absent on another, logs one `FW` warning per component. BMC firmware is not compared with AMC firmware. A target that failed to connect is excluded. One connected target does not warn. Set `compare_firmware` to `false` to skip the check. + +The console summary reports the warning count. Each component is in the analyzer `events.json`, and `result.json` has the summary. Collection profiles and the other inventory switches are in the [plugin reference](docs/PLUGIN_DOC.md). + **Notes:** - If using SSH keys, specify `key_filename` instead of `password`. - The remote user must have permissions to run the requested plugins and access required files. If needed, use the `--skip-sudo` argument to skip plugins requiring sudo. @@ -449,9 +546,11 @@ Use a plugin config that points at your LogService and lists the types to collec The RedfishEndpointPlugin collects Redfish URIs (GET responses) and optionally runs checks on the returned JSON. It requires a Redfish connection config (same as RedfishOemDiagPlugin). +**Multi-target support:** `RedfishEndpointPlugin` collects from each BMC in the `targets` list at the same time. Use the [Redfish multi-target](#redfish-multi-target) connection config. The same `uris` and `checks` apply to every target. Per-target results are written to `redfish_endpoint_plugin/redfish_endpoint_collector//` under the run log directory. A BMC that cannot be reached is reported as a warning when another target succeeds. + **How to run** -1. Create a connection config (e.g. `connection-config.json`) with `RedfishConnectionManager` and your BMC host, credentials, and API root. +1. Create a connection config (e.g. `connection-config.json`) with `RedfishConnectionManager` and your BMC host (single target) or `targets` list (multi-target). 2. Create a plugin config with `uris` to collect and optional `checks` for analysis (see example below). For example save as `plugin_config_redfish_endpoint.json`. 3. Run: ```sh @@ -498,6 +597,25 @@ The RedfishEndpointPlugin collects Redfish URIs (GET responses) and optionally r **`analysis_args`** - **`checks`**: Optional. Map of URI to expected values or constraints for analysis. Supports exact match (e.g. `"PowerState": "On"`), `anyOf`, `min`/`max`, etc. +#### **RedfishEventLogPlugin** + +The RedfishEventLogPlugin reads every entry from a Redfish LogService (for example an EventLog) and reports Critical and Warning entries. It requires a Redfish connection config and supports [Redfish multi-target](#redfish-multi-target). Use `AmcRedfishEventLogPlugin` for the AMC EventLog via the SSH proxy. + +The collector reads `/Entries` and pages with `$skip` until `Members@odata.count` is reached. Each page is saved as an artifact, and entries are stored newest first by `Created`. It uses `$skip` instead of `Members@odata.nextLink` because some services return the same `nextLink` on every page. + +The analyzer reports Critical entries as errors and Warning entries as warnings. Entries with the same `Severity` and `MessageId` are reported once, with a count and the newest message. + +**`collection_args`** +- **`log_service_path`**: Redfish path to the LogService. Entries are read from `/Entries`. +- **`max_entries`**: Optional. Keep only the newest N entries. All pages are still read. +- **`max_pages`**: Optional (default `100`). Safety cap on the number of `$skip` pages requested. +- **`clear_logs_after_collection`**: Optional (default `false`). POST `LogService.ClearLog` after a successful collection. + +**`analysis_args`** +- **`since`**: Optional ISO 8601 time. Only entries created at or after it are analyzed. +- **`message_id_include`**: Optional list of regexes. Only entries whose `MessageId` matches one are analyzed. +- **`message_id_exclude`**: Optional list of regexes. Entries whose `MessageId` matches any are skipped. + #### **'summary' sub command** The 'summary' subcommand can be used to combine results from multiple runs of node-scraper to a @@ -551,8 +669,8 @@ Below is an example that skips sudo requiring plugins and disables analysis. #### Plugin config: **'--plugin-configs' command** A plugin config can be used to compare the system data against the config specifications. -Built-in configs include **NodeStatus** (a subset of plugins) and **AllPlugins** (runs every -registered plugin with default arguments—useful for generating a reference config from the full system). +Built-in configs include **NodeStatus** (a subset of plugins) and **AllIbPlugins** (runs every +registered in-band plugin with default arguments—useful for generating a reference config from the full system). **NodeStatus plus additional plugins** — built-in configs merge with plugins named after `run-plugins`. Values are comma-separated; pass as **`--plugin-configs=…`** or **`--plugin-configs` …** (same as other @@ -623,14 +741,106 @@ Here is an example of a comprehensive plugin config that specifies analyzer args } ``` +#### Post-action plugins + +Post-action plugins run automatically **after all primary plugins have completed**, but only when +one or more configurable conditions are met. They are defined in the same plugin config JSON as +the primary plugins, under the `post_action_plugins` key. + +**Use cases:** +- Run a follow-up data-collection plugin only when a primary plugin detects errors +- Trigger remediation or additional diagnostic steps based on specific event categories or severities + +##### Config structure + +```json +{ + "plugins": { ... }, + "post_action_plugins": [ + { + "plugin": "", + "plugin_args": { ... }, + "conditions": [ + { "": "", ... }, + { "": "", ... } + ] + } + ] +} +``` + +- **`plugin`** — the name of the plugin to run (same registry name used in the `plugins` dict). +- **`plugin_args`** — arguments forwarded to the plugin's `run()` method (same shape as a normal + `plugins` entry, e.g. `collection`, `analysis`, `collection_args`, `analysis_args`). +- **`conditions`** — a list of condition objects. The post-action fires if **any** condition in the + list is satisfied (**OR** semantics). Within a single condition all specified fields must match + (**AND** semantics); unspecified fields are ignored. + +##### Condition fields + +All fields are optional. A condition with no fields specified matches any result. + +| Field | Type | Description | +|---|---|---| +| `plugin` | string | If set, only the result whose `source` matches this name is inspected. If omitted, all primary results are candidates. | +| `status` | string | The primary plugin's `ExecutionStatus` must be **≥** this value. Accepted values (in ascending order): `OK`, `WARNING`, `ERROR`, `EXECUTION_FAILURE`. | +| `event_category` | string | At least one event (from analysis or collection) must have this category. Normalised to uppercase with spaces/hyphens converted to underscores before comparison. | +| `event_priority` | string | At least one event's priority must be **≥** this value. Accepted values: `INFO`, `WARNING`, `ERROR`, `CRITICAL`. | +| `event_description_contains` | string | At least one event's description must contain this substring (case-sensitive). | + +##### Example: run OsPlugin if DmesgPlugin finds error-level events + +```json +{ + "name": "DmesgWithOsPostAction", + "desc": "Run DmesgPlugin; if any error-level event is found, run OsPlugin to capture OS state.", + "global_args": {}, + "plugins": { + "DmesgPlugin": { + "collection": true, + "analysis": true + } + }, + "result_collators": {}, + "post_action_plugins": [ + { + "plugin": "OsPlugin", + "plugin_args": { + "collection": true, + "analysis": true + }, + "conditions": [ + { + "plugin": "DmesgPlugin", + "event_priority": "ERROR" + } + ] + } + ] +} +``` + +Save to a file and pass it with `--plugin-configs`: + +```sh +node-scraper --plugin-configs=plugin_config_dmesg_os_post_action.json +``` + +Post-action plugin results are included in the same result list as primary plugins — they appear +in the console summary table, the `nodescraper.csv` output, and any result hooks. + +> **Note:** Post-action plugins run before connections are closed, so they have access to the same +> live connection managers as primary plugins. Post-action plugins cannot enqueue additional +> plugins into the primary queue. + #### Reference config: **'gen-reference-config' command** This command can be used to generate a reference config that is populated with current system configurations. Plugins that use analyzer args (where applicable) will be populated with system data. -**Run all registered plugins (AllPlugins config):** +**Run all registered in-band plugins (AllIbPlugins config):** ```sh -node-scraper --plugin-configs=AllPlugins +node-scraper --plugin-configs=AllIbPlugins ``` diff --git a/SECURITY.md b/SECURITY.md index 89d7b1ba..6e7fffa5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,34 +1,34 @@ -# Security Policy - -## Supported Versions - -We provide security updates for the **latest released version** and for the **default branch**. - -| Version | Supported | -| ------- | ------------------ | -| Default branch (`main` / `development`) | :white_check_mark: | -| Latest PyPI release (`amd-node-scraper`) | :white_check_mark: | -| Older releases | :x: | - -## Reporting a Vulnerability - -Please **do not** open a public issue for security vulnerabilities. - -### Preferred: private vulnerability report (GitHub) - -If enabled for this repository, use GitHub’s private vulnerability reporting: - -- `https://github.com/amd/node-scraper/security/advisories/new` - -### What to include - -- A clear description of the issue and potential impact -- Steps to reproduce (or a proof-of-concept, if safe to share) -- The output of `node-scraper --version` (or a git commit hash) -- Any relevant configuration details (please redact secrets/credentials/hostnames/IPs) - -### What to expect - -- We will acknowledge receipt as soon as practical. -- If the report is accepted, we’ll work on a fix and coordinate disclosure timing. -- If the report is declined (e.g., not a security issue), we’ll explain why. +# Security Policy + +## Supported Versions + +We provide security updates for the **latest released version** and for the **default branch**. + +| Version | Supported | +| ------- | ------------------ | +| Default branch (`main` / `development`) | :white_check_mark: | +| Latest PyPI release (`amd-node-scraper`) | :white_check_mark: | +| Older releases | :x: | + +## Reporting a Vulnerability + +Please **do not** open a public issue for security vulnerabilities. + +### Preferred: private vulnerability report (GitHub) + +If enabled for this repository, use GitHub’s private vulnerability reporting: + +- `https://github.com/amd/node-scraper/security/advisories/new` + +### What to include + +- A clear description of the issue and potential impact +- Steps to reproduce (or a proof-of-concept, if safe to share) +- The output of `node-scraper --version` (or a git commit hash) +- Any relevant configuration details (please redact secrets/credentials/hostnames/IPs) + +### What to expect + +- We will acknowledge receipt as soon as practical. +- If the report is accepted, we’ll work on a fix and coordinate disclosure timing. +- If the report is declined (e.g., not a security issue), we’ll explain why. diff --git a/config/connection-config_inband.example.json b/config/connection-config_inband.example.json new file mode 100644 index 00000000..cc05b2ff --- /dev/null +++ b/config/connection-config_inband.example.json @@ -0,0 +1,8 @@ +{ + "InBandConnectionManager": { + "hostname": "host.example.com", + "port": 22, + "username": "admin", + "password": "placeholder" + } +} diff --git a/config/connection-config_oob.example.json b/config/connection-config_oob.example.json new file mode 100644 index 00000000..1edd627b --- /dev/null +++ b/config/connection-config_oob.example.json @@ -0,0 +1,12 @@ +{ + "RedfishConnectionManager": { + "host": "bmc.example.com", + "port": 443, + "username": "admin", + "password": "placeholder", + "use_https": true, + "verify_ssl": false, + "timeout_seconds": 30, + "api_root": "redfish/v1" + } +} diff --git a/config/connection-config_redfish_multi_target.example.json b/config/connection-config_redfish_multi_target.example.json new file mode 100644 index 00000000..34cb845e --- /dev/null +++ b/config/connection-config_redfish_multi_target.example.json @@ -0,0 +1,25 @@ +{ + "RedfishConnectionManager": { + "max_workers": 2, + "targets": [ + { + "target_key": "node-a", + "host": "bmc-node-a.example.com", + "username": "admin", + "password": "placeholder", + "use_https": true, + "verify_ssl": false, + "timeout_seconds": 30 + }, + { + "target_key": "node-b", + "host": "bmc-node-b.example.com", + "username": "admin", + "password": "placeholder", + "use_https": true, + "verify_ssl": false, + "timeout_seconds": 30 + } + ] + } +} diff --git a/dev-setup.sh b/dev-setup.sh index c34df108..32c90e3f 100755 --- a/dev-setup.sh +++ b/dev-setup.sh @@ -1,16 +1,29 @@ #!/usr/bin/env bash +set -euo pipefail -# Create venv if not already present -if [ ! -d "venv" ]; then - python3 -m venv venv +if ! which uv >/dev/null 2>&1; then + if ! which curl >/dev/null 2>&1; then + wget -qO- https://astral.sh/uv/install.sh | sh + else + curl -LsSf https://astral.sh/uv/install.sh | sh + fi + export PATH="$HOME/.local/bin:$PATH" +fi + +# Use system certificate store so uv works behind corporate SSL proxies +export UV_SYSTEM_CERTS=1 + +# Create venv if not already present or if Python version is below 3.9 +if [ ! -d "venv" ] || ! venv/bin/python -c 'import sys; exit(0 if sys.version_info >= (3, 9) else 1)' 2>/dev/null; then + uv venv --python python3 --no-python-downloads --seed venv fi # Activate the desired venv source venv/bin/activate -python3 -m pip install --editable .[dev] --upgrade +uv pip install -e . --group dev # Only install pre-commit hooks if not in CI environment -if [ -z "$CI" ]; then +if [ -z "${CI:-}" ]; then pre-commit install fi diff --git a/docs/PLUGIN_DOC.md b/docs/PLUGIN_DOC.md index cc34c3ef..63002024 100644 --- a/docs/PLUGIN_DOC.md +++ b/docs/PLUGIN_DOC.md @@ -5,32 +5,34 @@ | Plugin | Collection | Analyzer Args | Collection Args | DataModel | Collector | Analyzer | | --- | --- | --- | --- | --- | --- | --- | | GenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_\* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | -| AmdSmiPlugin | bad-pages
firmware --json
list --json
metric -g all
partition --json
process --json
ras --cper --folder={folder}
ras --afid --cper-file {cper_file}
static -g all --json
static -g {gpu_id} --json
topology
version --json
xgmi -l
xgmi -m | **Analyzer Args:**
- `check_static_data`: bool — If True, run static data checks (e.g. driver version, partition mode).
- `expected_gpu_processes`: Optional[int] — Expected number of GPU processes.
- `expected_max_power`: Optional[int] — Expected maximum power value (e.g. watts).
- `expected_power_management`: Optional[str] — Expected amd-smi metric power_management value per GPU (e.g. DISABLED for active/full power, ENABLED for power-manage...
- `expected_driver_version`: Optional[str] — Expected AMD driver version string.
- `expected_memory_partition_mode`: Optional[str] — Expected memory partition mode (e.g. sp3, dp).
- `expected_compute_partition_mode`: Optional[str] — Expected compute partition mode.
- `expected_firmware_versions`: Optional[dict[str, str]] — Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).
- `l0_to_recovery_count_error_threshold`: Optional[int] — L0-to-recovery count above which an error is raised.
- `l0_to_recovery_count_warning_threshold`: Optional[int] — L0-to-recovery count above which a warning is raised.
- `vendorid_ep`: Optional[str] — Expected endpoint vendor ID (e.g. for PCIe).
- `vendorid_ep_vf`: Optional[str] — Expected endpoint VF vendor ID.
- `devid_ep`: Optional[str] — Expected endpoint device ID.
- `devid_ep_vf`: Optional[str] — Expected endpoint VF device ID.
- `sku_name`: Optional[str] — Expected SKU name string for GPU.
- `expected_xgmi_speed`: Optional[list[float]] — Expected xGMI speed value(s) (e.g. link rate).
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for time-windowed analysis.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for time-windowed analysis. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `analysis_firmware_ids`: Optional[list[str]] — amd-smi fw_id values to record in analysis_ref.firmware_versions
- `cper_file_path`: Optional[str] — Path to CPER folder or file for RAS AFID collection (ras --afid --cper-file). | [AmdSmiDataModel](#AmdSmiDataModel-Model) | [AmdSmiCollector](#Collector-Class-AmdSmiCollector) | [AmdSmiAnalyzer](#Data-Analyzer-Class-AmdSmiAnalyzer) | -| BiosPlugin | sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'
wmic bios get SMBIOSBIOSVersion /Value | **Analyzer Args:**
- `exp_bios_version`: list[str] — Expected BIOS version(s) to match against collected value (str or list).
- `regex_match`: bool — If True, match exp_bios_version as regex; otherwise exact match. | - | [BiosDataModel](#BiosDataModel-Model) | [BiosCollector](#Collector-Class-BiosCollector) | [BiosAnalyzer](#Data-Analyzer-Class-BiosAnalyzer) | +| AmdSmiFlavorPlugin | bad-pages
metric --cpu-dimm-pow-consumption {dimm_addr}
metric --cpu-dimm-temp-range-rate {dimm_addr}
metric --cpu-dimm-thermal-sensor {dimm_addr}
metric --cpu-io-bandwidth {bw} {link_id}
metric --cpu-lclk-dpm-level {nbio_id}
metric --cpu-svi3-vr-controller-temp 0
metric --cpu-svi3-vr-controller-temp 1 {rail_index}
metric --cpu-xgmi-bandwidth {bw} {link_id}
fabric
firmware
list
metric -g all
metric --core all
metric --cpu all
node
partition
process
ras --cper --folder={folder}
ras --afid --cper-file {cper_file}
static -g all
static -g {gpu_id}
topology
version
xgmi -l
xgmi -m | **Analyzer Args:**
- `check_static_data`: bool — If True, run static data checks (e.g. driver version, partition mode).
- `expected_gpu_processes`: Optional[int] — Expected number of GPU processes.
- `expected_max_power`: Optional[int] — Expected maximum power value (e.g. watts).
- `power`: Optional[nodescraper.plugins.inband.amdsmi.analyzer_args.PowerConfig] — GPU power-cap consistency policy.
- `expected_power_management`: Optional[str] — Expected amd-smi metric power_management value per GPU (e.g. DISABLED for active/full power, ENABLED for power-manage...
- `expected_driver_version`: Optional[str] — Expected AMD driver version string.
- `expected_memory_partition_mode`: Optional[str] — Expected memory partition mode (e.g. sp3, dp).
- `expected_compute_partition_mode`: Optional[str] — Expected compute partition mode.
- `expected_firmware_versions`: Optional[dict[str, str]] — Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).
- `gpu_memory`: Optional[nodescraper.plugins.inband.amdsmi.analyzer_args.GpuMemoryConfig] — Minimum free VRAM threshold to validate for each GPU.
- `check_xgmi_or_peer_links_status`: bool — Check XGMI or peer-link status for each GPU.
- `l0_to_recovery_count_error_threshold`: Optional[int] — L0-to-recovery count above which an error is raised.
- `l0_to_recovery_count_warning_threshold`: Optional[int] — L0-to-recovery count above which a warning is raised.
- `vendorid_ep`: Optional[str] — Expected endpoint vendor ID (e.g. for PCIe).
- `vendorid_ep_vf`: Optional[str] — Expected endpoint VF vendor ID.
- `devid_ep`: Optional[str] — Expected endpoint device ID.
- `devid_ep_vf`: Optional[str] — Expected endpoint VF device ID.
- `sku_name`: Optional[str] — Expected SKU name string for GPU.
- `expected_xgmi_speed`: Optional[list[float]] — Expected xGMI speed value(s) (e.g. link rate).
- `expected_accel_state`: str — Expected amd-smi fabric accel_state value.
- `expected_fabric_type`: str — Expected amd-smi fabric fabric_type value.
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for time-windowed analysis.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for time-windowed analysis. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `analysis_firmware_ids`: Optional[list[str]] — amd-smi fw_id values to record in analysis_ref.firmware_versions
- `cper_file_path`: Optional[str] — Path to CPER folder or file for RAS AFID collection (ras --afid --cper-file). | [AmdSmiFlavorDataModel](#AmdSmiFlavorDataModel-Model) | [AmdSmiFlavorCollector](#Collector-Class-AmdSmiFlavorCollector) | [AmdSmiAnalyzer](#Data-Analyzer-Class-AmdSmiAnalyzer) | +| AmdSmiPlugin | bad-pages
metric --cpu-dimm-pow-consumption {dimm_addr}
metric --cpu-dimm-temp-range-rate {dimm_addr}
metric --cpu-dimm-thermal-sensor {dimm_addr}
metric --cpu-io-bandwidth {bw} {link_id}
metric --cpu-lclk-dpm-level {nbio_id}
metric --cpu-svi3-vr-controller-temp 0
metric --cpu-svi3-vr-controller-temp 1 {rail_index}
metric --cpu-xgmi-bandwidth {bw} {link_id}
fabric
firmware
list
metric -g all
metric --core all
metric --cpu all
node
partition
process
ras --cper --folder={folder}
ras --afid --cper-file {cper_file}
static -g all
static -g {gpu_id}
topology
version
xgmi -l
xgmi -m | **Analyzer Args:**
- `check_static_data`: bool — If True, run static data checks (e.g. driver version, partition mode).
- `expected_gpu_processes`: Optional[int] — Expected number of GPU processes.
- `expected_max_power`: Optional[int] — Expected maximum power value (e.g. watts).
- `power`: Optional[nodescraper.plugins.inband.amdsmi.analyzer_args.PowerConfig] — GPU power-cap consistency policy.
- `expected_power_management`: Optional[str] — Expected amd-smi metric power_management value per GPU (e.g. DISABLED for active/full power, ENABLED for power-manage...
- `expected_driver_version`: Optional[str] — Expected AMD driver version string.
- `expected_memory_partition_mode`: Optional[str] — Expected memory partition mode (e.g. sp3, dp).
- `expected_compute_partition_mode`: Optional[str] — Expected compute partition mode.
- `expected_firmware_versions`: Optional[dict[str, str]] — Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).
- `gpu_memory`: Optional[nodescraper.plugins.inband.amdsmi.analyzer_args.GpuMemoryConfig] — Minimum free VRAM threshold to validate for each GPU.
- `check_xgmi_or_peer_links_status`: bool — Check XGMI or peer-link status for each GPU.
- `l0_to_recovery_count_error_threshold`: Optional[int] — L0-to-recovery count above which an error is raised.
- `l0_to_recovery_count_warning_threshold`: Optional[int] — L0-to-recovery count above which a warning is raised.
- `vendorid_ep`: Optional[str] — Expected endpoint vendor ID (e.g. for PCIe).
- `vendorid_ep_vf`: Optional[str] — Expected endpoint VF vendor ID.
- `devid_ep`: Optional[str] — Expected endpoint device ID.
- `devid_ep_vf`: Optional[str] — Expected endpoint VF device ID.
- `sku_name`: Optional[str] — Expected SKU name string for GPU.
- `expected_xgmi_speed`: Optional[list[float]] — Expected xGMI speed value(s) (e.g. link rate).
- `expected_accel_state`: str — Expected amd-smi fabric accel_state value.
- `expected_fabric_type`: str — Expected amd-smi fabric fabric_type value.
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for time-windowed analysis.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for time-windowed analysis. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `analysis_firmware_ids`: Optional[list[str]] — amd-smi fw_id values to record in analysis_ref.firmware_versions
- `cper_file_path`: Optional[str] — Path to CPER folder or file for RAS AFID collection (ras --afid --cper-file). | [AmdSmiDataModel](#AmdSmiDataModel-Model) | [AmdSmiCollector](#Collector-Class-AmdSmiCollector) | [AmdSmiAnalyzer](#Data-Analyzer-Class-AmdSmiAnalyzer) | +| BiosPlugin | sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'
smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1
wmic bios get SMBIOSBIOSVersion /Value | **Analyzer Args:**
- `exp_bios_version`: list[str] — Expected BIOS version(s) to match against collected value (str or list).
- `regex_match`: bool — If True, match exp_bios_version as regex; otherwise exact match. | - | [BiosDataModel](#BiosDataModel-Model) | [BiosCollector](#Collector-Class-BiosCollector) | [BiosAnalyzer](#Data-Analyzer-Class-BiosAnalyzer) | | CmdlinePlugin | cat /proc/cmdline | **Analyzer Args:**
- `required_cmdline`: Union[str, List] — Command-line parameters that must be present (e.g. 'pci=bfsort').
- `banned_cmdline`: Union[str, List] — Command-line parameters that must not be present.
- `os_overrides`: Dict[str, nodescraper.plugins.inband.cmdline.cmdlineconfig.OverrideConfig] — Per-OS overrides for required_cmdline and banned_cmdline (keyed by OS identifier).
- `platform_overrides`: Dict[str, nodescraper.plugins.inband.cmdline.cmdlineconfig.OverrideConfig] — Per-platform overrides for required_cmdline and banned_cmdline (keyed by platform). | - | [CmdlineDataModel](#CmdlineDataModel-Model) | [CmdlineCollector](#Collector-Class-CmdlineCollector) | [CmdlineAnalyzer](#Data-Analyzer-Class-CmdlineAnalyzer) | -| DeviceEnumerationPlugin | powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count"
lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l
powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count"
lscpu
lshw
lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l
powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count" | **Analyzer Args:**
- `cpu_count`: Optional[list[int]] — Expected CPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `gpu_count`: Optional[list[int]] — Expected GPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `vf_count`: Optional[list[int]] — Expected virtual function count(s); pass as int or list of ints. Analysis passes if actual is in list. | - | [DeviceEnumerationDataModel](#DeviceEnumerationDataModel-Model) | [DeviceEnumerationCollector](#Collector-Class-DeviceEnumerationCollector) | [DeviceEnumerationAnalyzer](#Data-Analyzer-Class-DeviceEnumerationAnalyzer) | -| DimmPlugin | sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\|None\|Module" | grep Size' 2>/dev/null
dmidecode
wmic memorychip get Capacity | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `skip_sudo`: bool — If True, do not use sudo when running dmidecode or wmic for memory info. | [DimmDataModel](#DimmDataModel-Model) | [DimmCollector](#Collector-Class-DimmCollector) | - | +| DeviceEnumerationPlugin | esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'
powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count"
lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l
powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count"
lscpu
lshw
esxcli hardware pci list | grep -iE '^ \*Device ID: 0x0\*{device_id}([^0-9a-f]|$)' | wc -l
lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l
powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count" | **Analyzer Args:**
- `cpu_count`: Optional[list[int]] — Expected CPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `gpu_count`: Optional[list[int]] — Expected GPU count(s); pass as int or list of ints. Analysis passes if actual is in list.
- `vf_count`: Optional[list[int]] — Expected virtual function count(s); pass as int or list of ints. Analysis passes if actual is in list. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `devid_ep`: Optional[int] — Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi device counting.
- `devid_ep_vf`: Optional[int] — Expected GPU VF PCI device ID (int) for ESXi VF counting. | [DeviceEnumerationDataModel](#DeviceEnumerationDataModel-Model) | [DeviceEnumerationCollector](#Collector-Class-DeviceEnumerationCollector) | [DeviceEnumerationAnalyzer](#Data-Analyzer-Class-DeviceEnumerationAnalyzer) | +| DimmPlugin | dmidecode -q --type 17
dmidecode
wmic memorychip get /format:csv | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `skip_sudo`: bool — If True, do not use sudo when running dmidecode or wmic for memory info. | [DimmDataModel](#DimmDataModel-Model) | [DimmCollector](#Collector-Class-DimmCollector) | - | | DkmsPlugin | dkms status
dkms --version | **Analyzer Args:**
- `dkms_status`: Union[str, list] — Expected dkms status string(s) to match (e.g. 'amd/1.0.0'). At least one of dkms_status or dkms_version required.
- `dkms_version`: Union[str, list] — Expected dkms version string(s) to match. At least one of dkms_status or dkms_version required.
- `regex_match`: bool — If True, match dkms_status and dkms_version as regex; otherwise exact match. | - | [DkmsDataModel](#DkmsDataModel-Model) | [DkmsCollector](#Collector-Class-DkmsCollector) | [DkmsAnalyzer](#Data-Analyzer-Class-DkmsAnalyzer) | -| DmesgPlugin | dmesg --time-format iso -x
ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...`
**Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_unknown_dmesg_errors`: Optional[bool] — If True, treat unknown/unmatched dmesg error lines as failures.
- `exclude_category`: Optional[set[str]] — Set of error categories to exclude from analysis.
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one (for rate limiting).
- `num_timestamps`: int — Number of timestamps to include per event in output.
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.
- `priority_override_rules`: Optional[list[dict]] — Rules to override the priority of matched ErrorRegex objects. Each rule is a dict where all keys except 'new_priority...
- `mce_threshold`: Optional[int] — When set, raise ERROR if correctable MCE/RAS error count for any component (CPU, GPU BDF/block, etc.) reaches or exce...
- `ignore_match_rules`: Optional[list[dict[str, object]]] — Rules that skip regex matches during analysis. Each rule may use line_regex, match_regex, message, and/or mce_banks.... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg\*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | +| DmesgPlugin | dmesg --time-format iso -x
cat /var/log/vmkernel.log
ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true
ls -1 /var/log/vmkernel.\* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true | **Built-in Regexes:**
- Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)`
- I/O Page Fault: `IO_PAGE_FAULT`
- Kernel Panic: `\bkernel panic\b.*`
- SQ Interrupt: `sq_intr`
- SRAM ECC: `sram_ecc.*`
- Failed to load driver. IP hardware init error.: `\[amdgpu\]\] \*ERROR\* hw_init of IP block.*`
- Failed to load driver. IP software init error.: `\[amdgpu\]\] \*ERROR\* sw_init of IP block.*`
- Real Time throttling activated: `sched: RT throttling activated.*`
- RCU preempt detected stalls: `rcu_preempt detected stalls.*`
- RCU preempt self-detected stall: `rcu_preempt self-detected stall.*`
- QCM fence timeout: `qcm fence wait loop timeout.*`
- General protection fault: `(?:[\w-]+(?:\[[0-9.]+\])?\s+)?general protectio...`
- Segmentation fault: `(?:segfault.*in .*\[)|(?:[Ss]egmentation [Ff]au...`
- Failed to disallow cf state: `amdgpu: Failed to disallow cf state.*`
- Failed to terminate tmr: `\*ERROR\* Failed to terminate tmr.*`
- Suspend of IP block failed: `\*ERROR\* suspend of IP block <\w+> failed.*`
- amdgpu Page Fault: `(amdgpu \w{4}:\w{2}:\w{2}\.\w:\s+amdgpu:\s+\[\S...`
- Page Fault: `page fault for address.*`
- Fatal error during GPU init: `(?:amdgpu)(.*Fatal error during GPU init)|(Fata...`
- PCIe AER Error Status: `(pcieport [\w:.]+: AER: aer_status:[^\n]*(?:\n[...`
- PCIe AER Correctable Error Status: `(.*aer_cor_status: 0x[0-9a-fA-F]+, aer_cor_mask...`
- PCIe AER Uncorrectable Error Status: `(.*aer_uncor_status: 0x[0-9a-fA-F]+, aer_uncor_...`
- PCIe AER Uncorrectable Error Severity with TLP Header: `(.*aer_uncor_severity: 0x[0-9a-fA-F]+.*)(\n.*TL...`
- Failed to read journal file: `Failed to read journal file.*`
- Journal file corrupted or uncleanly shut down: `journal corrupted or uncleanly shut down.*`
- ACPI BIOS Error: `ACPI BIOS Error`
- ACPI Error: `ACPI Error`
- Filesystem corrupted!: `EXT4-fs error \(device .*\):`
- Error in buffered IO, check filesystem integrity: `(Buffer I\/O error on dev)(?:ice)? (\w+)`
- PCIe card no longer present: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- PCIe Link Down: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(Slot\(...`
- Mismatched clock configuration between PCIe device and host: `pcieport (\w+:\w+:\w+\.\w+):\s+(\w+):\s+(curren...`
- RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- RAS Uncorrectable Error: `(\d+ new uncorrectable hardware errors detected...`
- RAS Correctable Error: `(\d+ new correctable hardware errors detected i...`
- RAS ECC Fatal Error: `(GPU detected ECC Fatal Error\.)`
- GPU Reset: `(Issuing Whole GPU reset\.)`
- RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...`
- GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- ACA Error: `(Accelerator Check Architecture[^\n]*)(?:\n[^\n...`
- MCE Corrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- MCE Uncorrected Error: `\[Hardware Error\]:[^\n]*MC\d+_STATUS\[[^\]]*\|...`
- Mode 2 Reset Failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)? (...`
- RAS Corrected Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....`
- SGX Error: `x86/cpu: SGX disabled by BIOS`
- MMP Error: `Failed to load MMP firmware qat_4xxx_mmp.bin`
- GPU Throttled: `amdgpu \w{4}:\w{2}:\w{2}.\w: amdgpu: WARN: GPU ...`
- RAS Poison Consumed: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- RAS Poison created: `amdgpu[ 0-9a-fA-F:.]+:(?:\s*amdgpu:)?\s+(?:{\d+...`
- Bad page threshold exceeded: `(amdgpu: Saved bad pages (\d+) reaches threshol...`
- RAS Hardware Error: `Hardware error from APEI Generic Hardware Error...`
- Error Address: `Error Address.*(?:\s.*)`
- RAS EDR Event: `EDR: EDR event received`
- DPC Event: `DPC: .*`
- LNet: ko2iblnd has no matching interfaces: `(?:\[[^\]]+\]\s*)?LNetError:.*ko2iblnd:\s*No ma...`
- LNet: Error starting up LNI: `(?:\[[^\]]+\]\s*)?LNetError:\s*.*Error\s*-?\d+\...`
- Lustre: network initialisation failed: `LustreError:.*ptlrpc_init_portals\(\).*network ...`
**Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_unknown_dmesg_errors`: Optional[bool] — If True, treat unknown/unmatched dmesg error lines as failures.
- `exclude_category`: Optional[set[str]] — Set of error categories to exclude from analysis.
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one (for rate limiting).
- `num_timestamps`: int — Number of timestamps to include per event in output.
- `error_regex`: Union[list[nodescraper.base.regexanalyzer.ErrorRegex], list[dict], NoneType] — Custom error regex patterns; each item can be ErrorRegex or dict with category/pattern.
- `priority_override_rules`: Optional[list[dict]] — Rules to override the priority of matched ErrorRegex objects. Each rule is a dict where all keys except 'new_priority...
- `mce_threshold`: Optional[int] — When set, raise ERROR if correctable MCE/RAS error count for any component (CPU, GPU BDF/block, etc.) reaches or exce...
- `ignore_match_rules`: Optional[list[dict[str, object]]] — Rules that skip regex matches during analysis. Each rule may use line_regex, match_regex, message, and/or mce_banks.... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `collect_rotated_logs`: bool — If True, also collect rotated dmesg log files from /var/log/dmesg\*.
- `skip_sudo`: bool — If True, do not use sudo when running dmesg or listing log files.
- `log_dmesg_data`: bool — If True, log the collected dmesg output in artifacts. | [DmesgData](#DmesgData-Model) | [DmesgCollector](#Collector-Class-DmesgCollector) | [DmesgAnalyzer](#Data-Analyzer-Class-DmesgAnalyzer) | | FabricsPlugin | lspci | grep -i cassini
lsmod | grep cxi
cxi_stat
ibstat
ibv_devinfo
ls -l /sys/class/infiniband/\*/device/net
fi_info -p cxi
mst start
mst status -v
ip link show
ofed_info -s | - | - | [FabricsDataModel](#FabricsDataModel-Model) | [FabricsCollector](#Collector-Class-FabricsCollector) | - | | JournalPlugin | journalctl --no-pager --system --output=short-iso
journalctl --no-pager --system --output=json | **Analyzer Args:**
- `analysis_range_start`: Optional[datetime.datetime] — Start of time range for analysis (ISO format). Only events on or after this time are analyzed.
- `analysis_range_end`: Optional[datetime.datetime] — End of time range for analysis (ISO format). Only events before this time are analyzed.
- `check_priority`: Optional[int] — Check against journal log priority (0=emergency..7=debug). If an entry has priority <= check_priority, an ERROR event...
- `group`: bool — If True, group entries that have the same priority and message. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `boot`: Optional[int] — Optional boot ID to limit journal collection to a specific boot. | [JournalData](#JournalData-Model) | [JournalCollector](#Collector-Class-JournalCollector) | [JournalAnalyzer](#Data-Analyzer-Class-JournalAnalyzer) | | KernelPlugin | sh -c 'uname -a'
sh -c 'cat /proc/sys/kernel/numa_balancing'
wmic os get Version /Value | **Analyzer Args:**
- `exp_kernel`: Union[str, list] — Expected kernel version string(s) to match (e.g. from uname -a).
- `exp_numa`: Optional[int] — Expected value for kernel.numa_balancing (e.g. 0 or 1).
- `regex_match`: bool — If True, match exp_kernel as regex; otherwise exact match. | - | [KernelDataModel](#KernelDataModel-Model) | [KernelCollector](#Collector-Class-KernelCollector) | [KernelAnalyzer](#Data-Analyzer-Class-KernelAnalyzer) | | KernelModulePlugin | cat /proc/modules
modinfo amdgpu
wmic os get Version /Value | **Analyzer Args:**
- `kernel_modules`: dict[str, dict] — Expected kernel module name -> {version, etc.}. Analyzer checks collected modules match.
- `regex_filter`: list[str] — List of regex patterns to filter which collected modules are checked (default: amd). | - | [KernelModuleDataModel](#KernelModuleDataModel-Model) | [KernelModuleCollector](#Collector-Class-KernelModuleCollector) | [KernelModuleAnalyzer](#Data-Analyzer-Class-KernelModuleAnalyzer) | -| MemoryPlugin | free -b
lsmem
numactl -H
wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value | **Analyzer Args:**
- `ratio`: float — Required free-memory ratio (0-1). Analysis fails if free/total < ratio.
- `memory_threshold`: str — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. | - | [MemoryDataModel](#MemoryDataModel-Model) | [MemoryCollector](#Collector-Class-MemoryCollector) | [MemoryAnalyzer](#Data-Analyzer-Class-MemoryAnalyzer) | -| NetworkPlugin | ip addr show
curl
ethtool -i {interface}
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
ip route show
ip rule show
wget | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | -| NicPlugin | bcmcli_config query AN_PROTOCOL -d {device_id}
bcmcli_debug dscdump --lane 0 -d {device_id}
bcmcli_fwmanager show certificate -d {device_id}
bcmcli_fwmanager show fwpackage -d {device_id}
bcmcli_show version
bcmcli_show device_list
bcmcli_show health -d {device_id}
bcmcli_link show -d {device_id}
bcmcli_livepatch show -d {device_id}
bcmcli_debug loopback -d {device_id}
bcmcli_config msixmv query --pf all -d {device_id}
bcmcli_config show_confs -d {device_id}
bcmcli_nvm show -d {device_id}
bcmcli_debug pcie counters -d {device_id}
bcmcli_config query pcie_relaxed_ordering -d {device_id}
bcmcli_show pci -d {device_id}
bcmcli_config query performance_profile -d {device_id}
bcmcli_qos show qos -d {device_id}
bcmcli_qos show hw-maps -d {device_id}
bcmcli_qos show rx-port-ratelimit -d {device_id}
bcmcli_qos show tx-ep-ratelimit --port 0 -d {device_id}
bcmcli_qos show ingress --cosq -d {device_id}
bcmcli_qos show egress --cosq -d {device_id}
bcmcli_show status -d {device_id}
bcmcli_show temperature -d {device_id}
bcmcli_show phy -d {device_id}
bcmcli_show resource_counts -d {device_id}
bcmcli_hwdbg dump_tx_counters -d {device_id}
bcmcli_hwdbg dump_rx_counters -d {device_id}
bcmcli_debug verify -d {device_id}
bcmcli_debug serdes_tx --get --modtype NRZ --lane 0 -d {device_id}
bcmcli_tunnel show rss -d {device_id}
bcmcli_tunnel show vxlan --type ipv4 -d {device_id}
bcmcli_tunnel show vxlan --type ipv6 -d {device_id}
bcmcli_timesync show dll -d {device_id}
bcmcli_timesync show duty-cycle -d {device_id}
bcmcli_timesync show ts-pin -d {device_id}
bcmcli_dump snap_dump -d {device_id}
bcmcli_config query support_rdma -d {device_id}
bcmcli_timesync show ptp -d {device_id}
bcmcli_timesync show synce -d {device_id}
bcmcli_timesync show tsio -d {device_id}
niccli --dev {device_num} cable --module_info --show
niccli --dev {device_num} counters --pcie
niccli devid
niccli --listdev
niccli --list
niccli --list_devices
niccli --dev {device_num} fw --livepatch --show
niccli --dev {device_num} linkdiag --dscdump --lane 0
niccli --dev {device_num} linkdiag --loopback --show
niccli --dev {device_num} linkdiag --txfir --show --modulation_type NRZ --lane 0
niccli --dev {device_num} link --counters --show
niccli --dev {device_num} link --status
niccli --dev {device_num} msix --max_vectors --show --pf 0
niccli --dev {device_num} nvm --listoptions
niccli --dev {device_num} nvm --list
niccli --dev {device_num} nvm --verify
niccli --dev {device_num} nvm --view
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --dev {device_num} show --all
niccli --dev {device_num} show --health
niccli --dev {device_num} show --certificate
niccli --dev {device_num} show --pkg_ver
niccli --dev {device_num} qos --egress --cosq --show
niccli --dev {device_num} qos --ingress --cosq --show
niccli --dev {device_num} qos --rx_rate_limit --show
niccli --dev {device_num} qos --listmap --pri2cos
niccli --dev {device_num} timesync --ptp --show
niccli --dev {device_num} timesync --tsio --show
niccli --dev {device_num} tunnel --cfg --rss --show
niccli --dev {device_num} tunnel --cfg --vxlan --type ipv4 --show
niccli --dev {device_num} resmgmt --all --profile --show
niccli --dev {device_num} qos --dscp2prio
niccli --dev {device_num} qos --tx_ep_rate_limit --port 0 --show
niccli --dev {device_num} show --device_info
niccli --dev {device_num} show --device_pci_ids
niccli --dev {device_num} show
niccli --dev {device_num} timesync --synce --show
niccli verify
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl/bcmcli commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required.
- `use_sudo_bcmcli`: bool — If True, run bcmcli commands with sudo when required.
- `broadcom_cli_override`: Optional[str] — Force 'niccli' or 'bcmcli' instead of auto-detecting which Broadcom CLI is present. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | -| NvmePlugin | nvme smart-log {dev}
nvme error-log {dev} --log-entries=256
nvme id-ctrl {dev}
nvme id-ns {dev}{ns}
nvme fw-log {dev}
nvme self-test-log {dev}
nvme get-log {dev} --log-id=6 --log-len=512
nvme telemetry-log {dev} --output-file={dev}_{f_name}
nvme list -o json | - | - | [NvmeDataModel](#NvmeDataModel-Model) | [NvmeCollector](#Collector-Class-NvmeCollector) | - | -| OsPlugin | sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
cat /etc/\*release | grep VERSION_ID
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | +| MemoryPlugin | free -b
lsmem
numactl -H
wmic OS get FreePhysicalMemory /Value; wmic ComputerSystem get TotalPhysicalMemory /Value | **Analyzer Args:**
- `minimum_free_memory_percent`: Optional[float] — Minimum available system memory percentage required.
- `ratio`: float — Required free-memory ratio (0-1). Analysis fails if free/total < ratio.
- `memory_threshold`: str — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. | - | [MemoryDataModel](#MemoryDataModel-Model) | [MemoryCollector](#Collector-Class-MemoryCollector) | [MemoryAnalyzer](#Data-Analyzer-Class-MemoryAnalyzer) | +| NetworkPlugin | ip addr show
curl
ethtool -i {interface}
ethtool -S {interface}
ethtool {interface}
lldpcli show neighbor
lldpctl
ip neighbor show
ping
ip route show
ip rule show
wget | **Analyzer Args:**
- `expected_nic_firmware`: Optional[str] — Exact firmware version expected from each collected NIC. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `url`: Optional[str] — Optional URL to probe for network connectivity (used with netprobe).
- `netprobe`: Optional[Literal['ping', 'wget', 'curl']] — Tool to use for network connectivity probe: ping, wget, or curl.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched (search) against netdev/interface names; any match is skipped (ethtool not run against it). | [NetworkDataModel](#NetworkDataModel-Model) | [NetworkCollector](#Collector-Class-NetworkCollector) | [NetworkAnalyzer](#Data-Analyzer-Class-NetworkAnalyzer) | +| NicPlugin | bcmcli_config query AN_PROTOCOL -d {device_id}
bcmcli_debug dscdump --lane 0 -d {device_id}
bcmcli_fwmanager show certificate -d {device_id}
bcmcli_fwmanager show fwpackage -d {device_id}
bcmcli_show version
bcmcli_show device_list
bcmcli_show health -d {device_id}
bcmcli_link show -d {device_id}
bcmcli_livepatch show -d {device_id}
bcmcli_debug loopback -d {device_id}
bcmcli_config msixmv query --pf all -d {device_id}
bcmcli_config show_confs -d {device_id}
bcmcli_nvm show -d {device_id}
bcmcli_debug pcie counters -d {device_id}
bcmcli_config query pcie_relaxed_ordering -d {device_id}
bcmcli_show pci -d {device_id}
bcmcli_config query performance_profile -d {device_id}
bcmcli_qos show qos -d {device_id}
bcmcli_qos show hw-maps -d {device_id}
bcmcli_qos show rx-port-ratelimit -d {device_id}
bcmcli_qos show tx-ep-ratelimit --port 0 -d {device_id}
bcmcli_qos show ingress --cosq -d {device_id}
bcmcli_qos show egress --cosq -d {device_id}
bcmcli_show status -d {device_id}
bcmcli_show temperature -d {device_id}
bcmcli_show phy -d {device_id}
bcmcli_show resource_counts -d {device_id}
bcmcli_hwdbg dump_tx_counters -d {device_id}
bcmcli_hwdbg dump_rx_counters -d {device_id}
bcmcli_debug verify -d {device_id}
bcmcli_debug serdes_tx --get --modtype NRZ --lane 0 -d {device_id}
bcmcli_tunnel show rss -d {device_id}
bcmcli_tunnel show vxlan --type ipv4 -d {device_id}
bcmcli_tunnel show vxlan --type ipv6 -d {device_id}
bcmcli_timesync show dll -d {device_id}
bcmcli_timesync show duty-cycle -d {device_id}
bcmcli_timesync show ts-pin -d {device_id}
bcmcli_dump snap_dump -d {device_id}
bcmcli_config query support_rdma -d {device_id}
bcmcli_timesync show ptp -d {device_id}
bcmcli_timesync show synce -d {device_id}
bcmcli_timesync show tsio -d {device_id}
niccli --dev {device_num} cable --module_info --show
niccli --dev {device_num} counters --pcie
niccli devid
niccli --listdev
niccli --list
niccli --list_devices
niccli --dev {device_num} fw --livepatch --show
niccli --dev {device_num} linkdiag --dscdump --lane 0
niccli --dev {device_num} linkdiag --loopback --show
niccli --dev {device_num} linkdiag --txfir --show --modulation_type NRZ --lane 0
niccli --dev {device_num} link --counters --show
niccli --dev {device_num} link --status
niccli --dev {device_num} msix --max_vectors --show --pf 0
niccli --dev {device_num} nvm --listoptions
niccli --dev {device_num} nvm --list
niccli --dev {device_num} nvm --verify
niccli --dev {device_num} nvm --view
niccli -dev {device_num} nvm -getoption pcie_relaxed_ordering
niccli --dev {device_num} nvm --getoption pcie_relaxed_ordering
niccli -dev {device_num} nvm -getoption performance_profile
niccli --dev {device_num} nvm --getoption performance_profile
niccli -dev {device_num} nvm -getoption support_rdma -scope 0
niccli -dev {device_num} getqos
niccli --dev {device_num} nvm --getoption support_rdma --scope 0
niccli --dev {device_num} qos --ets --show
niccli --dev {device_num} show --all
niccli --dev {device_num} show --health
niccli --dev {device_num} show --certificate
niccli --dev {device_num} show --pkg_ver
niccli --dev {device_num} qos --egress --cosq --show
niccli --dev {device_num} qos --ingress --cosq --show
niccli --dev {device_num} qos --rx_rate_limit --show
niccli --dev {device_num} qos --listmap --pri2cos
niccli --dev {device_num} timesync --ptp --show
niccli --dev {device_num} timesync --tsio --show
niccli --dev {device_num} tunnel --cfg --rss --show
niccli --dev {device_num} tunnel --cfg --vxlan --type ipv4 --show
niccli --dev {device_num} resmgmt --all --profile --show
niccli --dev {device_num} qos --dscp2prio
niccli --dev {device_num} qos --tx_ep_rate_limit --port 0 --show
niccli --dev {device_num} show --device_info
niccli --dev {device_num} show --device_pci_ids
niccli --dev {device_num} show
niccli --dev {device_num} timesync --synce --show
niccli verify
niccli --version
nicctl show card
nicctl --version
nicctl show card flash partition --json
nicctl show card interrupts --json
nicctl show card logs --non-persistent
nicctl show card logs --boot-fault
nicctl show card logs --persistent
nicctl show card profile --json
nicctl show card time --json
nicctl show card statistics packet-buffer summary --json
nicctl show lif statistics --json
nicctl show lif internal queue-to-ud-pinning
nicctl show pipeline internal anomalies
nicctl show pipeline internal rsq-ring
nicctl show pipeline internal statistics memory
nicctl show port fsm
nicctl show port transceiver --json
nicctl show port statistics --json
nicctl show port internal mac
nicctl show qos headroom --json
nicctl show rdma queue --json
nicctl show rdma queue-pair --detail --json
nicctl show version firmware
nicctl show dcqcn
nicctl show environment
nicctl show lif
nicctl show pcie ats
nicctl show port
nicctl show qos
nicctl show rdma statistics
nicctl show version host-software
nicctl show dcqcn --card {card_id} --json
nicctl show card hardware-config --card {card_id} | **Analyzer Args:**
- `expected_nic_firmware`: Optional[str] — Exact firmware version expected from each collected NIC.
- `expected_values`: Optional[Dict[str, Dict[str, Any]]] — Per-command expected checks keyed by canonical key (see command_to_canonical_key).
- `performance_profile_expected`: str — Expected Broadcom performance_profile value (case-insensitive). Default RoCE.
- `support_rdma_disabled_values`: List[str] — Values that indicate RDMA is not supported (case-insensitive).
- `pcie_relaxed_ordering_expected`: str — Expected Broadcom pcie_relaxed_ordering value (e.g. 'Relaxed ordering = enabled'); checked case-insensitively. Defaul...
- `expected_qos_prio_map`: Optional[Dict[Any, Any]] — Expected priority-to-TC map (e.g. {0: 0, 1: 1}; keys may be int or str in config). Checked per device when set.
- `expected_qos_pfc_enabled`: Optional[int] — Expected PFC enabled value (0/1 or bitmask). Checked per device when set.
- `expected_qos_tsa_map`: Optional[Dict[Any, Any]] — Expected TSA map for ETS (e.g. {0: 'ets', 1: 'strict'}; keys may be int or str in config). Checked per device when set.
- `expected_qos_tc_bandwidth`: Optional[List[int]] — Expected TC bandwidth percentages. Checked per device when set.
- `require_qos_consistent_across_adapters`: bool — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map.
- `nicctl_log_error_regex`: Optional[List[Dict[str, Any]]] — Optional list of error patterns for nicctl show card logs. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `commands`: Optional[List[str]] — Optional list of niccli/nicctl/bcmcli commands to run. When None, default command set is used.
- `use_sudo_niccli`: bool — If True, run niccli commands with sudo when required.
- `use_sudo_nicctl`: bool — If True, run nicctl commands with sudo when required.
- `use_sudo_bcmcli`: bool — If True, run bcmcli commands with sudo when required.
- `broadcom_cli_override`: Optional[str] — Force 'niccli' or 'bcmcli' instead of auto-detecting which Broadcom CLI is present. | [NicDataModel](#NicDataModel-Model) | [NicCollector](#Collector-Class-NicCollector) | [NicAnalyzer](#Data-Analyzer-Class-NicAnalyzer) | +| NvmePlugin | nvme smart-log {dev}
nvme error-log {dev} --log-entries=256
nvme id-ctrl {dev}
nvme id-ns {dev}{ns}
nvme fw-log {dev}
nvme self-test-log {dev}
nvme get-log {dev} --log-id=6 --log-len=512
nvme telemetry-log {dev} --output-file={dev}_{f_name}
nvme list -o json | **Analyzer Args:**
- `maximum_smart_error_count`: Optional[int] — Maximum allowed NVMe SMART media error count per device. | - | [NvmeDataModel](#NvmeDataModel-Model) | [NvmeCollector](#Collector-Class-NvmeCollector) | [NvmeAnalyzer](#Data-Analyzer-Class-NvmeAnalyzer) | +| OsPlugin | sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'
grep -c ^processor /proc/cpuinfo
vmware -v
cat /proc/loadavg
cat /etc/\*release | grep VERSION_ID
esxcli system version get
wmic os get Version /value
wmic os get Caption /Value | **Analyzer Args:**
- `maximum_load_per_cpu_core`: Optional[float] — Maximum allowed 1-minute load average per CPU core.
- `exp_os`: Union[str, list] — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).
- `exact_match`: bool — If True, require exact match for exp_os; otherwise substring match. | - | [OsDataModel](#OsDataModel-Model) | [OsCollector](#Collector-Class-OsCollector) | [OsAnalyzer](#Data-Analyzer-Class-OsAnalyzer) | | PackagePlugin | dnf list --installed
dpkg-query -W
pacman -Q
cat /etc/\*release
wmic product get name,version | **Analyzer Args:**
- `exp_package_ver`: Dict[str, Optional[str]] — Map package name -> expected version (None = any version). Checked against installed packages.
- `regex_match`: bool — If True, match package versions with regex; otherwise exact or prefix match.
- `rocm_regex`: Optional[str] — Optional regex to identify ROCm package version (used when enable_rocm_regex is True).
- `enable_rocm_regex`: bool — If True, use rocm_regex (or default pattern) to extract ROCm version for checks. | - | [PackageDataModel](#PackageDataModel-Model) | [PackageCollector](#Collector-Class-PackageCollector) | [PackageAnalyzer](#Data-Analyzer-Class-PackageAnalyzer) | -| PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | - | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | -| ProcessPlugin | top -b -n 1
rocm-smi --showpids
top -b -n 1 -o %CPU | **Analyzer Args:**
- `max_kfd_processes`: int — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.
- `max_cpu_usage`: float — Maximum allowed CPU usage (percent) for process checks. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `top_n_process`: int — Number of top processes by CPU usage to collect (e.g. for top -b -n 1 -o %%CPU). | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | -| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | +| PciePlugin | lspci -d {vendor_id}: -nn
lspci -x
lspci -xxxx
lspci -PP
lspci -PP -d {vendor_id}:{dev_id}
lspci -PP -D -d {vendor_id}:{dev_id}
lspci -PP -D
lspci -vvv
lspci -vvvt | **Analyzer Args:**
- `exp_speed`: int — Expected PCIe link speed (generation 1–5).
- `exp_width`: int — Expected PCIe link width in lanes (1–16).
- `exp_sriov_count`: int — Expected SR-IOV virtual function count.
- `exp_gpu_count_override`: Optional[int] — Override expected GPU count for validation.
- `exp_max_payload_size`: Union[Dict[int, int], int, NoneType] — Expected max payload size: int for all devices, or dict keyed by device ID.
- `exp_max_rd_req_size`: Union[Dict[int, int], int, NoneType] — Expected max read request size: int for all devices, or dict keyed by device ID.
- `exp_ten_bit_tag_req_en`: Union[Dict[int, int], int, NoneType] — Expected 10-bit tag request enable: int for all devices, or dict keyed by device ID. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `devid_ep`: Optional[int] — Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi BDF resolution.
- `devid_ep_vf`: Optional[int] — Expected GPU VF PCI device ID (int) for ESXi VF BDF resolution. | [PcieDataModel](#PcieDataModel-Model) | [PcieCollector](#Collector-Class-PcieCollector) | [PcieAnalyzer](#Data-Analyzer-Class-PcieAnalyzer) | +| ProcessPlugin | for p in {pids}; do printf "%s:" "$p"; cat /proc/$p/comm 2>/dev/null || true; printf "\n"; done
printf "__SAMPLER__:%s\n" "$$"; for f in /proc/[0-9]\*/stat; do [ -r "$f" ] || continue; pid="${f#/proc/}"; pid="${pid%/stat}"; [ "$pid" = "$$" ] && continue; printf "%s|" "$pid"; cat "$f" 2>/dev/null || true; printf "\n"; done
cat /proc/stat | **Analyzer Args:**
- `max_cpu_usage`: float — Maximum allowed aggregate CPU execution percentage; idle and I/O-wait time are excluded. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `top_n_process`: int — Number of top processes by CPU usage to collect.
- `sample_interval_seconds`: float — Wall-clock interval between procfs CPU samples. | [ProcessDataModel](#ProcessDataModel-Model) | [ProcessCollector](#Collector-Class-ProcessCollector) | [ProcessAnalyzer](#Data-Analyzer-Class-ProcessAnalyzer) | +| RdmaPlugin | rdma link -j
rdma dev
rdma link
rdma statistic -j | **Analyzer Args:**
- `expected_nic_firmware`: Optional[str] — Exact firmware version expected from each collected RDMA NIC.
- `exclusion_regex`: Optional[list[str]] — Regex patterns matched against an interface netdev; matching interfaces are skipped. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. | [RdmaDataModel](#RdmaDataModel-Model) | [RdmaCollector](#Collector-Class-RdmaCollector) | [RdmaAnalyzer](#Data-Analyzer-Class-RdmaAnalyzer) | | RegexSearchPlugin | No collector step: reads local text from the CLI --data path (file or directory).
Directory scans load each file's contents into RegexSearchData for analysis. | Runs RegexSearchAnalyzer: user-defined patterns via analysis_args.error_regex (same shape as Dmesg).
Emits regex match events with optional per-file source in the description when scanning directories.
**Analyzer Args:**
- `error_regex`: Optional[list[dict[str, Any]]] — Regex patterns to search for; each dict may include regex (str), message, event_category, event_priority (same as Dme...
- `interval_to_collapse_event`: int — Seconds within which repeated events are collapsed into one.
- `num_timestamps`: int — Number of timestamps to include per event in output. | - | [RegexSearchData](#RegexSearchData-Model) | - | [RegexSearchAnalyzer](#Data-Analyzer-Class-RegexSearchAnalyzer) | | RocmPlugin | {rocm_path}/opencl/bin/\*/clinfo
env | grep -Ei 'rocm|hsa|hip|mpi|openmp|ucx|miopen'
ls /sys/class/kfd/kfd/proc/
grep -i -E 'rocm' /etc/ld.so.conf.d/\*
{rocm_path}/bin/rocminfo
ls -v -d {rocm_path}\*
ls -v -d {rocm_path}-[3-7]\* | tail -1
ldconfig -p | grep -i -E 'rocm'
grep . -H -r -i {rocm_path}/.info/\* | **Analyzer Args:**
- `exp_rocm`: Union[str, list] — Expected ROCm version string(s) to match (e.g. from rocminfo).
- `exp_rocm_latest`: str — Expected 'latest' ROCm path or version string for versioned installs.
- `exp_rocm_sub_versions`: dict[str, Union[str, list]] — Map sub-version name (e.g. version_rocm) to expected string or list of allowed strings. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `rocm_path`: str — Base path to ROCm installation (e.g. /opt/rocm). Used for rocminfo, clinfo, and version discovery. | [RocmDataModel](#RocmDataModel-Model) | [RocmCollector](#Collector-Class-RocmCollector) | [RocmAnalyzer](#Data-Analyzer-Class-RocmAnalyzer) | -| StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | **Analyzer Args:**
- `min_required_free_space_abs`: Optional[str] — Minimum required free space per mount (e.g. '10G', '1T').
- `min_required_free_space_prct`: Optional[int] — Minimum required free space as percentage of total (0–100).
- `ignore_devices`: Optional[list[str]] — Mount points or devices to exclude from free-space checks.
- `check_devices`: Optional[list[str]] — If non-empty, only these mount points or devices are checked.
- `regex_match`: bool — If True, match device/mount names with regex; otherwise exact match. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | +| StoragePlugin | sh -c 'df -lH -B1 | grep -v 'boot''
esxcli storage filesystem list
wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace | **LOOP_DEVICE_REGEX**: `^/dev/loop\d+$`
**Analyzer Args:**
- `min_required_free_space_abs`: Optional[str] — Minimum required free space per mount (e.g. '10G', '1T').
- `min_required_free_space_prct`: Optional[int] — Minimum required free space as percentage of total (0–100).
- `ignore_devices`: Optional[list[str]] — Mount points or devices to exclude from free-space checks.
- `check_devices`: Optional[list[str]] — If non-empty, only these mount points or devices are checked.
- `regex_match`: bool — If True, match device/mount names with regex; otherwise exact match.
- `skip_loop_devices`: bool — If True, skip /dev/loopN devices (read-only image mounts that always report 100 percent used) unless listed in check_... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `skip_sudo`: bool — If True, do not use sudo when running df and related storage commands. | [StorageDataModel](#StorageDataModel-Model) | [StorageCollector](#Collector-Class-StorageCollector) | [StorageAnalyzer](#Data-Analyzer-Class-StorageAnalyzer) | | ScaleOutAristaPlugin | show interfaces counters bins | json | no-more
show interfaces counters queue | no-more
show interfaces counters queue drop-precedence | no-more
show qos interfaces ecn counters queue | json | no-more
show interfaces counters errors | json | no-more
show interfaces phy | no-more
show interfaces phy detail | no-more
show interfaces counters ip | json | no-more
show ip interface | no-more
show lldp | no-more
show lldp neighbors | json | no-more
show interfaces counters | json | no-more
show interfaces flow-control | json | no-more
show interfaces counters queue detail | no-more
show priority-flow-control counters | json | no-more
show priority-flow-control status | no-more
show interfaces status | json | no-more
show qos interfaces | no-more
show qos interfaces ecn | no-more
show qos interfaces trust | no-more
show qos maps | no-more
show qos profile | no-more
show qos profile summary | no-more
show interfaces counters rates | json | no-more
show running-config | no-more
show startup-config | no-more
show system environment cooling | json | no-more
show platform trident mmu queue status | no-more
show version | json | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Ports are S/P/[SP] where subport is optional (e.g. ['1/1', '1/31', '1/...
- `expected_port_bandwidth`: int — Expected interface bandwidth (bps) from show interfaces status (AristaPortStatus.bandwidth). Ports with a different b... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. Re-runs successful... | [ScaleOutAristaDataModel](#ScaleOutAristaDataModel-Model) | [ScaleOutAristaCollector](#Collector-Class-ScaleOutAristaCollector) | [ScaleOutAristaAnalyzer](#Data-Analyzer-Class-ScaleOutAristaAnalyzer) | | ScaleOutDellPlugin | show alarm | no-more
show buffer pool | no-more
show buffer profile | no-more
show clock | no-more
show interface counters {port} | no-more
show event details | no-more
show interface fec status | no-more
show interface counters | no-more
show interface counters rate | no-more
show interface Eth | no-more
show interface phy counters | no-more
show interface status | no-more
show interface transceiver | no-more
show interface transceiver dom | no-more
show interface transceiver summary | no-more
show ip arp | no-more
show ip interfaces | no-more
show ip route | no-more
show lldp neighbor | no-more
show lldp table | no-more
show qos interface Ethall priority-flow-control statistics | no-more
show priority-flow-control watchdog | no-more
show qos interface Ethall queue all priority-flow-control watchdog-statistics | no-more
show platform environment | no-more
show platform firmware detail | no-more
show platform syseeprom | no-more
show qos interface Eth all | no-more
show qos interface Eth all queue all | no-more
show qos map dot1p-tc | no-more
show qos map dscp-tc | no-more
show qos map pfc-priority-pg | no-more
show qos map pfc-priority-queue | no-more
show qos map tc-dot1p | no-more
show qos map tc-dscp | no-more
show qos map tc-pg | no-more
show qos map tc-queue | no-more
show qos scheduler-policy | no-more
show qos wred-policy | no-more
show queue counters | no-more
show queue persistent-watermark multicast | no-more
show queue persistent-watermark unicast | no-more
show queue watermark multicast | no-more
show queue watermark unicast | no-more
show running-configuration | no-more
show version | no-more | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given ports. Accepts optional Eth prefix (e.g. ['1/1', '1/31', '1/1/1'] or ['Eth1/1...
- `expected_port_speed`: int — Expected interface speed (Mbps) from show interface status (DellInterfaceStatus.speed). Ports with a different speed... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `collection_ports`: Optional[List[str]] — Restrict detail counter collection to these ports. Accepts the same tokens as analysis_ports (e.g. ['1/1', '1/1/2'] o... | [ScaleOutDellDataModel](#ScaleOutDellDataModel-Model) | [ScaleOutDellCollector](#Collector-Class-ScaleOutDellCollector) | [ScaleOutDellAnalyzer](#Data-Analyzer-Class-ScaleOutDellAnalyzer) | -| SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '\*' are collected with ls -l (e.g. class/net/\*/device).
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | +| ScaleUpSonicPlugin | show clock
show interface fec status
show interface counters
show interface status
show ip interfaces
show lldp neighbors
show lldp table
show platform summary
show platform firmware version
show platform syseeprom
show queue counters
show queue persistent-watermark multicast
show queue persistent-watermark unicast
show queue watermark multicast
show queue watermark unicast
show version | **Analyzer Args:**
- `analysis_ports`: Optional[List[str]] — Restrict per-port analysis to the given full port names (e.g. ['Ethernet1', 'Ethernet31']). When omitted, every port...
- `expected_port_speed`: str — Expected interface speed from show interface status | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output. | [ScaleUpSonicDataModel](#ScaleUpSonicDataModel-Model) | [ScaleUpSonicCollector](#Collector-Class-ScaleUpSonicCollector) | [ScaleUpSonicAnalyzer](#Data-Analyzer-Class-ScaleUpSonicAnalyzer) | +| SysSettingsPlugin | cat /sys/{}
ls -1 /sys/{}
ls -l /sys/{} | **Analyzer Args:**
- `checks`: Optional[list[nodescraper.plugins.inband.sys_settings.analyzer_args.SysfsCheck]] — List of sysfs checks (path, expected values or pattern, display name). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `paths`: list[str] — Sysfs paths to read (cat). Paths with '\*' are cat'd via bash (e.g. class/drm/card\*/device/vbios_version); if cat fail...
- `directory_paths`: list[str] — Sysfs paths to list (ls -1); used for checks that match entry names by regex. | [SysSettingsDataModel](#SysSettingsDataModel-Model) | [SysSettingsCollector](#Collector-Class-SysSettingsCollector) | [SysSettingsAnalyzer](#Data-Analyzer-Class-SysSettingsAnalyzer) | | SysctlPlugin | sysctl -n | **Analyzer Args:**
- `exp_vm_swappiness`: Optional[int] — Expected vm.swappiness value.
- `exp_vm_numa_balancing`: Optional[int] — Expected vm.numa_balancing value.
- `exp_vm_oom_kill_allocating_task`: Optional[int] — Expected vm.oom_kill_allocating_task value.
- `exp_vm_compaction_proactiveness`: Optional[int] — Expected vm.compaction_proactiveness value.
- `exp_vm_compact_unevictable_allowed`: Optional[int] — Expected vm.compact_unevictable_allowed value.
- `exp_vm_extfrag_threshold`: Optional[int] — Expected vm.extfrag_threshold value.
- `exp_vm_zone_reclaim_mode`: Optional[int] — Expected vm.zone_reclaim_mode value.
- `exp_vm_dirty_background_ratio`: Optional[int] — Expected vm.dirty_background_ratio value.
- `exp_vm_dirty_ratio`: Optional[int] — Expected vm.dirty_ratio value.
- `exp_vm_dirty_writeback_centisecs`: Optional[int] — Expected vm.dirty_writeback_centisecs value.
- `exp_kernel_numa_balancing`: Optional[int] — Expected kernel.numa_balancing value. | - | [SysctlDataModel](#SysctlDataModel-Model) | [SysctlCollector](#Collector-Class-SysctlCollector) | [SysctlAnalyzer](#Data-Analyzer-Class-SysctlAnalyzer) | | SyslogPlugin | ls -1 /var/log/syslog\* 2>/dev/null | grep -E '^/var/log/syslog(\.[0-9]+(\.gz)?)?$' || true
ls -1 /var/log/messages\* 2>/dev/null | grep -E '^/var/log/messages(\.[0-9]+(\.gz)?)?$' || true | - | - | [SyslogData](#SyslogData-Model) | [SyslogCollector](#Collector-Class-SyslogCollector) | - | | UptimePlugin | uptime | - | - | [UptimeDataModel](#UptimeDataModel-Model) | [UptimeCollector](#Collector-Class-UptimeCollector) | - | @@ -41,10 +43,15 @@ | --- | --- | --- | --- | --- | --- | --- | | OobGenericCollectionPlugin | Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH).
Commands are user-configured; there are no fixed CMD_\* class fields. | **Analyzer Args:**
- `checks`: list[nodescraper.plugins.generic_collection.analyzer_args.CommandCheck] — Per-command validation rules keyed by collected command name. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `commands`: list[nodescraper.plugins.generic_collection.collector_args.CommandSpec] — Named commands to run. Each entry must include 'name' and 'command'. Prefer small textual stdout; see class docstring...
- `sudo`: bool — Default sudo setting for commands that do not specify sudo.
- `timeout`: int — Default per-command timeout in seconds.
- `include_stdout`: bool — Default: include each command's stdout in collected results for analysis. When false, stdout is omitted from stored r... | [GenericCollectionDataModel](#GenericCollectionDataModel-Model) | [GenericCollectionCollector](#Collector-Class-GenericCollectionCollector) | [GenericAnalyzer](#Data-Analyzer-Class-GenericAnalyzer) | | OobBmcArchivePlugin | SSH (BMC) shell: tar+gzip archives for each path in collection_args (see PathSpec entries).
Uses sudo on the BMC when collection_args paths require elevated access. | - | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `paths`: list[nodescraper.plugins.ooband.bmc_archive.collector_args.PathSpec] — Named BMC paths to archive with tar czf -. Configure in plugin config under plugins.OobBmcArchivePlugin.collection_ar...
- `sudo`: bool — Default sudo setting for paths that do not specify sudo.
- `timeout`: int — Default per-path tar timeout in seconds.
- `skip_if_missing`: bool — Skip paths that do not exist on the BMC instead of failing collection.
- `ignore_failed_read`: bool — When true, pass GNU tar's --ignore-failed-read when the remote tar supports it. | [BmcArchiveDataModel](#BmcArchiveDataModel-Model) | [BmcArchiveCollector](#Collector-Class-BmcArchiveCollector) | - | -| RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.).
URI key `*` runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200). | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | -| RedfishOemDiagPlugin | Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types (collection_args.log_service_path selects the LogService).
Optional binary archives under the plugin log path when log_path is set. | Summarizes success/failure per OEM diagnostic type from collected results.
When analysis_args.require_all_success is true, fails the run if any type failed collection.
**Analyzer Args:**
- `require_all_success`: bool — If True, analysis fails when any OEM type collection failed. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `log_service_path`: str — Redfish path to the LogService (e.g. DiagLogs).
- `oem_diagnostic_types_allowable`: Optional[list[str]] — Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagno...
- `oem_diagnostic_types`: list[str] — OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.
- `task_timeout_s`: int — Max seconds to wait for each BMC task. | [RedfishOemDiagDataModel](#RedfishOemDiagDataModel-Model) | [RedfishOemDiagCollector](#Collector-Class-RedfishOemDiagCollector) | [RedfishOemDiagAnalyzer](#Data-Analyzer-Class-RedfishOemDiagAnalyzer) | +| AmcRedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using Redfish resource links. Oem and OriginOfCondition subtrees are not followed. Collection pages and individual log entries are capped by collection_args. | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, exists).
URI key `*` runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `tree_max_collection_pages`: int — When discover_tree is True: max pages to collect for each paginated collection.
- `tree_max_log_entry_members`: int — When discover_tree is True: max individual LogService entries to collect per collection. Zero keeps every referenced...
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200).
- `clear_log_uris`: list[str] — Redfish LogService.ClearLog action URIs to POST {} to after successful collection. Each URI is posted with an empty J... | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | +| RedfishEndpointPlugin | Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).
Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.
Redfish GET tree: when discover_tree is true, walks from api_root using Redfish resource links. Oem and OriginOfCondition subtrees are not followed. Collection pages and individual log entries are capped by collection_args. | For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, exists).
URI key `*` runs checks against every collected response body.
**Analyzer Args:**
- `checks`: dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]] — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match).... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `uris`: list[str] — Redfish URIs to GET. Ignored when discover_tree is True.
- `discover_tree`: bool — If True, discover endpoints from the BMC Redfish tree (service root and links) instead of using uris.
- `tree_max_depth`: int — When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).
- `tree_max_endpoints`: int — When discover_tree is True: max endpoints to discover (0=no limit).
- `tree_max_collection_pages`: int — When discover_tree is True: max pages to collect for each paginated collection.
- `tree_max_log_entry_members`: int — When discover_tree is True: max individual LogService entries to collect per collection. Zero keeps every referenced...
- `max_workers`: int — Max concurrent GETs (1=sequential). Use >1 for async endpoint fetches.
- `follow_next_link`: bool — If True, follow Redfish Members collection OData nextLink pagination for each URI and merge all pages into a single r...
- `max_pages`: int — When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200).
- `clear_log_uris`: list[str] — Redfish LogService.ClearLog action URIs to POST {} to after successful collection. Each URI is posted with an empty J... | [RedfishEndpointDataModel](#RedfishEndpointDataModel-Model) | [RedfishEndpointCollector](#Collector-Class-RedfishEndpointCollector) | [RedfishEndpointAnalyzer](#Data-Analyzer-Class-RedfishEndpointAnalyzer) | +| AmcRedfishEventLogPlugin | Redfish GET of /Entries, paged with $skip until Members@odata.count is reached; each page is saved as an artifact.
Entries are sorted newest first by Created; collection_args.max_entries keeps only the newest N.
Optional LogService.ClearLog after collection (collection_args.clear_logs_after_collection). | Critical entries are reported as ERROR events and Warning entries as WARNING events; OK and other severities are ignored.
Entries with the same Severity and MessageId are reported once with a count and the newest message.
analysis_args.since, message_id_include and message_id_exclude limit which entries are analyzed.
**Analyzer Args:**
- `since`: Optional[datetime.datetime] — Only analyze entries created at or after this ISO 8601 time (naive values are UTC). Entries without a parseable Creat...
- `message_id_include`: Optional[list[str]] — If set, only analyze entries whose MessageId matches one of these regexes.
- `message_id_exclude`: list[str] — Skip entries whose MessageId matches any of these regexes. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `log_service_path`: str — Redfish path to the LogService; entries are read from /Entries.
- `max_entries`: Optional[int] — Keep only the newest N entries (by Created). All pages are still read.
- `max_pages`: int — Safety cap on the number of $skip pages requested.
- `clear_logs_after_collection`: bool — If True, POST LogService.ClearLog to log_service_path after a successful collection. Requires system_interaction_leve... | [RedfishEventLogDataModel](#RedfishEventLogDataModel-Model) | [RedfishEventLogCollector](#Collector-Class-RedfishEventLogCollector) | [RedfishEventLogAnalyzer](#Data-Analyzer-Class-RedfishEventLogAnalyzer) | +| RedfishEventLogPlugin | Redfish GET of /Entries, paged with $skip until Members@odata.count is reached; each page is saved as an artifact.
Entries are sorted newest first by Created; collection_args.max_entries keeps only the newest N.
Optional LogService.ClearLog after collection (collection_args.clear_logs_after_collection). | Critical entries are reported as ERROR events and Warning entries as WARNING events; OK and other severities are ignored.
Entries with the same Severity and MessageId are reported once with a count and the newest message.
analysis_args.since, message_id_include and message_id_exclude limit which entries are analyzed.
**Analyzer Args:**
- `since`: Optional[datetime.datetime] — Only analyze entries created at or after this ISO 8601 time (naive values are UTC). Entries without a parseable Creat...
- `message_id_include`: Optional[list[str]] — If set, only analyze entries whose MessageId matches one of these regexes.
- `message_id_exclude`: list[str] — Skip entries whose MessageId matches any of these regexes. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `log_service_path`: str — Redfish path to the LogService; entries are read from /Entries.
- `max_entries`: Optional[int] — Keep only the newest N entries (by Created). All pages are still read.
- `max_pages`: int — Safety cap on the number of $skip pages requested.
- `clear_logs_after_collection`: bool — If True, POST LogService.ClearLog to log_service_path after a successful collection. Requires system_interaction_leve... | [RedfishEventLogDataModel](#RedfishEventLogDataModel-Model) | [RedfishEventLogCollector](#Collector-Class-RedfishEventLogCollector) | [RedfishEventLogAnalyzer](#Data-Analyzer-Class-RedfishEventLogAnalyzer) | +| AmcRedfishInventoryPlugin | GET the Redfish service root and follow standard collection links such as Chassis, Managers, Systems, TelemetryService, and UpdateService.
Expand collection members and follow Assembly, PowerSubsystem, FruDevices, Memory, Processors, FirmwareInventory, and metric-report links when the service publishes them.
collection_args.collections and collection_args.member_links append extra property names to that profile set. Names already in the set are ignored.
Uses the active Redfish connection, including a direct session or the SSH proxy.
An SSH-proxy HTTP 404 for a listed collection member is an empty slot and is not a failure. | analysis_args.compare_firmware defaults to true and compares FirmwareInventory across the targets this plugin connected.
RedfishInventoryPlugin compares its direct Redfish targets. AmcRedfishInventoryPlugin compares its SSH-proxy targets. The two are not mixed.
A different Version, a member missing on a connected target, or a missing or empty Version (recorded as N/A) logs one FW warning per component.
Each warning records Version, Status Health, Status State, SoftwareId, Updateable, and Oem SVN. The component id is the published Id and is not normalized.
Warnings are written to the analyzer events.json. result.json has the summary count. The console summary is that count, not the per-component table.
Fewer than two connected targets, a target that failed to connect, or compare_firmware false does not warn. --data loads one model and does not compare nodes.
When no FirmwareInventory members were collected, one warning says the comparison did not run.
**Analyzer Args:**
- `compare_firmware`: bool — Compare FirmwareInventory across this plugin's connected targets. Default true. A different Version, a member missing... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `profile`: Literal['bmc', 'smc', 'all'] — bmc reads chassis, managers, telemetry, firmware inventory, and systems. smc reads systems, memory, processors, and f...
- `collections`: list[str] — Extra service-root property names appended to the profile set. Duplicates are ignored. Empty leaves the profile uncha...
- `member_links`: list[str] — Extra property names appended to the profile link set. Duplicates are ignored. Empty leaves the profile unchanged.
- `max_pages`: int — Maximum pages to keep for each collection, including the first page.
- `enabled`: dict[str, bool] — Turn inventory areas on or off. Omitted areas stay at the profile default, and the default profile enables every area... | [RedfishInventoryDataModel](#RedfishInventoryDataModel-Model) | [RedfishInventoryCollector](#Collector-Class-RedfishInventoryCollector) | [RedfishInventoryAnalyzer](#Data-Analyzer-Class-RedfishInventoryAnalyzer) | +| RedfishInventoryPlugin | GET the Redfish service root and follow standard collection links such as Chassis, Managers, Systems, TelemetryService, and UpdateService.
Expand collection members and follow Assembly, PowerSubsystem, FruDevices, Memory, Processors, FirmwareInventory, and metric-report links when the service publishes them.
collection_args.collections and collection_args.member_links append extra property names to that profile set. Names already in the set are ignored.
Uses the active Redfish connection, including a direct session or the SSH proxy.
An SSH-proxy HTTP 404 for a listed collection member is an empty slot and is not a failure. | analysis_args.compare_firmware defaults to true and compares FirmwareInventory across the targets this plugin connected.
RedfishInventoryPlugin compares its direct Redfish targets. AmcRedfishInventoryPlugin compares its SSH-proxy targets. The two are not mixed.
A different Version, a member missing on a connected target, or a missing or empty Version (recorded as N/A) logs one FW warning per component.
Each warning records Version, Status Health, Status State, SoftwareId, Updateable, and Oem SVN. The component id is the published Id and is not normalized.
Warnings are written to the analyzer events.json. result.json has the summary count. The console summary is that count, not the per-component table.
Fewer than two connected targets, a target that failed to connect, or compare_firmware false does not warn. --data loads one model and does not compare nodes.
When no FirmwareInventory members were collected, one warning says the comparison did not run.
**Analyzer Args:**
- `compare_firmware`: bool — Compare FirmwareInventory across this plugin's connected targets. Default true. A different Version, a member missing... | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `profile`: Literal['bmc', 'smc', 'all'] — bmc reads chassis, managers, telemetry, firmware inventory, and systems. smc reads systems, memory, processors, and f...
- `collections`: list[str] — Extra service-root property names appended to the profile set. Duplicates are ignored. Empty leaves the profile uncha...
- `member_links`: list[str] — Extra property names appended to the profile link set. Duplicates are ignored. Empty leaves the profile unchanged.
- `max_pages`: int — Maximum pages to keep for each collection, including the first page.
- `enabled`: dict[str, bool] — Turn inventory areas on or off. Omitted areas stay at the profile default, and the default profile enables every area... | [RedfishInventoryDataModel](#RedfishInventoryDataModel-Model) | [RedfishInventoryCollector](#Collector-Class-RedfishInventoryCollector) | [RedfishInventoryAnalyzer](#Data-Analyzer-Class-RedfishInventoryAnalyzer) | +| RedfishOemDiagPlugin | Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types (collection_args.log_service_path selects the LogService).
Optional binary archives under the plugin log path when log_path is set. | Summarizes success/failure per OEM diagnostic type from collected results.
When analysis_args.require_all_success is true, fails the run if any type failed collection.
**Analyzer Args:**
- `require_all_success`: bool — If True, analysis fails when any OEM type collection failed. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `log_service_path`: str — Redfish path to the LogService (e.g. DiagLogs).
- `oem_diagnostic_types_allowable`: Optional[list[str]] — Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagno...
- `oem_diagnostic_types`: list[str] — OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.
- `task_timeout_s`: int — Max seconds to wait for each BMC task.
- `clear_logs_after_collection`: bool — If True, POST LogService.ClearLog to the configured log_service_path after a successful collection. Requires system_i... | [RedfishOemDiagDataModel](#RedfishOemDiagDataModel-Model) | [RedfishOemDiagCollector](#Collector-Class-RedfishOemDiagCollector) | [RedfishOemDiagAnalyzer](#Data-Analyzer-Class-RedfishOemDiagAnalyzer) | | ServiceabilityPluginMI3XX | Redfish GET: BMC event log Entries (collection_args.rf_event_log_uri; optional uri alias).
Paginated Members collection and optional top, reference_time/time_operator filters.
Redfish GET: CPER AdditionalDataURI binaries for DiagnosticDataType=CPER events (base64 in data model).
Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).
Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details. | Builds AFID events from collected Redfish event log members (and optional assembly metadata).
Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis.
Runs the configured Python service hub (hub_python_module) to produce service recommendations.
When analysis_args.skip_hub is true, only builds AFID events without running the hub.
**Analyzer Args:**
- `hub_python_module`: Optional[str] — Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.
- `hub_entry_point`: Optional[str] — Registered hub entry point name when hub_python_module is omitted (required in analysis_args).
- `hub_raise_on_error`: bool — When True, entry-point hub analyze() exceptions raise instead of status:error.
- `hub_prefer_rf_events`: bool — When True and Redfish rf_events are available without decoded CPER data, pass raw rf_events to the entry-point hub fo...
- `hub_display_name`: Optional[str] — Optional label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to AFID_SAG.json. When omitted, uses /opt/amd/afid/AFID_SAG.json.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options). | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `uri`: Optional[str] — Optional alias for ``rf_event_log_uri``. When both ``uri`` and ``rf_event_log_uri`` are explicitly set to non-empty v...
- `rf_event_log_uri`: str — Redfish URI for the event log ``Entries`` collection.
- `rf_chassis_devices`: Optional[List[str]] — Chassis designations for Assembly GETs; required with ``rf_assembly_uri_template``.
- `rf_assembly_uri_template`: Optional[str] — Redfish URI template containing ``{device}`` for each chassis Assembly resource.
- `rf_firmware_bundle_uri`: Optional[str] — Redfish URI for firmware bundle inventory when subclasses extract component details.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI3XXCollector](#Collector-Class-MI3XXCollector) | [MI3XXAnalyzer](#Data-Analyzer-Class-MI3XXAnalyzer) | -| Mi4xxServiceabilityPlugin | Redfish GET: Instinct accelerator event log Entries (collection_args.rf_event_log_uri).
MI4xx-only pagination: follows Members@odata.nextLink and falls back to $skip when the BMC reports more entries than one page returns.
Paginated Members collection and optional top, reference_time/time_operator filters.
Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).
Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details.
Optional AFID_SAG-backed FRU grouping summary when collection_args.afid_sag_path is set. | Builds AFID events from collected Redfish event log members (and optional assembly metadata).
Runs the configured entry-point service hub (hub_entry_point in analysis_args) to produce service recommendations.
When analysis_args.skip_hub is true, only builds AFID events without running the hub.
Supports offline analysis from a prior collection via --data with --collection False.
**Analyzer Args:**
- `hub_python_module`: Optional[str] — Not used for MI4XX; AFSE is selected via hub_entry_point afse.
- `hub_entry_point`: str — Registered AFSE entry point name (MI4XX service hub).
- `hub_raise_on_error`: bool — When True, entry-point hub analyze() exceptions raise instead of status:error.
- `hub_prefer_rf_events`: bool — When True and Redfish rf_events are available without decoded CPER data, pass raw rf_events to the entry-point hub fo...
- `hub_display_name`: Optional[str] — Label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to AFID_SAG.json. When omitted, uses /opt/amd/afid/AFID_SAG.json.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options).
- `rf_event_log_uri`: str — Redfish URI for the Instinct accelerator event log Entries collection. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `rf_event_log_uri`: str — Redfish URI for the Instinct accelerator event log Entries collection.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set.
- `rf_assembly_uri_template`: Optional[str] — Optional Redfish URI template containing {device} for chassis Assembly GETs.
- `rf_chassis_devices`: Optional[List[str]] — Optional chassis designations paired with rf_assembly_uri_template.
- `rf_firmware_bundle_uri`: Optional[str] — Optional Redfish URI for firmware bundle inventory.
- `afid_sag_path`: Optional[str] — Optional AFID_SAG.json path for collector FRU grouping logs. When omitted, FRU summary logging is skipped. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI4XXCollector](#Collector-Class-MI4XXCollector) | [MI4XXAnalyzer](#Data-Analyzer-Class-MI4XXAnalyzer) | +| Mi4xxServiceabilityPlugin | Redfish GET: Instinct accelerator event log Entries (collection_args.rf_event_log_uri).
MI4xx-only pagination: follows Members@odata.nextLink and falls back to $skip when the BMC reports more entries than one page returns.
Paginated Members collection and optional top, reference_time/time_operator filters.
Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).
Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details.
Optional AFID_SAG-backed FRU grouping summary when collection_args.afid_sag_path is set. | Builds AFID events from collected Redfish event log members (and optional assembly metadata).
Runs the configured entry-point service hub (hub_entry_point in analysis_args) to produce service recommendations.
When analysis_args.skip_hub is true, only builds AFID events without running the hub.
Supports offline analysis from a prior collection via --data with --collection False.
Writes the top service action to summary.json in the analyzer log directory.
**Analyzer Args:**
- `hub_python_module`: Optional[str] — Not used for MI4XX; Hub is selected via hub_entry_point afse.
- `hub_entry_point`: str — Registered Hub entry point name (default afse).
- `hub_raise_on_error`: bool — When True, entry-point hub analyze() exceptions raise instead of status:error.
- `hub_prefer_rf_events`: bool — When True and Redfish rf_events are available without decoded CPER data, pass raw rf_events to the entry-point hub fo...
- `hub_display_name`: Optional[str] — Label for analyzer status messages.
- `afid_sag_path`: Optional[str] — Path to AFID_SAG.json. When omitted, uses /opt/amd/afid/AFID_SAG.json.
- `hub_init_path_kwarg`: str — Hub __init__ keyword that receives afid_sag_path.
- `hub_analyze_method`: str — Hub method called with rf_events first (default get_service_info).
- `skip_hub`: bool — If True, only build afid_events without running the service hub.
- `cper_decode_module`: Optional[str] — Module import path for CPER decoding when events include CPER attachments.
- `cper_decode_method`: str — Callable on cper_decode_module: file-like CPER in, (return_code, decode_dict) out.
- `hub_options`: Optional[dict[str, Any]] — Extra kwargs for hub __init__ and analyze; collected cper_data overrides cper_data key.
- `from_ac_cycle`: int — from_ac_cycle kwarg for the hub analyze call (merged after hub_options).
- `from_date`: Optional[str] — Optional from_date for the hub analyze call (merged after hub_options).
- `designation_serials`: Optional[dict[str, str]] — Optional designation_serials for the hub analyze call (merged after hub_options).
- `suppress_service_actions`: Optional[list[str]] — Optional suppress_service_actions for the hub analyze call (merged after hub_options).
- `rf_event_log_uri`: str — Redfish URI for the Instinct accelerator event log Entries collection. | **Collection Args:**
- `html_view`: bool — When true, include logged command artifacts in command_artifacts.html using human-readable output.
- `rf_event_log_uri`: str — Redfish URI for the Instinct accelerator event log Entries collection.
- `follow_next_link`: bool — If True, follow Members@odata.nextLink up to max_pages; else single GET.
- `max_pages`: int — Safety cap on the number of pages when following event log pagination.
- `top`: Optional[int] — Most recent N entries via $skip after count probe; None collects full window.
- `reference_time`: Optional[str] — Optional ISO-8601 date or date-time used with time_operator (e.g. 2026-05-17 or 2026-05-17T13:01:00).
- `time_operator`: Optional[Literal['>', '>=', '<', '<=', '==']] — Comparison operator applied when reference_time is set.
- `rf_assembly_uri_template`: Optional[str] — Optional Redfish URI template containing {device} for chassis Assembly GETs.
- `rf_chassis_devices`: Optional[List[str]] — Optional chassis designations paired with rf_assembly_uri_template.
- `rf_firmware_bundle_uri`: Optional[str] — Optional Redfish URI for firmware bundle inventory.
- `afid_sag_path`: Optional[str] — Optional AFID_SAG.json path for collector FRU grouping logs. When omitted, FRU summary logging is skipped. | [ServiceabilityDataModel](#ServiceabilityDataModel-Model) | [MI4XXCollector](#Collector-Class-MI4XXCollector) | [MI4XXAnalyzer](#Data-Analyzer-Class-MI4XXAnalyzer) | # Collectors @@ -71,6 +78,56 @@ GenericCollectionDataModel - Runs each command from collection_args.commands on the target (in-band host or BMC over OOB SSH). - Commands are user-configured; there are no fixed CMD_\* class fields. +## Collector Class AmdSmiFlavorCollector + +### Description + +amd-smi collector with host/guest driver-flavor support (incl. ESXi mxGPU). + +**Bases**: ['AmdSmiCollector'] + +**Link to code**: [amdsmi_flavor_collector.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py) + +### Class Variables + +- **SUPPORTED_OS_FAMILY**: `{, }` +- **_HOST_DRIVER_MODULES**: `('gim', 'amdgpuv')` +- **_GUEST_DRIVER_MODULES**: `('amdgpu',)` +- **_PARTITION_COLUMNS**: `('gpu_id', 'memory', 'accelerator_type', 'accelerator_profile_index', 'partition_id')` + +### Provides Data + +AmdSmiFlavorDataModel + +### Commands + +- bad-pages +- metric --cpu-dimm-pow-consumption {dimm_addr} +- metric --cpu-dimm-temp-range-rate {dimm_addr} +- metric --cpu-dimm-thermal-sensor {dimm_addr} +- metric --cpu-io-bandwidth {bw} {link_id} +- metric --cpu-lclk-dpm-level {nbio_id} +- metric --cpu-svi3-vr-controller-temp 0 +- metric --cpu-svi3-vr-controller-temp 1 {rail_index} +- metric --cpu-xgmi-bandwidth {bw} {link_id} +- fabric +- firmware +- list +- metric -g all +- metric --core all +- metric --cpu all +- node +- partition +- process +- ras --cper --folder={folder} +- ras --afid --cper-file {cper_file} +- static -g all +- static -g {gpu_id} +- topology +- version +- xgmi -l +- xgmi -m + ## Collector Class AmdSmiCollector ### Description @@ -84,14 +141,15 @@ Class for collection of inband tool amd-smi data. ### Class Variables - **AMD_SMI_EXE**: `amd-smi` +- **AMD_SMI_FALLBACK_PATHS**: `('/opt/rocm/bin/amd-smi', '/opt/rocm-*/bin/amd-smi', '/usr/local/bin/amd-smi')` - **SUPPORTED_OS_FAMILY**: `{}` -- **CMD_VERSION**: `version --json` -- **CMD_LIST**: `list --json` -- **CMD_PROCESS**: `process --json` -- **CMD_PARTITION**: `partition --json` -- **CMD_FIRMWARE**: `firmware --json` -- **CMD_STATIC**: `static -g all --json` -- **CMD_STATIC_GPU**: `static -g {gpu_id} --json` +- **CMD_VERSION**: `version` +- **CMD_LIST**: `list` +- **CMD_PROCESS**: `process` +- **CMD_PARTITION**: `partition` +- **CMD_FIRMWARE**: `firmware` +- **CMD_STATIC**: `static -g all` +- **CMD_STATIC_GPU**: `static -g {gpu_id}` - **CMD_TOPOLOGY**: `topology` - **CMD_METRIC**: `metric -g all` - **CMD_BAD_PAGES**: `bad-pages` @@ -99,6 +157,25 @@ Class for collection of inband tool amd-smi data. - **CMD_XGMI_LINK**: `xgmi -l` - **CMD_RAS**: `ras --cper --folder={folder}` - **CMD_RAS_AFID**: `ras --afid --cper-file {cper_file}` +- **CMD_FABRIC**: `fabric` +- **CMD_NODE**: `node` +- **CMD_METRIC_CPU**: `metric --cpu all` +- **CMD_METRIC_CORE**: `metric --core all` +- **CMD_CPU_DIMM_POW**: `metric --cpu-dimm-pow-consumption {dimm_addr}` +- **CMD_CPU_DIMM_THERMAL**: `metric --cpu-dimm-thermal-sensor {dimm_addr}` +- **CMD_CPU_DIMM_TEMP_RANGE_RATE**: `metric --cpu-dimm-temp-range-rate {dimm_addr}` +- **CMD_CPU_SVI3_HOTTEST**: `metric --cpu-svi3-vr-controller-temp 0` +- **CMD_CPU_SVI3_RAIL**: `metric --cpu-svi3-vr-controller-temp 1 {rail_index}` +- **CMD_CPU_XGMI_BANDWIDTH**: `metric --cpu-xgmi-bandwidth {bw} {link_id}` +- **CMD_CPU_IO_BANDWIDTH**: `metric --cpu-io-bandwidth {bw} {link_id}` +- **CMD_CPU_LCLK_DPM_LEVEL**: `metric --cpu-lclk-dpm-level {nbio_id}` +- **CPU_LINK_ID_CANDIDATES**: `('P2', 'P3', 'G0', 'G1', 'G2', 'G3', 'G4', 'G5', 'G6', 'G7')` +- **CPU_XGMI_BW_VALUES**: `(1, 2, 4)` +- **CPU_IO_BW_VALUES**: `(1,)` +- **CPU_SVI3_RAIL_CANDIDATES**: `(0, 1, 2, 3, 4)` +- **MAX_DIMM_ADDR**: `24` +- **MAX_NBIO_ID**: `8` +- **CPU_SEQUENTIAL_NA_GRACE**: `2` ### Provides Data @@ -107,17 +184,29 @@ AmdSmiDataModel ### Commands - bad-pages -- firmware --json -- list --json +- metric --cpu-dimm-pow-consumption {dimm_addr} +- metric --cpu-dimm-temp-range-rate {dimm_addr} +- metric --cpu-dimm-thermal-sensor {dimm_addr} +- metric --cpu-io-bandwidth {bw} {link_id} +- metric --cpu-lclk-dpm-level {nbio_id} +- metric --cpu-svi3-vr-controller-temp 0 +- metric --cpu-svi3-vr-controller-temp 1 {rail_index} +- metric --cpu-xgmi-bandwidth {bw} {link_id} +- fabric +- firmware +- list - metric -g all -- partition --json -- process --json +- metric --core all +- metric --cpu all +- node +- partition +- process - ras --cper --folder={folder} - ras --afid --cper-file {cper_file} -- static -g all --json -- static -g {gpu_id} --json +- static -g all +- static -g {gpu_id} - topology -- version --json +- version - xgmi -l - xgmi -m @@ -133,8 +222,10 @@ Collect BIOS details ### Class Variables +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_WINDOWS**: `wmic bios get SMBIOSBIOSVersion /Value` - **CMD**: `sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'` +- **CMD_ESXI**: `smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1` ### Provides Data @@ -143,6 +234,7 @@ BiosDataModel ### Commands - sh -c 'cat /sys/devices/virtual/dmi/id/bios_version' +- smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1 - wmic bios get SMBIOSBIOSVersion /Value ## Collector Class CmdlineCollector @@ -180,6 +272,7 @@ Collect CPU and GPU count ### Class Variables +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_GPU_COUNT_LINUX**: `lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l` - **CMD_VF_COUNT_LINUX**: `lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l` - **CMD_LSCPU_LINUX**: `lscpu` @@ -187,6 +280,8 @@ Collect CPU and GPU count - **CMD_CPU_COUNT_WINDOWS**: `powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count"` - **CMD_GPU_COUNT_WINDOWS**: `powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count"` - **CMD_VF_COUNT_WINDOWS**: `powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count"` +- **CMD_CPU_COUNT_ESXI**: `esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'` +- **CMD_PCI_COUNT_ESXI**: `esxcli hardware pci list | grep -iE '^ *Device ID: 0x0*{device_id}([^0-9a-f]|$)' | wc -l` ### Provides Data @@ -194,11 +289,13 @@ DeviceEnumerationDataModel ### Commands +- esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}' - powershell -Command "(Get-WmiObject -Class Win32_Processor | Measure-Object).Count" - lspci -d {vendorid_ep}: | grep -iE 'VGA|Display|3D|Processing accelerators|Co-processor|Accelerator' | grep -vi 'Virtual Function' | wc -l - powershell -Command "(wmic path win32_VideoController get name | findstr AMD | Measure-Object).Count" - lscpu - lshw +- esxcli hardware pci list | grep -iE '^ \*Device ID: 0x0\*{device_id}([^0-9a-f]|$)' | wc -l - lspci -d {vendorid_ep}: | grep -i 'Virtual Function' | wc -l - powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count" @@ -214,9 +311,10 @@ Collect data on installed DIMMs ### Class Variables -- **CMD_WINDOWS**: `wmic memorychip get Capacity` -- **CMD**: `sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\|None\|Module" | grep Size' 2>/dev/null` +- **CMD_WINDOWS**: `wmic memorychip get /format:csv` +- **CMD_DMIDECODE**: `dmidecode -q --type 17` - **CMD_DMIDECODE_FULL**: `dmidecode` +- **MEMORY_DEVICE**: `Memory Device` ### Provides Data @@ -224,9 +322,9 @@ DimmDataModel ### Commands -- sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\|None\|Module" | grep Size' 2>/dev/null +- dmidecode -q --type 17 - dmidecode -- wmic memorychip get Capacity +- wmic memorychip get /format:csv ## Collector Class DkmsCollector @@ -265,9 +363,11 @@ Read dmesg log ### Class Variables -- **SUPPORTED_OS_FAMILY**: `{}` +- **SUPPORTED_OS_FAMILY**: `{, }` - **CMD**: `dmesg --time-format iso -x` +- **CMD_ESXI**: `cat /var/log/vmkernel.log` - **CMD_LOGS**: `ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true` +- **CMD_LOGS_ESXI**: `ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true` ### Provides Data @@ -276,7 +376,9 @@ DmesgData ### Commands - dmesg --time-format iso -x +- cat /var/log/vmkernel.log - ls -1 /var/log/dmesg\* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true +- ls -1 /var/log/vmkernel.\* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true ## Collector Class FabricsCollector @@ -357,6 +459,7 @@ Read kernel version ### Class Variables +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_WINDOWS**: `wmic os get Version /Value` - **CMD**: `sh -c 'uname -a'` - **CMD_NUMA_BALANCING**: `sh -c 'cat /proc/sys/kernel/numa_balancing'` @@ -871,11 +974,16 @@ Collect OS details ### Class Variables +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_VERSION_WINDOWS**: `wmic os get Version /value` - **CMD_VERSION**: `cat /etc/*release | grep VERSION_ID` +- **CMD_VERSION_ESXI**: `esxcli system version get` - **CMD_WINDOWS**: `wmic os get Caption /Value` +- **CMD_ESXI**: `vmware -v` - **PRETTY_STR**: `PRETTY_NAME` - **CMD**: `sh -c '( lsb_release -ds || (cat /etc/*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1'` +- **CMD_LOAD_AVERAGE**: `cat /proc/loadavg` +- **CMD_CPU_COUNT**: `grep -c ^processor /proc/cpuinfo` ### Provides Data @@ -884,7 +992,11 @@ OsDataModel ### Commands - sh -c '( lsb_release -ds || (cat /etc/\*release | grep PRETTY_NAME) || uname -om ) 2>/dev/null | head -n1' +- grep -c ^processor /proc/cpuinfo +- vmware -v +- cat /proc/loadavg - cat /etc/\*release | grep VERSION_ID +- esxcli system version get - wmic os get Version /value - wmic os get Caption /Value @@ -951,7 +1063,8 @@ class for collection of PCIe data only supports Linux OS type. ### Class Variables -- **SUPPORTED_OS_FAMILY**: `{}` +- **SUPPORTED_OS_FAMILY**: `{, }` +- **_BDF_LINE**: `re.compile('^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\\.[0-9a-f]+', re.IGNORECASE)` - **CMD_LSPCI_VERBOSE**: `lspci -vvv` - **CMD_LSPCI_VERBOSE_TREE**: `lspci -vvvt` - **CMD_LSPCI_PATH**: `lspci -PP` @@ -982,7 +1095,7 @@ PcieDataModel ### Description -Collect Process details +Collect aggregate CPU usage and top processes from Linux procfs. **Bases**: ['InBandDataCollector'] @@ -991,9 +1104,9 @@ Collect Process details ### Class Variables - **SUPPORTED_OS_FAMILY**: `{}` -- **CMD_KFD**: `rocm-smi --showpids` -- **CMD_CPU_USAGE**: `top -b -n 1` -- **CMD_PROCESS**: `top -b -n 1 -o %CPU ` +- **CMD_PROC_STAT**: `cat /proc/stat` +- **CMD_PROCESS_STAT**: `printf "__SAMPLER__:%s\n" "$$"; for f in /proc/[0-9]*/stat; do [ -r "$f" ] || continue; pid="${f#/proc/}"; pid="${pid%/stat}"; [ "$pid" = "$$" ] && continue; printf "%s|" "$pid"; cat "$f" 2>/dev/null || true; printf "\n"; done` +- **CMD_PROCESS_NAMES**: `for p in {pids}; do printf "%s:" "$p"; cat /proc/$p/comm 2>/dev/null || true; printf "\n"; done` ### Provides Data @@ -1001,9 +1114,9 @@ ProcessDataModel ### Commands -- top -b -n 1 -- rocm-smi --showpids -- top -b -n 1 -o %CPU +- for p in {pids}; do printf "%s:" "$p"; cat /proc/$p/comm 2>/dev/null || true; printf "\n"; done +- printf "__SAMPLER__:%s\n" "$$"; for f in /proc/[0-9]\*/stat; do [ -r "$f" ] || continue; pid="${f#/proc/}"; pid="${pid%/stat}"; [ "$pid" = "$$" ] && continue; printf "%s|" "$pid"; cat "$f" 2>/dev/null || true; printf "\n"; done +- cat /proc/stat ## Collector Class RdmaCollector @@ -1085,8 +1198,10 @@ Collect disk usage details ### Class Variables +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_WINDOWS**: `wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace` - **CMD**: `sh -c 'df -lH -B1 | grep -v 'boot''` +- **CMD_ESXI**: `esxcli storage filesystem list` ### Provides Data @@ -1095,6 +1210,7 @@ StorageDataModel ### Commands - sh -c 'df -lH -B1 | grep -v 'boot'' +- esxcli storage filesystem list - wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace ## Collector Class ScaleOutAristaCollector @@ -1112,7 +1228,7 @@ Collect Arista switch data. ### Class Variables -- **SUPPORTED_OS_FAMILY**: `{, , }` +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_VERSION**: `show version | json | no-more` - **CMD_LLDP_NEIGHBORS**: `show lldp neighbors | json | no-more` - **CMD_SYSTEM_ENV**: `show system environment cooling | json | no-more` @@ -1210,7 +1326,7 @@ Collect Dell SONiC switch data. ### Class Variables -- **SUPPORTED_OS_FAMILY**: `{, , }` +- **SUPPORTED_OS_FAMILY**: `{, , }` - **CMD_VERSION**: `show version | no-more` - **CMD_INTERFACE_STATUS**: `show interface status | no-more` - **CMD_INTERFACE_COUNTERS**: `show interface counters | no-more` @@ -1347,6 +1463,77 @@ ScaleOutDellDataModel - show running-configuration | no-more - show version | no-more +## Collector Class ScaleUpSonicCollector + +### Description + +Collect scale up SONiC switch data. + + Runs SONiC ``show`` commands over SSH and parses their text output into a + :class:`ScaleUpSonicDataModel`. Unlike the Dell collector, commands are + issued directly and are **not** wrapped in ``sonic-cli -c``. + +**Bases**: ['InBandDataCollector'] + +**Link to code**: [scale_up_sonic_collector.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_collector.py) + +### Class Variables + +- **SUPPORTED_OS_FAMILY**: `{, , }` +- **CMD_VERSION**: `show version` +- **CMD_INTERFACE_STATUS**: `show interface status` +- **CMD_INTERFACE_COUNTERS**: `show interface counters` +- **CMD_FEC_STATUS**: `show interface fec status` +- **CMD_QUEUE_COUNTERS**: `show queue counters` +- **CMD_CLOCK**: `show clock` +- **CMD_PLATFORM_SYSEEPROM**: `show platform syseeprom` +- **CMD_PLATFORM_FIRMWARE_DETAIL**: `show platform firmware version` +- **CMD_IP_INTERFACES**: `show ip interfaces` +- **CMD_LLDP_TABLE**: `show lldp table` +- **CMD_LLDP_NEIGHBOR**: `show lldp neighbors` +- **CMD_QUEUE_WATERMARK_UNICAST**: `show queue watermark unicast` +- **CMD_QUEUE_WATERMARK_MULTICAST**: `show queue watermark multicast` +- **CMD_QUEUE_PERSISTENT_WATERMARK_UNICAST**: `show queue persistent-watermark unicast` +- **CMD_QUEUE_PERSISTENT_WATERMARK_MULTICAST**: `show queue persistent-watermark multicast` +- **CMD_PLATFORM_ENVIRONMENT**: `show platform summary` +- **_INTERFACE_STATUS_LINE_RE**: `re.compile('^(?PEthernet\\S*)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P\\S+)\\s+(?P1). - Optional paged GET following the Members collection OData nextLink field when follow_next_link is true. -- Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and Members navigation (depth and endpoint caps from collection_args). +- Redfish GET tree: when discover_tree is true, walks from api_root using Redfish resource links. Oem and OriginOfCondition subtrees are not followed. Collection pages and individual log entries are capped by collection_args. + +## Collector Class RedfishEventLogCollector + +### Description + +Reads every entry from a Redfish LogService's Entries collection. + +**Bases**: ['RedfishDataCollector'] + +**Link to code**: [event_log_collector.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_event_log/event_log_collector.py) + +### Provides Data + +RedfishEventLogDataModel + +### Documented collection + +- Redfish GET of /Entries, paged with $skip until Members@odata.count is reached; each page is saved as an artifact. +- Entries are sorted newest first by Created; collection_args.max_entries keeps only the newest N. +- Optional LogService.ClearLog after collection (collection_args.clear_logs_after_collection). + +## Collector Class RedfishInventoryCollector + +### Description + +Collect standard Redfish inventory resources without product-specific validation. + +**Bases**: ['RedfishDataCollector'] + +**Link to code**: [inventory_collector.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_inventory/inventory_collector.py) + +### Provides Data + +RedfishInventoryDataModel + +### Documented collection + +- GET the Redfish service root and follow standard collection links such as Chassis, Managers, Systems, TelemetryService, and UpdateService. +- Expand collection members and follow Assembly, PowerSubsystem, FruDevices, Memory, Processors, FirmwareInventory, and metric-report links when the service publishes them. +- collection_args.collections and collection_args.member_links append extra property names to that profile set. Names already in the set are ignored. +- Uses the active Redfish connection, including a direct session or the SSH proxy. +- An SSH-proxy HTTP 404 for a listed collection member is an empty slot and is not a failure. ## Collector Class RedfishOemDiagCollector @@ -1570,6 +1799,27 @@ Results for each command configured in collection_args. - **results**: `list[nodescraper.plugins.generic_collection.generic_collection_data.CommandCollectionResult]` +## AmdSmiFlavorDataModel Model + +### Description + +AmdSmiDataModel with host/guest driver-flavor model variants allowed. + +**Link to code**: [amdsmidata_flavor.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py) + +**Bases**: ['AmdSmiDataModel'] + +### Model annotations and fields + +- **version**: `Optional[Union[AmdSmiVersion, HostDriverAmdSmiVersion]]` +- **gpu_list**: `Optional[list[Union[AmdSmiListItem, HostDriverAmdSmiListItem]]]` +- **topology**: `Optional[list[Union[Topo, HostDriverTopo]]]` +- **bad_pages**: `Optional[list[Union[BadPages, HostDriverBadPages]]]` +- **static**: `Optional[list[Union[AmdSmiStatic, GuestAmdSmiStatic, HostDriverAmdSmiStatic]]]` +- **metric**: `Optional[list[Union[AmdSmiMetric, GuestAmdSmiMetric, HostDriverAmdSmiMetric]]]` +- **xgmi_metric**: `Optional[list[Union[XgmiMetrics, HostDriverXgmiMetrics]]]` +- **xgmi_link**: `Optional[list[Union[XgmiLinks, HostDriverXgmiLinks]]]` + ## AmdSmiDataModel Model ### Description @@ -1599,6 +1849,17 @@ Data model for amd-smi data. - **metric**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.AmdSmiMetric]]` - **xgmi_metric**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.XgmiMetrics]]` - **xgmi_link**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.XgmiLinks]]` +- **fabric**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.Fabric]]` +- **node**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.NodeInfo]]` +- **cpu_metric**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuMetric]]` +- **core_metric**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CoreMetric]]` +- **cpu_dimm_pow_consumption**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuDimmPowConsumption]]` +- **cpu_dimm_thermal_sensor**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuDimmThermalSensor]]` +- **cpu_dimm_temp_range_rate**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuDimmTempRangeRate]]` +- **cpu_svi3_vr_controller_temp**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuSvi3VrControllerTemp]]` +- **cpu_xgmi_bandwidth**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuXgmiBandwidth]]` +- **cpu_io_bandwidth**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuIoBandwidth]]` +- **cpu_lclk_dpm_level**: `Optional[list[nodescraper.plugins.inband.amdsmi.amdsmidata.CpuLclkDpmLevel]]` - **cper_data**: `Optional[list[nodescraper.models.datamodel.FileModel]]` - **cper_afids**: `dict[str, int]` - **analysis_firmware_ids**: `Optional[list[str]]` @@ -1640,13 +1901,17 @@ Data model for amd-smi data. ## DimmDataModel Model +### Description + +Inventory of the memory modules installed in the system + **Link to code**: [dimmdata.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/dimm/dimmdata.py) **Bases**: ['DataModel'] ### Model annotations and fields -- **dimms**: `str` +- **dimms**: `list[nodescraper.plugins.inband.dimm.dimmdata.DimmInfo]` ## DkmsDataModel Model @@ -1797,6 +2062,7 @@ Collected output of niccli (Broadcom) and nicctl (Pensando) commands. - **broadcom_nic_support_rdma**: `Dict[int, str]` - **broadcom_nic_performance_profile**: `Dict[int, str]` - **broadcom_nic_pcie_relaxed_ordering**: `Dict[int, str]` +- **broadcom_nic_firmware**: `Dict[int, str]` - **pensando_nic_cards**: `List[nodescraper.plugins.inband.nic.nic_data.PensandoNicCard]` - **pensando_nic_dcqcn**: `List[nodescraper.plugins.inband.nic.nic_data.PensandoNicDcqcn]` - **pensando_nic_environment**: `List[nodescraper.plugins.inband.nic.nic_data.PensandoNicEnvironment]` @@ -1835,6 +2101,8 @@ NVMe collection output: parsed 'nvme list' entries and per-device command output - **os_name**: `str` - **os_version**: `str` +- **load_average_1m**: `Optional[float]` +- **cpu_count**: `Optional[int]` ## PackageDataModel Model @@ -1882,13 +2150,16 @@ class for collection of PCIe data. ## ProcessDataModel Model +### Description + +CPU execution usage (excluding idle and I/O-wait) and top processes from procfs. + **Link to code**: [processdata.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/process/processdata.py) **Bases**: ['DataModel'] ### Model annotations and fields -- **kfd_process**: `Optional[int]` - **cpu_usage**: `Optional[float]` - **processes**: `Optional[list[tuple[str, str]]]` @@ -1990,6 +2261,21 @@ Collected output of Dell SONiC switch commands - **port_list**: `Optional[List[str]]` - **port**: `Optional[Dict[str, nodescraper.plugins.inband.switch.scale_out_dell.scaleoutdelldata.DellPortData]]` +## ScaleUpSonicDataModel Model + +### Description + +Collected output of SONiC switch commands + +**Link to code**: [scaleupsonicdata.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/switch/scale_up_sonic/scaleupsonicdata.py) + +**Bases**: ['DataModel'] + +### Model annotations and fields + +- **port_list**: `Optional[List[str]]` +- **port**: `Optional[Dict[str, nodescraper.plugins.inband.switch.scale_up_sonic.scaleupsonicdata.SonicPortData]]` + ## SysSettingsDataModel Model ### Description @@ -2081,6 +2367,36 @@ Collected Redfish endpoint responses: URI -> JSON body. - **responses**: `dict[str, dict]` +## RedfishEventLogDataModel Model + +### Description + +Entries read from one Redfish LogService, newest first. + +**Link to code**: [event_log_data.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_event_log/event_log_data.py) + +**Bases**: ['DataModel'] + +### Model annotations and fields + +- **log_service_path**: `str` +- **total_count**: `int` +- **entries**: `list[EventLogEntry]` + +## RedfishInventoryDataModel Model + +### Description + +Collected standard Redfish inventory responses, keyed by resource path. + +**Link to code**: [inventory_data.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_inventory/inventory_data.py) + +**Bases**: ['DataModel'] + +### Model annotations and fields + +- **responses**: `dict[str, dict]` + ## RedfishOemDiagDataModel Model ### Description @@ -2187,7 +2503,7 @@ Check dkms matches expected status and version ### Description -Check dmesg for errors +Check dmesg (Linux) or vmkernel.log (ESXi) for errors **Bases**: ['RegexAnalyzer'] @@ -2195,6 +2511,7 @@ Check dmesg for errors ### Class Variables +- **ESXI_TIMESTAMP_PATTERN**: `re.compile('(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z)')` - **ERROR_REGEX**: `[ regex=re.compile('(?:oom_kill_process.*)|(?:Out of memory.*)') message='Out of memory error' event_category= event_priority=, regex=re.compile('IO_PAGE_FAULT') message='I/O Page Fault' event_category= event_priority=, @@ -2231,6 +2548,10 @@ Check dmesg for errors regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.* correctable hardware errors detected in total in \\w+ block.*)') message='RAS Correctable Error' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.* uncorrectable hardware errors detected in \\w+ block.*)') message='RAS Uncorrectable Error' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.* deferred hardware errors detected in \\w+ block.*)') message='RAS Deferred Error' event_category= event_priority=, + regex=re.compile('(\\d+ new uncorrectable hardware errors detected in \\w+ Block.*)') message='RAS Uncorrectable Error' event_category= event_priority=, + regex=re.compile('(\\d+ new correctable hardware errors detected in \\w+ Block.*)') message='RAS Correctable Error' event_category= event_priority=, + regex=re.compile('(GPU detected ECC Fatal Error\\.)') message='RAS ECC Fatal Error' event_category= event_priority=, + regex=re.compile('(Issuing Whole GPU reset\\.)') message='GPU Reset' event_category= event_priority=, regex=re.compile('((?:\\[Hardware Error\\]:\\s+)?event severity: corrected.*)\\n.*(\\[Hardware Error\\]:\\s+Error \\d+, type: corrected.*)\\n.*(\\[Hardware Error\\]:\\s+section_type: PCIe error.*)') message='RAS Corrected PCIe Error' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.*GPU reset begin.*)') message='GPU Reset' event_category= event_priority=, regex=re.compile('(?:\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+,\\d+[+-]\\d+:\\d+)?(.*GPU reset(?:\\(\\d+\\))? failed.*)') message='GPU reset failed' event_category= event_priority=, @@ -2254,10 +2575,11 @@ Check dmesg for errors regex=re.compile('(?:\\[[^\\]]+\\]\\s*)?LNetError:\\s*.*Error\\s*-?\\d+\\s+starting up LNI\\s+\\w+', re.IGNORECASE) message='LNet: Error starting up LNI' event_category= event_priority=, regex=re.compile('LustreError:.*ptlrpc_init_portals\\(\\).*network initiali[sz]ation failed', re.IGNORECASE) message='Lustre: network initialisation failed' event_category= event_priority= ]` +- **_FILTER_TIMESTAMP_PATTERN**: `re.compile('(\\d{4}-\\d+-\\d+T\\d+:\\d+:\\d+),(\\d+[+-]\\d+:\\d+)|(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z)')` ### Regex Patterns -*58 items defined* +*62 items defined* - **Built-in Regexes:** - - Out of memory error: `(?:oom_kill_process.*)|(?:Out of memory.*)` @@ -2295,6 +2617,10 @@ Check dmesg for errors - - RAS Correctable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - RAS Uncorrectable Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - RAS Deferred Error: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` +- - RAS Uncorrectable Error: `(\d+ new uncorrectable hardware errors detected...` +- - RAS Correctable Error: `(\d+ new correctable hardware errors detected i...` +- - RAS ECC Fatal Error: `(GPU detected ECC Fatal Error\.)` +- - GPU Reset: `(Issuing Whole GPU reset\.)` - - RAS Corrected PCIe Error: `((?:\[Hardware Error\]:\s+)?event severity: cor...` - - GPU Reset: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` - - GPU reset failed: `(?:\d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+)?(....` @@ -2378,6 +2704,16 @@ Analyze niccli/nicctl data; checks Broadcom support_rdma, performance_profile (R **Link to code**: [nic_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/nic/nic_analyzer.py) +## Data Analyzer Class NvmeAnalyzer + +### Description + +Check NVMe SMART health data. + +**Bases**: ['DataAnalyzer'] + +**Link to code**: [nvme_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/nvme/nvme_analyzer.py) + ## Data Analyzer Class OsAnalyzer ### Description @@ -2430,7 +2766,7 @@ Check PCIe Data for errors ### Description -Check cpu and kfd processes are within allowed maximum cpu and gpu usage +Check aggregate CPU usage against the configured maximum. **Bases**: ['DataAnalyzer'] @@ -2484,6 +2820,14 @@ Check storage usage **Link to code**: [storage_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/storage/storage_analyzer.py) +### Class Variables + +- **LOOP_DEVICE_REGEX**: `re.compile('^/dev/loop\\d+$')` + +### Regex Patterns + +- **LOOP_DEVICE_REGEX**: `^/dev/loop\d+$` + ## Data Analyzer Class ScaleOutAristaAnalyzer ### Description @@ -2524,6 +2868,26 @@ Check Dell SONiC switch data for errors and warnings. - **PORT_NAME_RE**: `re.compile('^(?:Eth)?(\\d+(?:/\\d+)*)$', re.IGNORECASE)` - **PORT_FORMAT_HINT**: `expected slash-separated decimals (e.g. 'M/S', 'A/B/C')` +## Data Analyzer Class ScaleUpSonicAnalyzer + +### Description + +Check scale up SONiC switch data for errors and warnings. + + Walks every model in the collected :class:`ScaleUpSonicDataModel` and checks + each ``error_fields`` / ``warning_fields`` ClassVar against an optional + ``ports`` filter. + +**Bases**: ['SwitchAnalyzerBase', 'DataAnalyzer'] + +**Link to code**: [scale_up_sonic_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_analyzer.py) + +### Class Variables + +- **VENDOR_NAME**: `SONiC` +- **PORT_NAME_RE**: `re.compile('^(Ethernet\\d+)$')` +- **PORT_FORMAT_HINT**: `expected full port name 'Ethernet' (e.g. 'Ethernet1')` + ## Data Analyzer Class SysSettingsAnalyzer ### Description @@ -2556,9 +2920,45 @@ Checks Redfish endpoint responses against configured thresholds and key/value ru ### Documented analysis -- For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, etc.). +- For each entry in analysis_args.checks, reads JSON paths in collected responses and compares values to constraints (eq, min/max, anyOf, regex, exists). - URI key `*` runs checks against every collected response body. +## Data Analyzer Class RedfishEventLogAnalyzer + +### Description + +Reports Critical and Warning LogService entries as events. + +**Bases**: ['DataAnalyzer'] + +**Link to code**: [event_log_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_event_log/event_log_analyzer.py) + +### Documented analysis + +- Critical entries are reported as ERROR events and Warning entries as WARNING events; OK and other severities are ignored. +- Entries with the same Severity and MessageId are reported once with a count and the newest message. +- analysis_args.since, message_id_include and message_id_exclude limit which entries are analyzed. + +## Data Analyzer Class RedfishInventoryAnalyzer + +### Description + +Compare FirmwareInventory members across the targets this plugin collected. + +**Bases**: ['DataAnalyzer'] + +**Link to code**: [inventory_analyzer.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_inventory/inventory_analyzer.py) + +### Documented analysis + +- analysis_args.compare_firmware defaults to true and compares FirmwareInventory across the targets this plugin connected. +- RedfishInventoryPlugin compares its direct Redfish targets. AmcRedfishInventoryPlugin compares its SSH-proxy targets. The two are not mixed. +- A different Version, a member missing on a connected target, or a missing or empty Version (recorded as N/A) logs one FW warning per component. +- Each warning records Version, Status Health, Status State, SoftwareId, Updateable, and Oem SVN. The component id is the published Id and is not normalized. +- Warnings are written to the analyzer events.json. result.json has the summary count. The console summary is that count, not the per-component table. +- Fewer than two connected targets, a target that failed to connect, or compare_firmware false does not warn. --data loads one model and does not compare nodes. +- When no FirmwareInventory members were collected, one warning says the comparison did not run. + ## Data Analyzer Class RedfishOemDiagAnalyzer ### Description @@ -2607,6 +3007,7 @@ Build AFID events from collected data and run the configured entry-point hub. - Runs the configured entry-point service hub (hub_entry_point in analysis_args) to produce service recommendations. - When analysis_args.skip_hub is true, only builds AFID events without running the hub. - Supports offline analysis from a prior collection via --data with --collection False. +- Writes the top service action to summary.json in the analyzer log directory. # Analyzer Args @@ -2631,11 +3032,14 @@ Build AFID events from collected data and run the configured entry-point hub. - **check_static_data**: `bool` — If True, run static data checks (e.g. driver version, partition mode). - **expected_gpu_processes**: `Optional[int]` — Expected number of GPU processes. - **expected_max_power**: `Optional[int]` — Expected maximum power value (e.g. watts). +- **power**: `Optional[nodescraper.plugins.inband.amdsmi.analyzer_args.PowerConfig]` — GPU power-cap consistency policy. - **expected_power_management**: `Optional[str]` — Expected amd-smi metric power_management value per GPU (e.g. DISABLED for active/full power, ENABLED for power-managed idle). - **expected_driver_version**: `Optional[str]` — Expected AMD driver version string. - **expected_memory_partition_mode**: `Optional[str]` — Expected memory partition mode (e.g. sp3, dp). - **expected_compute_partition_mode**: `Optional[str]` — Expected compute partition mode. - **expected_firmware_versions**: `Optional[dict[str, str]]` — Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE). +- **gpu_memory**: `Optional[nodescraper.plugins.inband.amdsmi.analyzer_args.GpuMemoryConfig]` — Minimum free VRAM threshold to validate for each GPU. +- **check_xgmi_or_peer_links_status**: `bool` — Check XGMI or peer-link status for each GPU. - **l0_to_recovery_count_error_threshold**: `Optional[int]` — L0-to-recovery count above which an error is raised. - **l0_to_recovery_count_warning_threshold**: `Optional[int]` — L0-to-recovery count above which a warning is raised. - **vendorid_ep**: `Optional[str]` — Expected endpoint vendor ID (e.g. for PCIe). @@ -2644,6 +3048,8 @@ Build AFID events from collected data and run the configured entry-point hub. - **devid_ep_vf**: `Optional[str]` — Expected endpoint VF device ID. - **sku_name**: `Optional[str]` — Expected SKU name string for GPU. - **expected_xgmi_speed**: `Optional[list[float]]` — Expected xGMI speed value(s) (e.g. link rate). +- **expected_accel_state**: `str` — Expected amd-smi fabric accel_state value. +- **expected_fabric_type**: `str` — Expected amd-smi fabric fabric_type value. - **analysis_range_start**: `Optional[datetime.datetime]` — Start of time range for time-windowed analysis. - **analysis_range_end**: `Optional[datetime.datetime]` — End of time range for time-windowed analysis. @@ -2762,9 +3168,24 @@ Arguments for journal analyzer ### Annotations / fields +- **minimum_free_memory_percent**: `Optional[float]` — Minimum available system memory percentage required. - **ratio**: `float` — Required free-memory ratio (0-1). Analysis fails if free/total < ratio. - **memory_threshold**: `str` — Minimum free memory required (e.g. '30Gi', '1T'). Used when ratio is not sufficient. +## Analyzer Args Class NetworkAnalyzerArgs + +### Description + +Arguments for network and ethtool analysis. + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/network/analyzer_args.py) + +### Annotations / fields + +- **expected_nic_firmware**: `Optional[str]` — Exact firmware version expected from each collected NIC. + ## Analyzer Args Class NicAnalyzerArgs ### Description @@ -2777,6 +3198,7 @@ Analyzer args for niccli/nicctl data, with expected_values keyed by canonical co ### Annotations / fields +- **expected_nic_firmware**: `Optional[str]` — Exact firmware version expected from each collected NIC. - **expected_values**: `Optional[Dict[str, Dict[str, Any]]]` — Per-command expected checks keyed by canonical key (see command_to_canonical_key). - **performance_profile_expected**: `str` — Expected Broadcom performance_profile value (case-insensitive). Default RoCE. - **support_rdma_disabled_values**: `List[str]` — Values that indicate RDMA is not supported (case-insensitive). @@ -2788,6 +3210,16 @@ Analyzer args for niccli/nicctl data, with expected_values keyed by canonical co - **require_qos_consistent_across_adapters**: `bool` — When True and no expected_qos_\* are set, require all adapters to have the same prio_map, pfc_enabled, and tsa_map. - **nicctl_log_error_regex**: `Optional[List[Dict[str, Any]]]` — Optional list of error patterns for nicctl show card logs. +## Analyzer Args Class NvmeAnalyzerArgs + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/nvme/analyzer_args.py) + +### Annotations / fields + +- **maximum_smart_error_count**: `Optional[int]` — Maximum allowed NVMe SMART media error count per device. + ## Analyzer Args Class OsAnalyzerArgs **Bases**: ['AnalyzerArgs'] @@ -2796,6 +3228,7 @@ Analyzer args for niccli/nicctl data, with expected_values keyed by canonical co ### Annotations / fields +- **maximum_load_per_cpu_core**: `Optional[float]` — Maximum allowed 1-minute load average per CPU core. - **exp_os**: `Union[str, list]` — Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release). - **exact_match**: `bool` — If True, require exact match for exp_os; otherwise substring match. @@ -2840,8 +3273,7 @@ Arguments for PCIe analyzer ### Annotations / fields -- **max_kfd_processes**: `int` — Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check. -- **max_cpu_usage**: `float` — Maximum allowed CPU usage (percent) for process checks. +- **max_cpu_usage**: `float` — Maximum allowed aggregate CPU execution percentage; idle and I/O-wait time are excluded. ## Analyzer Args Class RdmaAnalyzerArgs @@ -2855,6 +3287,7 @@ Arguments for the RDMA analyzer. ### Annotations / fields +- **expected_nic_firmware**: `Optional[str]` — Exact firmware version expected from each collected RDMA NIC. - **exclusion_regex**: `Optional[list[str]]` — Regex patterns matched against an interface netdev; matching interfaces are skipped. ## Analyzer Args Class RegexSearchAnalyzerArgs @@ -2898,6 +3331,7 @@ Arguments for RegexSearchAnalyzer (dict items match Dmesg-style error_regex). - **ignore_devices**: `Optional[list[str]]` — Mount points or devices to exclude from free-space checks. - **check_devices**: `Optional[list[str]]` — If non-empty, only these mount points or devices are checked. - **regex_match**: `bool` — If True, match device/mount names with regex; otherwise exact match. +- **skip_loop_devices**: `bool` — If True, skip /dev/loopN devices (read-only image mounts that always report 100 percent used) unless listed in check_devices. ## Analyzer Args Class ScaleOutAristaAnalyzerArgs @@ -2929,6 +3363,21 @@ Arguments for the Dell SONiC switch analyzer. - **analysis_ports**: `Optional[List[str]]` — Restrict per-port analysis to the given ports. Accepts optional Eth prefix (e.g. ['1/1', '1/31', '1/1/1'] or ['Eth1/1/1']). When omitted, every port present in the data is analyzed. Independent of any collection-time filter. - **expected_port_speed**: `int` — Expected interface speed (Mbps) from show interface status (DellInterfaceStatus.speed). Ports with a different speed are flagged. +## Analyzer Args Class ScaleUpSonicAnalyzerArgs + +### Description + +Arguments for the scale up SONiC switch analyzer. + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/inband/switch/scale_up_sonic/analyzer_args.py) + +### Annotations / fields + +- **analysis_ports**: `Optional[List[str]]` — Restrict per-port analysis to the given full port names (e.g. ['Ethernet1', 'Ethernet31']). When omitted, every port present in the data is analyzed. +- **expected_port_speed**: `str` — Expected interface speed from show interface status + ## Analyzer Args Class SysSettingsAnalyzerArgs ### Description @@ -2978,7 +3427,37 @@ Analyzer args for config-driven Redfish checks. ### Annotations / fields -- **checks**: `dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]]` — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match). Use `*` as the key to apply the inner constraints to every collected response body. Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. Constraints: 'eq' — value must equal the given literal (int, float, str, bool). 'min' — value must be numeric and >= the given number. 'max' — value must be numeric and <= the given number. 'anyOf' — value must be in the given list (OR; any match passes). Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "\*": { "Status/Health": { "anyOf": ["OK"] } } }. +- **checks**: `dict[str, dict[str, Union[int, float, str, bool, dict[str, Any]]]]` — Map: URI or `*` -> { property_path: constraint }. URI keys must match a key in the collected responses (exact match). Use `*` as the key to apply the inner constraints to every collected response body. Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. Constraints: 'eq' — value must equal the given literal (int, float, str, bool). 'min' — value must be numeric and >= the given number. 'max' — value must be numeric and <= the given number. 'anyOf' — value must be in the given list (OR; any match passes). 'regex' — value must be a string matching the pattern (Python re.search; anchor with ^ and $ for a full match). 'exists' — true: property must be present and not null; false: property must be absent or null. Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "\*": { "Status/Health": { "anyOf": ["OK"] } } }. + +## Analyzer Args Class RedfishEventLogAnalyzerArgs + +### Description + +Analyzer args for Redfish LogService entries. + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_event_log/analyzer_args.py) + +### Annotations / fields + +- **since**: `Optional[datetime.datetime]` — Only analyze entries created at or after this ISO 8601 time (naive values are UTC). Entries without a parseable Created time are always analyzed. +- **message_id_include**: `Optional[list[str]]` — If set, only analyze entries whose MessageId matches one of these regexes. +- **message_id_exclude**: `list[str]` — Skip entries whose MessageId matches any of these regexes. + +## Analyzer Args Class RedfishInventoryAnalyzerArgs + +### Description + +Analyzer args for comparing Redfish firmware inventory across targets. + +**Bases**: ['AnalyzerArgs'] + +**Link to code**: [analyzer_args.py](https://github.com/amd/node-scraper/blob/HEAD/nodescraper/plugins/ooband/redfish_inventory/analyzer_args.py) + +### Annotations / fields + +- **compare_firmware**: `bool` — Compare FirmwareInventory across this plugin's connected targets. Default true. A different Version, a member missing on a connected target, or a missing Version (recorded as N/A) logs one FW warning per component. The warning records Version, Status Health, Status State, SoftwareId, Updateable, and Oem SVN. Ids are compared as published. One connected target, a failed connection, or false skips the comparison. --data loads one model and does not compare nodes. ## Analyzer Args Class RedfishOemDiagAnalyzerArgs @@ -3027,7 +3506,7 @@ Analyzer args for serviceability plugins that run a Python or entry-point servic ### Description -Analysis args for Mi4xxServiceabilityPlugin (AFSE entry point). +Analysis args for Mi4xxServiceabilityPlugin (Hub entry point afse). **Bases**: ['ServiceabilityAnalyzerArgs'] @@ -3035,8 +3514,8 @@ Analysis args for Mi4xxServiceabilityPlugin (AFSE entry point). ### Annotations / fields -- **hub_python_module**: `Optional[str]` — Not used for MI4XX; AFSE is selected via hub_entry_point afse. -- **hub_entry_point**: `str` — Registered AFSE entry point name (MI4XX service hub). +- **hub_python_module**: `Optional[str]` — Not used for MI4XX; Hub is selected via hub_entry_point afse. +- **hub_entry_point**: `str` — Registered Hub entry point name (default afse). - **hub_raise_on_error**: `bool` — When True, entry-point hub analyze() exceptions raise instead of status:error. - **hub_prefer_rf_events**: `bool` — When True and Redfish rf_events are available without decoded CPER data, pass raw rf_events to the entry-point hub for redfish transformer parsing. - **hub_display_name**: `Optional[str]` — Label for analyzer status messages. diff --git a/docs/generate_plugin_doc_bundle.py b/docs/generate_plugin_doc_bundle.py index 4e148d96..3f2ff819 100644 --- a/docs/generate_plugin_doc_bundle.py +++ b/docs/generate_plugin_doc_bundle.py @@ -212,9 +212,12 @@ def all_subclasses_union(bases: Iterable[type]) -> set[type]: def all_subclasses_single(cls: type) -> set[type]: - seen, out, work = set(), set(), [cls] + seen: set[type] = set() + out: set[type] = set() + work: list[type] = [cls] while work: parent = work.pop() + sub: type for sub in parent.__subclasses__(): if sub not in seen: seen.add(sub) diff --git a/nodescraper/base/inbandcollectortask.py b/nodescraper/base/inbandcollectortask.py index 25a3c55b..877fd4f9 100644 --- a/nodescraper/base/inbandcollectortask.py +++ b/nodescraper/base/inbandcollectortask.py @@ -55,8 +55,25 @@ def __init__( task_result_hooks: Optional[list[TaskResultHook]] = None, event_reporter: str = DEFAULT_EVENT_REPORTER, session_id: Optional[str] = None, + log_path: Optional[str] = None, **kwargs, ): + """Creates a InBandDataCollector Class + + Args: + system_info (SystemInfo): Information about the system being targeted for data collection. + connection (InBandConnection): The in-band connection used to communicate with the target system. + logger (Optional[logging.Logger], optional): Logger instance for logging messages. Defaults to None. + system_interaction_level (SystemInteractionLevel, optional): Level of interaction with the system. Defaults to SystemInteractionLevel.INTERACTIVE. + max_event_priority_level (Union[EventPriority, str], optional): Maximum priority level for events. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): Parent task identifier. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook]], optional): List of hooks to be called with task results. Defaults to None. + event_reporter (str, optional): Event reporter identifier. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): Session identifier for the data collection task. Defaults to None. + + Raises: + SystemCompatibilityError: Raised if the system's OS family is not supported. + """ super().__init__( system_info=system_info, system_interaction_level=system_interaction_level, @@ -67,6 +84,7 @@ def __init__( task_result_hooks=task_result_hooks, event_reporter=event_reporter, session_id=session_id, + log_path=log_path, ) if self.system_info.os_family not in self.SUPPORTED_OS_FAMILY: raise SystemCompatibilityError( @@ -97,6 +115,9 @@ def _run_sut_cmd( Returns: CommandArtifact: The result of the command execution, which includes stdout, stderr, and exit code. """ + if self.system_info.os_family == OSFamily.ESXI: + # ESXi runs as root and has no sudo binary; prefixing sudo would fail. + sudo = False command_res = self.connection.run_command( command=command, sudo=sudo, timeout=timeout, strip=strip ) diff --git a/nodescraper/base/match_ignore.py b/nodescraper/base/match_ignore.py index f1521a58..6e66b3c3 100644 --- a/nodescraper/base/match_ignore.py +++ b/nodescraper/base/match_ignore.py @@ -57,13 +57,15 @@ def parse_mce_bank_spec(spec: Sequence[MceBankSpec]) -> frozenset[int]: raise ValueError(f"Invalid MCE bank number: {entry}") banks.add(entry) continue - token = str(entry).strip() if not token: raise ValueError("Empty MCE bank entry") - if "-" in token: start_text, end_text = token.split("-", 1) + # Check if it looks like "-5" + if not start_text.strip() or not end_text.strip() or (int(end_text.strip()) < 0): + raise ValueError(f"Invalid MCE bank range: {entry}") + start = int(start_text.strip()) end = int(end_text.strip()) if start < 0 or end < 0 or start > end: diff --git a/nodescraper/base/oobanddataplugin.py b/nodescraper/base/oobanddataplugin.py index c88ffc91..a17283ee 100644 --- a/nodescraper/base/oobanddataplugin.py +++ b/nodescraper/base/oobanddataplugin.py @@ -1,48 +1,169 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Generic - -from nodescraper.connection.redfish import ( - RedfishConnectionManager, - RedfishConnectionParams, -) -from nodescraper.generictypes import TAnalyzeArg, TCollectArg, TDataModel -from nodescraper.interfaces import DataPlugin - - -class OOBandDataPlugin( - DataPlugin[ - RedfishConnectionManager, - RedfishConnectionParams, - TDataModel, - TCollectArg, - TAnalyzeArg, - ], - Generic[TDataModel, TCollectArg, TAnalyzeArg], -): - """Base class for out-of-band (OOB) plugins that use Redfish connection.""" - - CONNECTION_TYPE = RedfishConnectionManager +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pathlib import Path +from typing import Any, Generic, Optional, Union + +from nodescraper.base.redfishcollectortask import _target_dir_name +from nodescraper.connection.redfish import ( + RedfishConnectionManager, + RedfishConnectionParams, + collected_multi_target_data, +) +from nodescraper.enums import EventPriority, ExecutionStatus, SystemInteractionLevel +from nodescraper.generictypes import TAnalyzeArg, TCollectArg, TDataModel +from nodescraper.interfaces import DataPlugin +from nodescraper.models import TaskResult +from nodescraper.taskresulthooks.filesystemloghook import hooks_for_fixed_directory +from nodescraper.utils import resolve_log_dir_name + + +class OOBandDataPlugin( + DataPlugin[ + RedfishConnectionManager, + RedfishConnectionParams, + TDataModel, + TCollectArg, + TAnalyzeArg, + ], + Generic[TDataModel, TCollectArg, TAnalyzeArg], +): + """Base class for OOB plugins (Redfish). Adds multi-target support. + + Multi-target collection is handled transparently by RedfishDataCollector's + __init_subclass__ wrapper; this class only needs to override analyze() to run + the analyzer once per target after collection completes. + """ + + CONNECTION_TYPE = RedfishConnectionManager + + def run( # type: ignore[override] + self, + collection: bool = True, + analysis: bool = True, + max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, + system_interaction_level: Union[ + SystemInteractionLevel, str + ] = SystemInteractionLevel.INTERACTIVE, + preserve_connection: bool = True, + data: Optional[Any] = None, + collection_args: Optional[Any] = None, + analysis_args: Optional[Any] = None, + ): + """Run plugin. For multi-target OK runs, the summary includes per-target collection detail.""" + result = super().run( + collection=collection, + analysis=analysis, + max_event_priority_level=max_event_priority_level, + system_interaction_level=system_interaction_level, + preserve_connection=preserve_connection, + data=data, + collection_args=collection_args, + analysis_args=analysis_args, + ) + cm = self.connection_manager + if collected_multi_target_data(cm): + # DataPlugin.run() replaces the message with "Plugin tasks completed successfully" + # for OK status, discarding per-target detail. Restore it for multi-target runs. + if result.status == ExecutionStatus.OK and getattr( + self.collection_result, "message", None + ): + result.message = self.collection_result.message + + return result + + def analyze( + self, + max_event_priority_level: Optional[Union[EventPriority, str]] = EventPriority.CRITICAL, + analysis_args: Optional[Union[TAnalyzeArg, dict]] = None, + data: Optional[Any] = None, + ) -> TaskResult: + """Analyze collected data for one BMC or once per Redfish target. + + Args: + max_event_priority_level: Priority limit for events. + analysis_args: Analyzer arguments. + data: Pre-collected data for a single-target run. + + Returns: + TaskResult: Analysis result for the targets that returned data. + """ + cm = self.connection_manager + multi_target_data: dict = collected_multi_target_data(cm) + + if not multi_target_data: + return super().analyze( + max_event_priority_level=max_event_priority_level, + analysis_args=analysis_args, + data=data, + ) + + if self.ANALYZER is None: + self.analysis_result = TaskResult( + status=ExecutionStatus.NOT_RAN, + parent=self.__class__.__name__, + message=f"Data analysis not supported for {self.__class__.__name__}", + ) + return self.analysis_result + + if ( + analysis_args is not None + and isinstance(analysis_args, dict) + and hasattr(self, "ANALYZER_ARGS") + and self.ANALYZER_ARGS is not None + ): + analysis_args = self.ANALYZER_ARGS.model_validate(analysis_args) # type: ignore[assignment] + + plugin_log_dir = ( + Path(self.log_path) / resolve_log_dir_name(self.__class__.__name__) + if self.log_path + else None + ) + analyzer_name = resolve_log_dir_name(self.ANALYZER.__name__) + + analysis_results: list[TaskResult] = [] + for target_key, target_data in multi_target_data.items(): + safe_key = _target_dir_name(target_key) + target_hooks = self.task_result_hooks + if plugin_log_dir is not None: + target_hooks = hooks_for_fixed_directory( + self.task_result_hooks, + str(plugin_log_dir / analyzer_name / safe_key), + ) + analyzer_task = self.ANALYZER( + system_info=self.system_info.model_copy(), + logger=self.logger, + max_event_priority_level=max_event_priority_level or EventPriority.CRITICAL, + parent=safe_key, + task_result_hooks=target_hooks, + event_reporter=self.event_reporter, + session_id=self.session_id, + ) + analysis_results.append(analyzer_task.analyze_data(target_data, analysis_args)) + + self.analysis_result = self._aggregate_collection_results( + self.__class__.__name__, analysis_results + ) + return self.analysis_result diff --git a/nodescraper/base/redfishcollectortask.py b/nodescraper/base/redfishcollectortask.py index bdeb65cd..2e8b4463 100644 --- a/nodescraper/base/redfishcollectortask.py +++ b/nodescraper/base/redfishcollectortask.py @@ -1,130 +1,370 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import logging -from typing import Generic, Optional, Union - -from nodescraper.connection.redfish import RedfishConnection, RedfishGetResult -from nodescraper.constants import DEFAULT_EVENT_REPORTER -from nodescraper.enums import EventPriority -from nodescraper.generictypes import TCollectArg, TDataModel -from nodescraper.interfaces import DataCollector, TaskResultHook -from nodescraper.models import SystemInfo - - -class RedfishDataCollector( - DataCollector[RedfishConnection, TDataModel, TCollectArg], - Generic[TDataModel, TCollectArg], -): - """Base class for data collectors that use a Redfish connection.""" - - def __init__( - self, - system_info: SystemInfo, - connection: RedfishConnection, - logger: Optional[logging.Logger] = None, - max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, - parent: Optional[str] = None, - task_result_hooks: Optional[list[TaskResultHook]] = None, - event_reporter: str = DEFAULT_EVENT_REPORTER, - session_id: Optional[str] = None, - **kwargs, - ): - super().__init__( - system_info=system_info, - connection=connection, - logger=logger, - max_event_priority_level=max_event_priority_level, - parent=parent, - task_result_hooks=task_result_hooks, - event_reporter=event_reporter, - session_id=session_id, - **kwargs, - ) - - def _run_redfish_get( - self, - path: str, - log_artifact: bool = True, - html_view: Optional[bool] = None, - ) -> RedfishGetResult: - """Run a Redfish GET request and return the result. - - Args: - path: Redfish URI path - log_artifact: If True, append the result to self.result.artifacts. - html_view: When set, controls HTML artifact output. When omitted, uses - collection_args.html_view. - - Returns: - RedfishGetResult: path, success, data (or error), status_code. - """ - res = self.connection.run_get(path) - effective_html_view = self._effective_html_view(html_view) - if log_artifact or effective_html_view: - res.log_html = effective_html_view - self.result.artifacts.append(res) - return res - - def _run_redfish_get_paged( - self, - path: str, - max_pages: int = 200, - log_artifact: bool = True, - html_view: Optional[bool] = None, - ) -> RedfishGetResult: - """ - Run a Redfish GET and follow Members@odata.nextLink pagination, merging all pages into a single response. - - Args: - path (str): Redfish URI path. - max_pages (int, optional): safety cap on the number of pages to follow. Defaults to 200. - log_artifact (bool, optional): whether we should log the merged result. Defaults to True. - html_view (Optional[bool], optional): whether to include this request in HTML artifacts. - When omitted, uses collection_args.html_view. - - Returns: - RedfishGetResult: path, success, merged data (or error), status_code. - """ - res = self.connection.run_get_paged(path, max_pages=max_pages) - effective_html_view = self._effective_html_view(html_view) - if log_artifact or effective_html_view: - res.log_html = effective_html_view - self.result.artifacts.append(res) - return res - - def _append_redfish_artifact( - self, - res: RedfishGetResult, - *, - log_artifact: bool = True, - html_view: Optional[bool] = None, - ) -> RedfishGetResult: - """Append a Redfish GET result to task artifacts with log flags applied.""" - effective_html_view = self._effective_html_view(html_view) - if log_artifact or effective_html_view: - res.log_html = effective_html_view - self.result.artifacts.append(res) - return res +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from functools import wraps +from pathlib import Path +from typing import Any, Generic, Optional, Union + +from nodescraper.connection.redfish import ( + MultiTargetRedfishConnection, + RedfishConnection, + RedfishGetResult, +) +from nodescraper.connection.redfish.redfish_connection import RedfishPostResult +from nodescraper.constants import DEFAULT_EVENT_REPORTER +from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.generictypes import TCollectArg, TDataModel +from nodescraper.interfaces import DataCollector, TaskResultHook +from nodescraper.interfaces.dataplugin import DataPlugin +from nodescraper.models import SystemInfo, TaskResult +from nodescraper.taskresulthooks.filesystemloghook import hooks_for_fixed_directory + +_HARD_FAIL = {ExecutionStatus.ERROR, ExecutionStatus.EXECUTION_FAILURE} +_TARGET_SUCCESS = {ExecutionStatus.OK, ExecutionStatus.WARNING} + + +class _TargetPrefixAdapter(logging.LoggerAdapter): + """Prepend ``[target_key]`` to every log message emitted by a per-target collector thread.""" + + def __init__(self, logger: logging.Logger, extra: dict[str, Any]) -> None: + super().__init__(logger, extra) + self._target_key: str = extra["target_key"] + + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f"[{self._target_key}] {msg}", kwargs + + +def _target_dir_name(target_key: str) -> str: + """Return a filesystem-safe directory name for a target key. + + Args: + target_key: Target identifier from the connection config. + + Returns: + str: Directory name containing only letters, numbers, dot, underscore, and dash. + """ + # Collapse characters that are unsafe in a log directory name. + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", target_key).strip("._") + return cleaned or "target" + + +def combine_target_results(parent: str, results: list[TaskResult]) -> TaskResult: + """Merge per-target task results. + + Args: + parent: Parent name stored on the combined result. + results: Per-target task results. + + Returns: + TaskResult: Combined result. Mixed success and failure stays at warning. + """ + aggregated = DataPlugin._aggregate_collection_results(parent, results) + hard_fail = any(result.status in _HARD_FAIL for result in results) + succeeded = any(result.status in _TARGET_SUCCESS for result in results) + + # Display one line per target so the summary table is easy to read. + messages = [r.message for r in results if r.message] + if messages: + prefix = ( + "One or more targets failed; continuing with the rest.\n" + if (hard_fail and succeeded) + else "" + ) + aggregated.message = prefix + "\n".join(messages) + if hard_fail and succeeded: + aggregated.status = ExecutionStatus.WARNING + aggregated.details["per_target_messages"] = True + + return aggregated + + +class RedfishDataCollector( + DataCollector[RedfishConnection, TDataModel, TCollectArg], + Generic[TDataModel, TCollectArg], +): + """Base class for data collectors that use a Redfish connection.""" + + def __init__( + self, + system_info: SystemInfo, + connection: RedfishConnection, + logger: Optional[logging.Logger] = None, + max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, + parent: Optional[str] = None, + task_result_hooks: Optional[list[TaskResultHook]] = None, + event_reporter: str = DEFAULT_EVENT_REPORTER, + session_id: Optional[str] = None, + **kwargs, + ): + """Creates a RedfishDataCollector instance. + + Args: + system_info (SystemInfo): system info object for target system for data collection + connection (TConnection): connection object for the data collector + logger (Optional[logging.Logger], optional): python logger object. Defaults to None. + max_event_priority_level (Union[EventPriority, str], optional): priority limit for events. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): parent task identifier. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook]], optional): list of task result hooks. Defaults to None. + event_reporter (str, optional): Reporter string stored on emitted events. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): session identifier. Defaults to None. + """ + super().__init__( + system_info=system_info, + connection=connection, + logger=logger, + max_event_priority_level=max_event_priority_level, + parent=parent, + task_result_hooks=task_result_hooks, + event_reporter=event_reporter, + session_id=session_id, + **kwargs, + ) + + def __init_subclass__(cls, **kwargs: Any) -> None: + """Wrap each concrete collector's collect_data with multi-target detection. + + Execution order when collect_data is called: + 1. _multi_target_wrapper (outermost) — checks for MultiTargetRedfishConnection + 2. collect_decorator (from DataCollector) — handles hooks, finalization + 3. Concrete collect_data implementation (innermost) + + In multi-target mode a fresh collector instance is created per target so that + concurrent threads never share mutable state. + """ + super().__init_subclass__(**kwargs) # applies collect_decorator via DataCollector + if "collect_data" not in vars(cls): + return + inner = cls.collect_data # already wrapped by collect_decorator at this point + + @wraps(inner) + def _multi_target_wrapper( + collector: "RedfishDataCollector", + args: Any = None, + *, + _fn: Any = inner, + ) -> tuple[TaskResult, Any]: + if not isinstance(collector.connection, MultiTargetRedfishConnection): + return _fn(collector, args) + + multi_conn: MultiTargetRedfishConnection = collector.connection # type: ignore[assignment] + parent_name = collector.parent or type(collector).__name__ + all_results: list[TaskResult] = [] + target_count = len(multi_conn.target_connections) + configured_workers = multi_conn.max_workers or target_count + max_workers = min(configured_workers, target_count, 32) if target_count else 1 + + def _run_for_target( + target_key: str, conn: RedfishConnection + ) -> tuple[str, TaskResult, Any]: + safe_key = _target_dir_name(target_key) + collector_log = getattr(collector, "log_path", None) + target_log = str(Path(collector_log) / safe_key) if collector_log else None + per_target_hooks = ( + hooks_for_fixed_directory(collector.task_result_hooks, target_log) + if target_log + else list(collector.task_result_hooks) + ) + + target_collector = type(collector)( + system_info=collector.system_info.model_copy(), + connection=conn, + logger=_TargetPrefixAdapter(collector.logger, {"target_key": target_key}), # type: ignore[arg-type] + max_event_priority_level=getattr( + collector, "max_event_priority_level", EventPriority.CRITICAL + ), + parent=safe_key, + task_result_hooks=per_target_hooks, + event_reporter=collector.event_reporter, + session_id=collector.session_id, + log_path=target_log, + system_interaction_level=getattr(collector, "system_interaction_level", None), + ) + collector.logger.info( + "Starting collection for target %r using %s", target_key, parent_name + ) + result, data = _fn(target_collector, args) + collector.logger.info("Finished collection for target %r", target_key) + + # Prefix result message with target key for attribution in the summary table. + if result.message: + result.message = f"[{target_key}] {result.message}" + + return target_key, result, data + + if multi_conn.target_connections: + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit(_run_for_target, tk, conn): tk + for tk, conn in multi_conn.target_connections.items() + } + for future in as_completed(futures): + target_key = futures[future] + try: + tk, result, data = future.result() + all_results.append(result) + if data is not None: + multi_conn.multi_target_data[tk] = data + except Exception as exc: + collector.logger.error( + "Collection failed for target %r: %s", target_key, exc + ) + all_results.append( + TaskResult( + status=ExecutionStatus.EXECUTION_FAILURE, + parent=f"{parent_name}[{target_key}]", + message=f"Collection failed for target {target_key!r}: {exc}", + ) + ) + + if not all_results: + return ( + TaskResult( + status=ExecutionStatus.NOT_RAN, + parent=parent_name, + message="No Redfish targets were collected", + ), + None, + ) + return combine_target_results(parent_name, all_results), None + + cls.collect_data = _multi_target_wrapper # type: ignore[method-assign, assignment] + + def _run_redfish_get( + self, + path: str, + log_artifact: bool = True, + html_view: Optional[bool] = None, + ) -> RedfishGetResult: + """Run a Redfish GET request and return the result. + + Args: + path: Redfish URI path + log_artifact: If True, append the result to self.result.artifacts. + html_view: When set, controls HTML artifact output. When omitted, uses + collection_args.html_view. + + Returns: + RedfishGetResult: path, success, data (or error), status_code. + """ + res = self.connection.run_get(path) + effective_html_view = self._effective_html_view(html_view) + if log_artifact or effective_html_view: + res.log_html = effective_html_view + self.result.artifacts.append(res) + return res + + def _run_redfish_get_paged( + self, + path: str, + max_pages: int = 200, + log_artifact: bool = True, + html_view: Optional[bool] = None, + ) -> RedfishGetResult: + """ + Run a Redfish GET and follow Members@odata.nextLink pagination, merging all pages into a single response. + + Args: + path (str): Redfish URI path. + max_pages (int, optional): safety cap on the number of pages to follow. Defaults to 200. + log_artifact (bool, optional): whether we should log the merged result. Defaults to True. + html_view (Optional[bool], optional): whether to include this request in HTML artifacts. + When omitted, uses collection_args.html_view. + + Returns: + RedfishGetResult: path, success, merged data (or error), status_code. + """ + res = self.connection.run_get_paged(path, max_pages=max_pages) + effective_html_view = self._effective_html_view(html_view) + if log_artifact or effective_html_view: + res.log_html = effective_html_view + self.result.artifacts.append(res) + return res + + def _append_redfish_artifact( + self, + res: RedfishGetResult, + *, + log_artifact: bool = True, + html_view: Optional[bool] = None, + ) -> RedfishGetResult: + """Append a Redfish GET result to task artifacts with log flags applied.""" + effective_html_view = self._effective_html_view(html_view) + if log_artifact or effective_html_view: + res.log_html = effective_html_view + self.result.artifacts.append(res) + return res + + def _run_redfish_post( + self, + path: str, + payload: Optional[dict] = None, + log_artifact: bool = True, + ) -> RedfishPostResult: + """Run a Redfish POST request and return the result. + + Args: + path: Redfish URI path. + payload: JSON payload body. Defaults to an empty dict. + log_artifact: If True, append the result to self.result.artifacts. + + Returns: + RedfishPostResult: path, success, data (or error), status_code. + """ + path_norm = path.strip() + if not path_norm.startswith("/"): + path_norm = "/" + path_norm + try: + resp = self.connection.post(path_norm, json=payload or {}) + success = resp.ok + status_code = resp.status_code + try: + data = resp.json() if resp.content else None + if not isinstance(data, dict): + data = None + except Exception: + data = None + error = ( + None if success else f"POST {path_norm} failed: {resp.status_code} {resp.reason}" + ) + result = RedfishPostResult( + path=path_norm, + success=success, + data=data, + error=error, + status_code=status_code, + ) + except Exception as exc: + result = RedfishPostResult( + path=path_norm, + success=False, + error=str(exc), + status_code=None, + ) + if log_artifact: + self.result.artifacts.append(result) + return result diff --git a/nodescraper/base/regexanalyzer.py b/nodescraper/base/regexanalyzer.py index 1d7adf61..ca482c0e 100644 --- a/nodescraper/base/regexanalyzer.py +++ b/nodescraper/base/regexanalyzer.py @@ -402,8 +402,9 @@ def _is_within_interval(new_timestamp_str: str, existing_timestamps: list[str]) new_event.data["timestamp"] = timestamp regex_event_list.append(new_event) - search_from = match_obj.end() + if search_from == match_obj.start(): # Zero-width match + search_from += 1 # Force advancement all_events = list(regex_map.values()) if group else regex_event_list diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 606dd5e8..cb917120 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -32,7 +32,7 @@ import platform import sys import uuid -from collections.abc import Callable, Sequence +from collections.abc import Callable, Collection, Sequence from typing import Optional import nodescraper @@ -72,7 +72,8 @@ from nodescraper.models import SystemInfo from nodescraper.models.pluginresult import PluginResult from nodescraper.pluginexecutor import PluginExecutor -from nodescraper.pluginregistry import PluginRegistry +from nodescraper.pluginrecipe.discovery import PluginDiscovery +from nodescraper.pluginregistry import PluginRegistry, set_default_registry def _parse_plugin_configs_csv(value: str) -> list[str]: @@ -80,21 +81,21 @@ def _parse_plugin_configs_csv(value: str) -> list[str]: return [p.strip() for p in value.split(",") if p.strip()] -def _default_config_registry(_plugin_reg: PluginRegistry) -> ConfigRegistry: +def _default_config_registry(lazy: bool = False) -> ConfigRegistry: """Build the config registry from bundled JSON and plugin-config entry points. Args: - _plugin_reg (PluginRegistry): Unused; retained for call-site compatibility. + lazy (bool): Evaluate entry-point configs (recipes) only when requested by name, so that + recipes which inspect every plugin do not defeat lazy plugin loading. Returns: ConfigRegistry: Registry containing bundled and entry-point plugin configs. """ - return ConfigRegistry() + return ConfigRegistry(lazy_entry_point_configs=lazy) def _add_cli_root_globals( parser: argparse.ArgumentParser, - plugin_reg: PluginRegistry, config_reg: ConfigRegistry, ) -> None: """Register top-level flags before ``subcmd`` subparsers (shared with :func:`build_global_argument_parser`).""" @@ -204,28 +205,39 @@ def _add_cli_root_globals( help="Skip plugins that require sudo permissions", ) + parser.add_argument( + "--lazy-load", + dest="lazy_load", + action="store_true", + help="Import plugin modules only when they are needed instead of at startup", + ) + def build_global_argument_parser(*, add_help: bool = True) -> argparse.ArgumentParser: """Globals only (no subcommands), for host CLIs.""" - plugin_reg = PluginRegistry() - config_reg = _default_config_registry(plugin_reg) + config_reg = _default_config_registry(lazy=True) parser = argparse.ArgumentParser( description="node scraper CLI (global options only)", formatter_class=argparse.ArgumentDefaultsHelpFormatter, add_help=add_help, ) - _add_cli_root_globals(parser, plugin_reg, config_reg) + _add_cli_root_globals(parser, config_reg) return parser def build_parser( plugin_reg: PluginRegistry, config_reg: ConfigRegistry, + only_plugins: Optional[Collection[str]] = None, ) -> tuple[argparse.ArgumentParser, dict[str, tuple[argparse.ArgumentParser, dict]]]: """Build an argument parser Args: plugin_reg (PluginRegistry): registry of plugins + config_reg (ConfigRegistry): registry of configs + only_plugins (Optional[Collection[str]]): if given, per-plugin ``run-plugins`` subparsers are + built (which requires importing the plugin class) only for these names. ``None`` builds + a subparser for every registered plugin. Returns: tuple[argparse.ArgumentParser, dict[str, tuple[argparse.ArgumentParser, dict]]]: tuple containing main @@ -236,7 +248,7 @@ def build_parser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - _add_cli_root_globals(parser, plugin_reg, config_reg) + _add_cli_root_globals(parser, config_reg) subparsers = parser.add_subparsers(dest="subcmd", help="Subcommands") subparsers.default = "run-plugins" @@ -370,7 +382,15 @@ def build_parser( ) plugin_subparser_map = {} - for plugin_name, plugin_class in plugin_reg.plugins.items(): + if only_plugins is None: + plugin_items = list(plugin_reg.plugins.items()) + else: + plugin_items = [ + (name, plugin_class) + for name, plugin_class in ((n, plugin_reg.plugins.get(n)) for n in only_plugins) + if plugin_class is not None + ] + for plugin_name, plugin_class in plugin_items: plugin_subparser = plugin_subparsers.add_parser( plugin_name, help=f"Run {plugin_name} plugin", @@ -410,9 +430,9 @@ def get_cli_top_level_subcommands() -> tuple[str, ...]: Returns: Tuple of ``subcmd`` subparser names; call ``cache_clear()`` if registries change in-process. """ - plugin_reg = PluginRegistry() - config_reg = _default_config_registry(plugin_reg) - parser, _plugin_subparser_map = build_parser(plugin_reg, config_reg) + plugin_reg = PluginRegistry(lazy_load=True) + config_reg = _default_config_registry(lazy=True) + parser, _plugin_subparser_map = build_parser(plugin_reg, config_reg, only_plugins=()) return _top_level_subcommand_names(parser) @@ -456,7 +476,8 @@ def setup_logger( encoding="utf-8", ) logging.root.setLevel(log_level_no) - logging.getLogger("paramiko").setLevel(logging.ERROR) + # Paramiko logs full tracebacks at ERROR; nodescraper surfaces SSH failures itself. + logging.getLogger("paramiko").setLevel(logging.CRITICAL) logger = logging.getLogger(DEFAULT_LOGGER) @@ -471,6 +492,34 @@ def _handle_keyboard_interrupt(logger: Optional[logging.Logger] = None) -> None: sys.exit(KEYBOARD_INTERRUPT_EXIT_CODE) +def _lazy_load_flag_present(arg_input: Sequence[str]) -> bool: + """True if ``--lazy-load`` appears among the top-level args (before ``run-plugins``). + + The registry must exist before the real parser is built, so this checks for the flag + on its own. + """ + top_level = list(arg_input) + if "run-plugins" in top_level: + top_level = top_level[: top_level.index("run-plugins")] + pre_parser = argparse.ArgumentParser(add_help=False) + pre_parser.add_argument("--lazy-load", dest="lazy_load", action="store_true") + known, _ = pre_parser.parse_known_args(top_level) + return bool(known.lazy_load) + + +def _plugins_needing_parsers( + plugin_reg: PluginRegistry, arg_input: Sequence[str] +) -> Optional[list[str]]: + """Plugin names whose ``run-plugins`` subparser must be built in lazy mode. + + Returns ``None`` (build everything) when help was requested so help output stays complete. + """ + if any(arg in ("-h", "--help") for arg in arg_input): + return None + _, plugin_arg_map, _ = process_args(list(arg_input), list(plugin_reg.plugins.keys())) + return list(plugin_arg_map) + + def main( arg_input: Optional[list[str]] = None, *, @@ -491,9 +540,14 @@ def main( if arg_input is None: arg_input = sys.argv[1:] - plugin_reg = PluginRegistry() - config_reg = _default_config_registry(plugin_reg) - parser, plugin_subparser_map = build_parser(plugin_reg, config_reg) + # The one registry for this invocation: shared with recipes/discovery and the executor. + plugin_reg = PluginRegistry(lazy_load=_lazy_load_flag_present(arg_input)) + set_default_registry(plugin_reg) + PluginDiscovery.clear_cache() + lazy = bool(getattr(plugin_reg, "lazy", False)) + config_reg = _default_config_registry(lazy=lazy) + only_plugins = _plugins_needing_parsers(plugin_reg, arg_input) if lazy else None + parser, plugin_subparser_map = build_parser(plugin_reg, config_reg, only_plugins) try: top_level_args, plugin_arg_map, invalid_plugins = process_args( @@ -648,6 +702,7 @@ def main( ) if parsed_args.skip_sudo: + # Add skip_sudo to the collection_args of the last plugin config instance plugin_config_inst_list[-1].global_args.setdefault("collection_args", {})[ "skip_sudo" ] = True diff --git a/nodescraper/cli/compare_runs.py b/nodescraper/cli/compare_runs.py index acbb92ca..f116e8fb 100644 --- a/nodescraper/cli/compare_runs.py +++ b/nodescraper/cli/compare_runs.py @@ -125,7 +125,7 @@ def _load_plugin_data_from_run( res_payload = json.loads(Path(res_path).read_text(encoding="utf-8")) task_res = TaskResult(**res_payload) plugin_name = task_res.parent - except (json.JSONDecodeError, TypeError, OSError) as e: + except (json.JSONDecodeError, TypeError, OSError, ValidationError) as e: logger.warning("Skipping %s: failed to load result: %s", res_path, e) continue diff --git a/nodescraper/cli/dynamicparserbuilder.py b/nodescraper/cli/dynamicparserbuilder.py index 8c0c9c68..7a608f8e 100644 --- a/nodescraper/cli/dynamicparserbuilder.py +++ b/nodescraper/cli/dynamicparserbuilder.py @@ -156,11 +156,14 @@ def get_literal_choices(cls, type_class_map: dict) -> Optional[list]: Returns: Optional[list]: list of valid choices for the Literal type, or None if not a Literal """ - # Check if Literal is in the type_class_map literal_type = type_class_map.get(Literal) - if literal_type and literal_type.inner_type is not None: + if literal_type is None or literal_type.inner_type is None: return None - return None + + values = literal_type.inner_type + if isinstance(values, (list, tuple)): + return list(values) + return [values] def add_argument( self, @@ -181,14 +184,16 @@ def add_argument( """ add_kw = {} if help_text is None else {"help": help_text} # Check for Literal types and extract choices - literal_choices = None - if Literal in type_class_map and annotation: - # Extract all arguments from the annotation - args = get_args(annotation) - for arg in args: - if get_origin(arg) is Literal: - literal_choices = list(get_args(arg)) - break + literal_choices = self.get_literal_choices(type_class_map) + if literal_choices is None and annotation is not None: + # fall back to pulling the choices out of the raw annotation + if get_origin(annotation) is Literal: + literal_choices = list(get_args(annotation)) + else: + for arg in get_args(annotation): + if get_origin(arg) is Literal: + literal_choices = list(get_args(arg)) + break if list in type_class_map: type_class = type_class_map[list] @@ -225,7 +230,7 @@ def add_argument( type=str, required=required, choices=literal_choices, - metavar=f"{{{','.join(literal_choices)}}}", + metavar=f"{{{','.join(str(choice) for choice in literal_choices)}}}", **add_kw, ) elif float in type_class_map: diff --git a/nodescraper/cli/helper.py b/nodescraper/cli/helper.py index 1156e369..60ab780c 100644 --- a/nodescraper/cli/helper.py +++ b/nodescraper/cli/helper.py @@ -30,6 +30,7 @@ import logging import os import sys +from copy import deepcopy from pathlib import Path from typing import Optional, Sequence, Tuple @@ -72,14 +73,14 @@ def get_system_info(args: argparse.Namespace) -> SystemInfo: if args.sys_platform: system_info.platform = args.sys_platform - if args.sys_location: + conn = getattr(args, "connection_config", None) or {} + location_name = args.sys_location or conn.get("sys_location") + if location_name: try: - location = getattr(SystemLocation, args.sys_location) + system_info.location = getattr(SystemLocation, str(location_name).upper()) except Exception as e: raise argparse.ArgumentTypeError("Invalid input for system location") from e - system_info.location = location - return system_info @@ -115,14 +116,15 @@ def get_plugin_configs( base_config.global_args["system_interaction_level"] = system_interaction_level - plugin_configs = [base_config] + # Copy each until we are done + plugin_configs = [deepcopy(base_config)] if plugin_config_input: for config in plugin_config_input: if os.path.exists(config): plugin_configs.append(ModelArgHandler(PluginConfig).process_file_arg(config)) elif config in built_in_configs: - plugin_configs.append(built_in_configs[config]) + plugin_configs.append(deepcopy(built_in_configs[config])) else: raise argparse.ArgumentTypeError(f"No plugin config found for: {config}") @@ -238,10 +240,10 @@ def parse_describe( log_cli_text_block(logger, out) elif parsed_args.type == "plugin": - if parsed_args.name not in plugin_reg.plugins: + plugin_class = plugin_reg.plugins.get(parsed_args.name) + if plugin_class is None: logger.error("No plugin found for name: %s", parsed_args.name) sys.exit(1) - plugin_class = plugin_reg.plugins[parsed_args.name] out = [ f"Plugin Name: {parsed_args.name}", f"Description: {getattr(plugin_class, '__doc__', '')}", @@ -269,7 +271,11 @@ def parse_gen_plugin_config( """ try: config = build_config( - config_reg, plugin_reg, logger, parsed_args.plugins, parsed_args.built_in_configs + config_reg, + plugin_reg, + logger, + parsed_args.plugins, + parsed_args.built_in_configs, ) config.name = parsed_args.config_name.split(".")[0] @@ -367,7 +373,7 @@ def generate_reference_config( data_model = obj.result_data.system_data if data_model is None: - logger.warning("Plugin: %s data model not found: %s, skipping", obj.source) + logger.warning("Plugin: %s data model not found: %s, skipping", obj.source, data_model) continue plugin = plugin_reg.plugins.get(obj.source) @@ -585,15 +591,20 @@ def dump_to_csv(all_rows: list, filename: str, fieldnames: list[str], logger: lo fieldnames (list[str]): header for csv file logger (logging.Logger): isntance of logger """ + DEFAULT_ERR_MSG_PREFIX = "Could not dump data to csv file" try: with open(filename, "w", newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) writer.writeheader() for row in all_rows: writer.writerow(row) + logger.info("Data written to csv file: %s", filename) + except FileNotFoundError as fnf_exp: + logger.error("%s, File not found: %s", DEFAULT_ERR_MSG_PREFIX, fnf_exp) + except ValueError as val_exp: + logger.error("%s, Value error: %s", DEFAULT_ERR_MSG_PREFIX, val_exp) except Exception as exp: - logger.error("Could not dump data to csv file: %s", exp) - logger.info("Data written to csv file: %s", filename) + logger.error("%s, Exception: %s", DEFAULT_ERR_MSG_PREFIX, exp) def generate_summary( diff --git a/nodescraper/cli/inputargtypes.py b/nodescraper/cli/inputargtypes.py index 7faa9c5c..18ead989 100644 --- a/nodescraper/cli/inputargtypes.py +++ b/nodescraper/cli/inputargtypes.py @@ -86,6 +86,13 @@ class ModelArgHandler(Generic[TModelType]): def __init__(self, model: Type[TModelType]) -> None: self.model = model + def arg_check(self, json_arg_input_data: dict): + """Check the validity of the JSON argument input data. + It must be a dict which can be ** into a pydantic model of the specified type. + """ + if not isinstance(json_arg_input_data, dict): + raise argparse.ArgumentTypeError("Input data must be a dictionary.") + def process_file_arg(self, file_path: str) -> TModelType: """load a json file into a pydantic model @@ -99,6 +106,7 @@ def process_file_arg(self, file_path: str) -> TModelType: TModelType: model instance """ data = json_arg(file_path) + self.arg_check(data) try: return self.model(**data) except ValidationError as e: diff --git a/nodescraper/configbuilder.py b/nodescraper/configbuilder.py index bc8f1b8a..03b4807d 100644 --- a/nodescraper/configbuilder.py +++ b/nodescraper/configbuilder.py @@ -54,10 +54,9 @@ def gen_config(self, plugin_names: list[str]) -> PluginConfig: """ config = PluginConfig() for plugin in plugin_names: - if plugin in self.plugin_registry.plugins: - config.plugins[plugin] = self._build_plugin_config( - self.plugin_registry.plugins[plugin] - ) + plugin_class = self.plugin_registry.plugins.get(plugin) + if plugin_class is not None: + config.plugins[plugin] = self._build_plugin_config(plugin_class) else: self.logger.warning("No plugin found with name: %s", plugin) return config diff --git a/nodescraper/configregistry.py b/nodescraper/configregistry.py index f6e9f37a..fd60169a 100644 --- a/nodescraper/configregistry.py +++ b/nodescraper/configregistry.py @@ -29,6 +29,7 @@ import inspect import json import os +from collections.abc import Callable, Iterator, MutableMapping from pathlib import Path from typing import Any, Optional @@ -43,6 +44,50 @@ class PluginConfigEntryPointError(RuntimeError): """Raised when a ``nodescraper.plugin_configs`` entry point cannot be loaded.""" +class LazyConfigMapping(MutableMapping): + """Config name to :class:`PluginConfig` mapping that builds each config on first access. + + Listing names never runs a loader, so recipes that inspect every plugin are not + evaluated until their config is actually requested. + """ + + def __init__(self) -> None: + self._values: dict[str, PluginConfig] = {} + self._loaders: dict[str, Callable[[], PluginConfig]] = {} + + def add_loader(self, name: str, loader: Callable[[], PluginConfig]) -> None: + self._values.pop(name, None) + self._loaders[name] = loader + + def __getitem__(self, name: str) -> PluginConfig: + if name in self._values: + return self._values[name] + loader = self._loaders.get(name) + if loader is None: + raise KeyError(name) + self._values[name] = loader() + return self._values[name] + + def __setitem__(self, name: str, value: PluginConfig) -> None: + self._loaders.pop(name, None) + self._values[name] = value + + def __delitem__(self, name: str) -> None: + if name not in self._values and name not in self._loaders: + raise KeyError(name) + self._values.pop(name, None) + self._loaders.pop(name, None) + + def __contains__(self, name: object) -> bool: + return name in self._values or name in self._loaders + + def __iter__(self) -> Iterator[str]: + return iter([*self._values, *(n for n in self._loaders if n not in self._values)]) + + def __len__(self) -> int: + return len(self._values) + sum(1 for n in self._loaders if n not in self._values) + + class ConfigRegistry: """Class to load json plugin configs into models""" @@ -52,6 +97,7 @@ def __init__( self, config_path: Optional[str] = None, load_entry_point_configs: bool = True, + lazy_entry_point_configs: bool = False, ) -> None: """Initialize the config registry. @@ -60,11 +106,28 @@ def __init__( Defaults to None. load_entry_point_configs (bool, optional): Whether to load ``nodescraper.plugin_configs`` entry points. Defaults to True. + lazy_entry_point_configs (bool, optional): If True, entry-point configs are loaded (and + their recipes evaluated) only when first accessed by name. Load errors then surface + at access time instead of construction. Defaults to False. """ - self.configs: dict[str, PluginConfig] = {} + self.configs: MutableMapping[str, PluginConfig] + if lazy_entry_point_configs: + self.configs = LazyConfigMapping() + else: + self.configs = {} self.load_configs(config_path) if load_entry_point_configs: - self.configs.update(self.load_plugin_configs_from_entry_points()) + if lazy_entry_point_configs: + assert isinstance(self.configs, LazyConfigMapping) + for entry_point in self._entry_points_for_group(PLUGIN_CONFIG_ENTRY_POINT_GROUP): + name = getattr(entry_point, "name", None) + if name: + self.configs.add_loader( + name, + lambda ep=entry_point: self._load_entry_point_config(ep)[1], + ) + else: + self.configs.update(self.load_plugin_configs_from_entry_points()) def load_configs(self, config_path: Optional[str] = None): """Load plugin config JSON files into pydantic models. @@ -79,16 +142,19 @@ def load_configs(self, config_path: Optional[str] = None): config_path = Path(config_path) for config_file in config_path.glob("*.json"): - with open(config_file, "r", encoding="utf-8") as in_file: - try: - file_data = json.load(in_file) - config_model = PluginConfig(**file_data) - if config_model.name: - self.configs[config_model.name] = config_model - else: - self.configs[config_file.name] = config_model - except (ValidationError, json.JSONDecodeError): - pass + try: + with open(config_file, "r", encoding="utf-8") as in_file: + try: + file_data = json.load(in_file) + config_model = PluginConfig(**file_data) + if config_model.name: + self.configs[config_model.name] = config_model + else: + self.configs[config_file.name] = config_model + except (ValidationError, json.JSONDecodeError, TypeError) as e: + raise RuntimeError(f"Failed to load config from {config_file}: {e}") from e + except (OSError, IOError, FileNotFoundError) as e: + raise RuntimeError(f"Failed to open config file {config_file}") from e @staticmethod def _entry_points_for_group(group: str): @@ -107,7 +173,9 @@ def _entry_points_for_group(group: str): return all_eps.get(group, []) # type: ignore[assignment, attr-defined, arg-type] @staticmethod - def _resolve_entry_point_config(loaded: Any) -> PluginConfig | dict[str, Any] | None: + def _resolve_entry_point_config( + loaded: Any, + ) -> PluginConfig | dict[str, Any] | None: """Resolve a loaded entry point object into a plugin config. Args: @@ -131,6 +199,50 @@ def _resolve_entry_point_config(loaded: Any) -> PluginConfig | dict[str, Any] | return config_data return None + @classmethod + def _load_entry_point_config(cls, entry_point: Any) -> tuple[Optional[str], PluginConfig]: + """Load one ``nodescraper.plugin_configs`` entry point. + + Returns: + tuple[Optional[str], PluginConfig]: Config key (entry-point name, else config name) and model. + + Raises: + PluginConfigEntryPointError: If the entry point target is missing, invalid, or unsupported. + """ + entry_point_name = getattr(entry_point, "name", None) + entry_point_label = entry_point_name or "" + try: + loaded = entry_point.load() + config_data = cls._resolve_entry_point_config(loaded) + if config_data is None: + raise PluginConfigEntryPointError( + f"Failed to load plugin config entry point {entry_point_label!r}: " + f"unsupported target {loaded!r}" + ) + + config_model = ( + config_data + if isinstance(config_data, PluginConfig) + else PluginConfig(**config_data) + ) + return entry_point_name or config_model.name, config_model + except ModuleNotFoundError as exc: + raise PluginConfigEntryPointError( + f"Failed to load plugin config entry point {entry_point_label!r}: " + f"module not found ({exc}). Check the entry point target in pyproject.toml." + ) from exc + except ValidationError as exc: + raise PluginConfigEntryPointError( + f"Failed to load plugin config entry point {entry_point_label!r}: " + "invalid plugin config" + ) from exc + except PluginConfigEntryPointError: + raise + except Exception as exc: + raise PluginConfigEntryPointError( + f"Failed to load plugin config entry point {entry_point_label!r}: {exc}" + ) from exc + @classmethod def load_plugin_configs_from_entry_points(cls) -> dict[str, PluginConfig]: """Load plugin configs registered under ``nodescraper.plugin_configs`` entry points. @@ -144,40 +256,8 @@ def load_plugin_configs_from_entry_points(cls) -> dict[str, PluginConfig]: configs: dict[str, PluginConfig] = {} for entry_point in cls._entry_points_for_group(PLUGIN_CONFIG_ENTRY_POINT_GROUP): - entry_point_name = getattr(entry_point, "name", None) - entry_point_label = entry_point_name or "" - try: - loaded = entry_point.load() # type: ignore[attr-defined] - config_data = cls._resolve_entry_point_config(loaded) - if config_data is None: - raise PluginConfigEntryPointError( - f"Failed to load plugin config entry point {entry_point_label!r}: " - f"unsupported target {loaded!r}" - ) - - config_model = ( - config_data - if isinstance(config_data, PluginConfig) - else PluginConfig(**config_data) - ) - config_key = entry_point_name or config_model.name - if config_key: - configs[config_key] = config_model - except ModuleNotFoundError as exc: - raise PluginConfigEntryPointError( - f"Failed to load plugin config entry point {entry_point_label!r}: " - f"module not found ({exc}). Check the entry point target in pyproject.toml." - ) from exc - except ValidationError as exc: - raise PluginConfigEntryPointError( - f"Failed to load plugin config entry point {entry_point_label!r}: " - "invalid plugin config" - ) from exc - except PluginConfigEntryPointError: - raise - except Exception as exc: - raise PluginConfigEntryPointError( - f"Failed to load plugin config entry point {entry_point_label!r}: {exc}" - ) from exc + config_key, config_model = cls._load_entry_point_config(entry_point) + if config_key: + configs[config_key] = config_model return configs diff --git a/nodescraper/connection/inband/inband.py b/nodescraper/connection/inband/inband.py index 291491b4..0a38f2b8 100644 --- a/nodescraper/connection/inband/inband.py +++ b/nodescraper/connection/inband/inband.py @@ -25,6 +25,7 @@ ############################################################################### import abc import os +from pathlib import Path from typing import Optional from pydantic import BaseModel @@ -67,6 +68,41 @@ def log_model(self, log_path: str) -> None: def contents_str(self) -> str: pass + def resolve_write_path(self, log_path: str) -> Path: + """Resolve where this artifact should be written and create any parent dirs + + The filename is always treated as relative to log_path. Any anchor and + parent dir references are dropped so that a filename can never write + outside of log_path, while intentional sub directories are preserved. + + Args: + log_path (str): dir that the artifact should be written into + + Raises: + IOError: if the filename has no usable name or the dirs cannot be created + + Returns: + Path: path to write the artifact to + """ + filename_pathlib = Path(self.filename) + parts = [ + part + for part in filename_pathlib.parts + if part not in (filename_pathlib.anchor, os.pardir, os.curdir) + ] + + if not parts: + raise IOError(f"Artifact filename '{self.filename}' does not contain a file name") + + write_path = Path(log_path).joinpath(*parts) + + try: + write_path.parent.mkdir(parents=True, exist_ok=True) + except Exception as e: + raise IOError(f"Failed to create log directory {write_path.parent}: {e}") from e + + return write_path + @classmethod def from_bytes( cls, @@ -107,8 +143,10 @@ def log_model(self, log_path: str) -> None: Args: log_path (str): Path for file """ - path = os.path.join(log_path, self.filename) - with open(path, "w", encoding="utf-8") as f: + # the filename can look like a folder or a file, make sure it goes in log_path + write_path = self.resolve_write_path(log_path) + + with open(write_path, "w", encoding="utf-8") as f: f.write(self.contents) def contents_str(self) -> str: @@ -131,8 +169,9 @@ def log_model(self, log_path: str) -> None: Args: log_path (str): Path for file """ - log_name = os.path.join(log_path, self.filename) - with open(log_name, "wb") as f: + write_path = self.resolve_write_path(log_path) + + with open(write_path, "wb") as f: f.write(self.contents) def contents_str(self) -> str: diff --git a/nodescraper/connection/inband/inbandlocal.py b/nodescraper/connection/inband/inbandlocal.py index 66e655f9..f017b81a 100644 --- a/nodescraper/connection/inband/inbandlocal.py +++ b/nodescraper/connection/inband/inbandlocal.py @@ -1,4 +1,4 @@ -############################################################################### +############################################################################## # # MIT License # @@ -46,7 +46,7 @@ def run_command( Args: command (str): shell command string. - sudo (bool, optional): run command with sudo (Linux only). Defaults to False. + sudo (bool, optional): run command with non-interactive sudo -n (Linux only). Defaults to False. timeout (int, optional): timeout for command in seconds. Defaults to 300. strip (bool, optional): strip output of command. Defaults to True. @@ -54,17 +54,37 @@ def run_command( CommandArtifact: command result object """ if sudo: - command = f"sudo {command}" + command = f"sudo -n {command}" - res = subprocess.run( - command, - encoding="utf-8", - shell=True, - errors="replace", - timeout=timeout, - capture_output=True, - check=False, - ) + try: + res = subprocess.run( + command, + encoding="utf-8", + shell=True, + errors="replace", + timeout=timeout, + capture_output=True, + stdin=subprocess.DEVNULL, + check=False, + ) + except subprocess.TimeoutExpired as exc: + stdout = exc.stdout or "" + stderr = exc.stderr or "" + if isinstance(stdout, bytes): + stdout = stdout.decode("utf-8", errors="replace") + if isinstance(stderr, bytes): + stderr = stderr.decode("utf-8", errors="replace") + timeout_msg = f"Command timed out after {timeout}s" + stderr = f"{stderr}\n{timeout_msg}".strip() if stderr else timeout_msg + if strip: + stdout = stdout.strip() + stderr = stderr.strip() + return CommandArtifact( + command=command, + stdout=stdout, + stderr=stderr, + exit_code=124, + ) return CommandArtifact( command=command, diff --git a/nodescraper/connection/inband/inbandmanager.py b/nodescraper/connection/inband/inbandmanager.py index c1c6bea5..04aead9a 100644 --- a/nodescraper/connection/inband/inbandmanager.py +++ b/nodescraper/connection/inband/inbandmanager.py @@ -26,28 +26,26 @@ from __future__ import annotations from logging import Logger -from typing import Optional, Union +from typing import Any, ClassVar, Optional, Union from nodescraper.enums import ( EventCategory, EventPriority, ExecutionStatus, - OSFamily, SystemLocation, ) from nodescraper.interfaces.connectionmanager import ConnectionManager from nodescraper.interfaces.taskresulthook import TaskResultHook from nodescraper.models import SystemInfo, TaskResult -from nodescraper.utils import get_exception_traceback from .inband import InBandConnection from .inbandlocal import LocalShell from .inbandremote import RemoteShell, SSHConnectionError -from .osdetection import NetworkOsDetection, detect_network_os from .sshparams import SSHConnectionParams class InBandConnectionManager(ConnectionManager[InBandConnection, SSHConnectionParams]): + CONNECTION_ARGS_MODEL: ClassVar[type[SSHConnectionParams]] = SSHConnectionParams def __init__( self, @@ -56,9 +54,20 @@ def __init__( max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, parent: Optional[str] = None, task_result_hooks: Optional[list[TaskResultHook]] = None, - connection_args: Optional[SSHConnectionParams] = None, + connection_args: Optional[SSHConnectionParams | dict[str, Any]] = None, **kwargs, ): + """Initialize a new InBandConnectionManager instance. + + Args: + system_info (SystemInfo): System info for the targeted system + logger (Optional[Logger], optional): Logger for the connection manager. Defaults to None. + max_event_priority_level (Union[EventPriority, str], optional): Maximum event priority level for logging. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): Parent task identifier. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook]], optional): List of task result hooks. Defaults to None. + connection_args (Optional[SSHConnectionParams], optional): SSH connection parameters + These connections args will be used to establish the SSH connection to the targeted system. Defaults to None. + """ super().__init__( system_info, logger, @@ -69,52 +78,6 @@ def __init__( **kwargs, ) - @staticmethod - def _apply_network_os_detection( - system_info: SystemInfo, - detection: NetworkOsDetection, - ) -> None: - """Apply network OS probe results to system info.""" - system_info.os_family = detection.os_family - system_info.platform = detection.platform - if system_info.metadata is None: - system_info.metadata = {} - system_info.metadata.update(detection.metadata) - - def _check_os_family(self): - """Check the OS family of the system under test (SUT) - - Raises: - RuntimeError: If the connection is not initialized - """ - if not self.connection: - raise RuntimeError("Connection not initialized") - - self.logger.info("Checking OS family") - res = self.connection.run_command("uname -s") - if "not recognized as an internal or external command" in res.stdout + res.stderr: - self.system_info.os_family = OSFamily.WINDOWS - elif res.exit_code == 0: - self.system_info.os_family = OSFamily.LINUX - else: - detection = detect_network_os(self.connection) - if detection is not None: - self._apply_network_os_detection(self.system_info, detection) - else: - self._log_event( - category=EventCategory.UNKNOWN, - description="Unable to determine SUT OS", - priority=EventPriority.WARNING, - ) - if self.system_info.platform: - self.logger.info( - "OS Family: %s (%s)", - self.system_info.os_family.name, - self.system_info.platform, - ) - else: - self.logger.info("OS Family: %s", self.system_info.os_family.name) - def connect( self, ) -> TaskResult: @@ -126,50 +89,52 @@ def connect( if self.system_info.location == SystemLocation.LOCAL: self.logger.info("Using local shell") self.connection = LocalShell() - self._check_os_family() return self.result - if not self.connection_args or not isinstance(self.connection_args, SSHConnectionParams): - if not self.connection_args: - message = "No SSH credentials provided" - else: - message = "Invalide SSH creddentials provided" + if not self.connection_args: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to the system: no SSH credentials were provided.", + "No SSH connection arguments provided to InBandConnectionManager", + {"connection_args": None}, + ) - self._log_event( - category=EventCategory.RUNTIME, - description=message, - priority=EventPriority.CRITICAL, - console_log=True, + if not isinstance(self.connection_args, SSHConnectionParams): + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to the system: the SSH credentials are not valid.", + "SSH connection arguments are not SSHConnectionParams " + f"(got {type(self.connection_args).__name__})", + {"connection_args_type": type(self.connection_args).__name__}, ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - return self.result + hostname = self.connection_args.hostname + self.logger.info("Initializing SSH connection to system '%s'", hostname) + shell = RemoteShell(self.connection_args) try: - self.logger.info( - "Initializing SSH connection to system '%s'", self.connection_args.hostname - ) - self.connection = RemoteShell(self.connection_args) - self.connection.connect_ssh() - self._check_os_family() + shell.connect_ssh() except SSHConnectionError as exception: - self._log_event( - category=EventCategory.SSH, - description=f"{str(exception)}", - priority=EventPriority.CRITICAL, - console_log=True, - ) - except Exception as exception: - self._log_event( - category=EventCategory.SSH, - description=f"Exception during SSH: {str(exception)}", - data=get_exception_traceback(exception), - priority=EventPriority.CRITICAL, - console_log=True, + shell.client.close() + port = self.connection_args.port + return self._fail_connect( + EventCategory.SSH, + "Could not connect to the system over SSH.", + f"SSH connection to '{hostname}:{port}' failed: {exception}", + { + "hostname": str(hostname), + "port": port, + "username": self.connection_args.username, + "exception": str(exception), + }, ) + + self.connection = shell + self.result.message = f"SSH connection to '{hostname}' established successfully" + self.result.status = ExecutionStatus.OK return self.result def disconnect(self): """Disconnect in-band connection""" - super().disconnect() if isinstance(self.connection, RemoteShell): self.connection.client.close() + super().disconnect() diff --git a/nodescraper/connection/inband/inbandremote.py b/nodescraper/connection/inband/inbandremote.py index 6b8dbf56..76f9fb4e 100644 --- a/nodescraper/connection/inband/inbandremote.py +++ b/nodescraper/connection/inband/inbandremote.py @@ -23,9 +23,10 @@ # SOFTWARE. # ############################################################################### +import logging import os import socket -from typing import Type, Union +from typing import Any, Type, Union import paramiko from paramiko.ssh_exception import ( @@ -46,20 +47,50 @@ class SSHConnectionError(Exception): """A general exception for ssh connection failures""" +CHANNEL_OPEN_TIMEOUT = 60 + + class RemoteShell(InBandConnection): """Utility class for running shell commands""" host_key_policy: Type[paramiko.MissingHostKeyPolicy] = paramiko.RejectPolicy + _proxy_ssh_down: bool = False def __init__( self, ssh_params: SSHConnectionParams, ) -> None: + """Creates a RemoteShell Instance. + + Args: + ssh_params (SSHConnectionParams): The parameters used to initiate the SSH connection username/password/ect. + """ self.ssh_params = ssh_params + self._new_client() + + def _new_client(self) -> None: + """Replace the SSH client with a new unconnected client. + + Returns: + None. + """ self.client = paramiko.SSHClient() self.client.load_system_host_keys() self.client.set_missing_host_key_policy(self.host_key_policy()) + def _arm_channel_timeout(self) -> None: + """Bound channel opens that do not pass their own timeout. + + Returns: + None. + """ + transport = self.client.get_transport() + if transport is None: + return + runtime_transport: Any = transport + if runtime_transport.channel_timeout is None: + runtime_transport.channel_timeout = CHANNEL_OPEN_TIMEOUT + def connect_ssh(self): """Connect to the remote host via SSH""" try: @@ -79,6 +110,7 @@ def connect_ssh(self): auth_timeout=60, banner_timeout=200, ) + self._arm_channel_timeout() except socket.timeout: raise SSHConnectionError("SSH Request timeout") from socket.timeout except socket.gaierror as e: @@ -93,8 +125,23 @@ def connect_ssh(self): raise SSHConnectionError(f"Unable to establish SSH connection: {str(e)}") from e except EOFError as e: raise SSHConnectionError("EOFError during SSH connection") from e + except OSError as e: + raise SSHConnectionError(f"SSH network error: {e}") from e except Exception as e: - raise e + raise SSHConnectionError(f"Unexpected error during SSH connection: {str(e)}") from e + + def reconnect_ssh(self) -> None: + """Close the current SSH session and open a new one. + + Returns: + None. + """ + try: + self.client.close() + except (OSError, SSHException): + pass + self._new_client() + self.connect_ssh() def read_file( self, @@ -112,8 +159,7 @@ def read_file( Returns: BaseFileArtifact: Object representing file contents """ - with self.client.open_sftp().open(filename, "rb") as remote_file: - raw_contents = remote_file.read() + raw_contents = self.read_files([filename])[0] return BaseFileArtifact.from_bytes( filename=os.path.basename(filename), raw_contents=raw_contents, @@ -121,6 +167,26 @@ def read_file( strip=strip, ) + def read_files(self, filenames: list[str]) -> list[bytes]: + """Read remote files through one SFTP session. + + Args: + filenames: Remote paths to read. + + Returns: + File contents in the same order as filenames. + """ + self._arm_channel_timeout() + sftp = self.client.open_sftp() + try: + contents: list[bytes] = [] + for filename in filenames: + with sftp.open(filename, "rb") as remote_file: + contents.append(remote_file.read()) + return contents + finally: + sftp.close() + def run_command( self, command: str, @@ -147,18 +213,21 @@ def run_command( elif sudo: cmd_str = f"sudo {cmd_str}" + channel = None try: stdin, stdout, stderr = self.client.exec_command(cmd_str, timeout=timeout) - + channel = stdout.channel if write_password: stdin.write( - self.ssh_params.password.get_secret_value() - if self.ssh_params.password - else "" + "\n" + ( + self.ssh_params.password.get_secret_value() + if self.ssh_params.password + else "" + ) + + "\n" ) stdin.flush() stdin.channel.shutdown_write() - stdout_str = stdout.read().decode("utf-8", errors="replace") stderr_str = stderr.read().decode("utf-8", errors="replace") exit_code = stdout.channel.recv_exit_status() @@ -166,6 +235,21 @@ def run_command( stderr_str = "Command timed out" stdout_str = "" exit_code = 124 + except SSHException as exc: + stderr_str = str(exc) or "SSH channel failed" + stdout_str = "" + exit_code = 124 + except Exception as exc: + logging.getLogger(__name__).exception("Unexpected error running SSH command") + stderr_str = f"{type(exc).__name__}: {exc}" + stdout_str = "" + exit_code = 124 + finally: + if channel is not None: + try: + channel.close() + except (OSError, SSHException): + pass return CommandArtifact( command=cmd_str, diff --git a/nodescraper/connection/inband/osdetection.py b/nodescraper/connection/inband/osdetection.py index 9353439d..2441cf48 100644 --- a/nodescraper/connection/inband/osdetection.py +++ b/nodescraper/connection/inband/osdetection.py @@ -24,11 +24,14 @@ # ############################################################################### import json +import logging import re from dataclasses import dataclass from typing import Optional +from nodescraper.connection.inband import InBandConnectionManager from nodescraper.enums import OSFamily +from nodescraper.models import SystemInfo from .inband import InBandConnection @@ -150,3 +153,57 @@ def detect_network_os(connection: InBandConnection) -> Optional[NetworkOsDetecti return detection return None + + +def apply_network_os_detection( + system_info: SystemInfo, + detection: NetworkOsDetection, +) -> None: + """Apply network OS probe results to system info.""" + system_info.os_family = detection.os_family + system_info.platform = detection.platform + if system_info.metadata is None: + system_info.metadata = {} + system_info.metadata.update(detection.metadata) + + +def discover_and_write_os_family( + connection_manager: InBandConnectionManager, + system_info: SystemInfo, + logger: logging.Logger, +) -> None: + """Check + + Args: + connection_manager (InBandConnectionManager): _description_ + system_info (SystemInfo): _description_ + logger (logging.Logger): _description_ + """ + if connection_manager.connection is None: + logger.error("Connection is not initialized, OS family check cannot be performed.") + return + logger.info("Checking OS family") + res = connection_manager.connection.run_command("uname -s") + if "not recognized as an internal or external command" in res.stdout + res.stderr: + system_info.os_family = OSFamily.WINDOWS + elif res.exit_code == 0 and "VMkernel" in res.stdout: + system_info.os_family = OSFamily.ESXI + elif res.exit_code == 0: + system_info.os_family = OSFamily.LINUX + else: + detection = detect_network_os(connection_manager.connection) + if detection is not None: + apply_network_os_detection(system_info, detection) + else: + logger.warning( + "Unable to determine OS family. uname failed and no supported network OS detected." + ) + + if system_info.platform: + logger.info( + "OS Family: %s (%s)", + system_info.os_family.name, + system_info.platform, + ) + else: + logger.info("OS Family: %s", system_info.os_family.name) diff --git a/nodescraper/connection/oob_ssh/oob_ssh_connection_manager.py b/nodescraper/connection/oob_ssh/oob_ssh_connection_manager.py index 823c00cb..cbd5d737 100644 --- a/nodescraper/connection/oob_ssh/oob_ssh_connection_manager.py +++ b/nodescraper/connection/oob_ssh/oob_ssh_connection_manager.py @@ -26,13 +26,12 @@ from __future__ import annotations from logging import Logger -from typing import Optional, Union +from typing import ClassVar, Optional, Union from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.interfaces.connectionmanager import ConnectionManager from nodescraper.interfaces.taskresulthook import TaskResultHook from nodescraper.models import SystemInfo, TaskResult -from nodescraper.utils import get_exception_traceback from ..inband.inband import InBandConnection from ..inband.inbandremote import RemoteShell, SSHConnectionError @@ -42,6 +41,8 @@ class OobSshConnectionManager(ConnectionManager[InBandConnection, RedfishConnectionParams]): """SSH to the BMC using the same host and credentials as Redfish (OOB shell).""" + CONNECTION_ARGS_MODEL: ClassVar[type[RedfishConnectionParams]] = RedfishConnectionParams + def __init__( self, system_info: SystemInfo, @@ -64,54 +65,83 @@ def __init__( def connect(self) -> TaskResult: if not self.connection_args: - self._log_event( - category=EventCategory.RUNTIME, - description="No Redfish connection parameters provided for OOB SSH", - priority=EventPriority.CRITICAL, - console_log=True, + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to the BMC over SSH: no connection details were provided.", + "No Redfish connection parameters provided for OOB SSH", + {"connection_args": None}, ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - return self.result raw = self.connection_args if isinstance(raw, dict): - params = RedfishConnectionParams.model_validate(raw) + try: + params = RedfishConnectionParams.model_validate(raw) + except ValueError as exception: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to the BMC over SSH: the connection details are not valid.", + f"Invalid Redfish connection parameters for OOB SSH: {exception}", + {"exception": str(exception)}, + ) elif isinstance(raw, RedfishConnectionParams): params = raw else: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to the BMC over SSH: the connection details are not valid.", + "Redfish connection_args must be dict or RedfishConnectionParams " + f"(got {type(raw).__name__})", + {"connection_args_type": type(raw).__name__}, + ) + + if params.is_multi_target and params.host is None: + message = ( + "OOB SSH plugins use a single BMC host. " + "A targets list applies only to Redfish plugins." + ) self._log_event( category=EventCategory.RUNTIME, - description="Redfish connection_args must be dict or RedfishConnectionParams", - priority=EventPriority.CRITICAL, + description=message, + priority=EventPriority.WARNING, console_log=True, ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE + self.result.status = ExecutionStatus.NOT_RAN + self.result.message = message return self.result try: ssh_params = redfish_params_to_ssh(params) - self.logger.info("Initializing OOB SSH to BMC host %s", ssh_params.hostname) - self.connection = RemoteShell(ssh_params) - self.connection.connect_ssh() - except SSHConnectionError as exception: - self._log_event( - category=EventCategory.SSH, - description=str(exception), - priority=EventPriority.CRITICAL, - console_log=True, + except ValueError as exception: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to the BMC over SSH: the connection details are not valid.", + f"Could not build OOB SSH parameters from the Redfish parameters: {exception}", + { + "host": str(params.host), + "exception": str(exception), + }, ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - self.connection = None - except Exception as exception: - self._log_event( - category=EventCategory.SSH, - description=f"Exception during OOB SSH: {exception!s}", - data=get_exception_traceback(exception), - priority=EventPriority.CRITICAL, - console_log=True, + + hostname = ssh_params.hostname + self.logger.info("Initializing OOB SSH to BMC host %s", hostname) + shell = RemoteShell(ssh_params) + try: + shell.connect_ssh() + except SSHConnectionError as exception: + shell.client.close() + return self._fail_connect( + EventCategory.SSH, + "Could not connect to the BMC over SSH.", + f"OOB SSH connection to BMC '{hostname}:{ssh_params.port}' failed: {exception}", + { + "hostname": str(hostname), + "port": ssh_params.port, + "username": ssh_params.username, + "exception": str(exception), + }, ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - self.connection = None + + self.connection = shell return self.result def disconnect(self) -> None: diff --git a/nodescraper/connection/redfish/__init__.py b/nodescraper/connection/redfish/__init__.py index 12b5af16..cd875a0f 100644 --- a/nodescraper/connection/redfish/__init__.py +++ b/nodescraper/connection/redfish/__init__.py @@ -1,59 +1,88 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from .redfish_connection import ( - RedfishConnection, - RedfishConnectionError, - RedfishGetResult, -) -from .redfish_constants import ( - RF_MEMBERS, - RF_MEMBERS_COUNT, - RF_MEMBERS_NEXT_LINK, - RF_ODATA_ID, -) -from .redfish_manager import RedfishConnectionManager -from .redfish_oem_diag import ( - collect_oem_diagnostic_data, - get_oem_diagnostic_allowable_values, -) -from .redfish_params import RedfishConnectionParams, redfish_params_to_ssh -from .redfish_path import RedfishPath - -__all__ = [ - "RedfishConnection", - "RedfishConnectionError", - "RedfishGetResult", - "RedfishConnectionManager", - "RedfishConnectionParams", - "redfish_params_to_ssh", - "RedfishPath", - "collect_oem_diagnostic_data", - "get_oem_diagnostic_allowable_values", - "RF_MEMBERS", - "RF_MEMBERS_COUNT", - "RF_MEMBERS_NEXT_LINK", - "RF_ODATA_ID", -] +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from .redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + clear_redfish_logs, + discover_clear_log_endpoints, +) +from .redfish_connection import ( + RedfishConnection, + RedfishConnectionError, + RedfishGetResult, + RedfishPostResult, +) +from .redfish_constants import ( + RF_MEMBERS, + RF_MEMBERS_COUNT, + RF_MEMBERS_NEXT_LINK, + RF_ODATA_ID, +) +from .redfish_manager import ( + MultiTargetRedfishConnection, + RedfishConnectionManager, + collected_multi_target_data, +) +from .redfish_oem_diag import ( + collect_oem_diagnostic_data, + get_oem_diagnostic_allowable_values, +) +from .redfish_params import ( + RedfishConnectionParams, + RedfishTargetParams, + redfish_params_to_ssh, +) +from .redfish_path import RedfishPath +from .ssh_proxy_connection import SshProxyRedfishConnection +from .ssh_proxy_manager import RedfishSshProxyConnectionManager +from .ssh_proxy_params import RedfishSshProxyConnectionParams + +__all__ = [ + "ClearLogEndpoint", + "ClearLogResult", + "clear_redfish_logs", + "discover_clear_log_endpoints", + "RedfishConnection", + "RedfishConnectionError", + "RedfishGetResult", + "RedfishPostResult", + "MultiTargetRedfishConnection", + "RedfishConnectionManager", + "collected_multi_target_data", + "RedfishSshProxyConnectionManager", + "RedfishSshProxyConnectionParams", + "SshProxyRedfishConnection", + "RedfishConnectionParams", + "RedfishTargetParams", + "redfish_params_to_ssh", + "RedfishPath", + "collect_oem_diagnostic_data", + "get_oem_diagnostic_allowable_values", + "RF_MEMBERS", + "RF_MEMBERS_COUNT", + "RF_MEMBERS_NEXT_LINK", + "RF_ODATA_ID", +] diff --git a/nodescraper/connection/redfish/redfish_clear_log.py b/nodescraper/connection/redfish/redfish_clear_log.py new file mode 100644 index 00000000..84ca6c60 --- /dev/null +++ b/nodescraper/connection/redfish/redfish_clear_log.py @@ -0,0 +1,267 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Redfish LogService.ClearLog discovery and execution.""" + +from __future__ import annotations + +import logging +from typing import Optional, Sequence + +from pydantic import BaseModel + +from .redfish_connection import RedfishConnection, RedfishConnectionError +from .redfish_constants import RF_MEMBERS, RF_ODATA_ID + +_module_logger = logging.getLogger(__name__) + +_ACTION_CLEAR_LOG_KEYS = ("#LogService.ClearLog", "LogService.ClearLog") +_NEVER_OVERWRITE = "NeverOverWrites" + +# Redfish DSP0266 standard LogServices sub-resource segment +_LOG_SERVICES_SEGMENT = "LogServices" + + +class ClearLogEndpoint(BaseModel): + """A discovered Redfish LogService.ClearLog action endpoint.""" + + path: str + overwrite_policy: str = "" + + +class ClearLogResult(BaseModel): + """The result of a single LogService.ClearLog POST.""" + + path: str + success: bool + status_code: Optional[int] = None + error: Optional[str] = None + + +def _odata_id_to_path(odata_id: str, base_url: str) -> str: + """Normalize an @odata.id value to a relative path (no leading slash). + + Args: + odata_id: Raw @odata.id string (relative path or full URL). + base_url: Connection base URL, used to strip the host prefix. + + Returns: + Relative path with no leading slash, or an empty string if invalid. + """ + if not odata_id or not isinstance(odata_id, str): + return "" + s = odata_id.strip() + base = base_url.rstrip("/") + if s.startswith(base + "/"): + s = s[len(base) :] + if s.startswith(("http://", "https://")): + from urllib.parse import urlparse + + s = urlparse(s).path or "/" + return s.lstrip("/") + + +def _parse_clear_endpoint(svc_data: dict) -> Optional[ClearLogEndpoint]: + """Extract a ClearLogEndpoint from a LogService resource body. + + Args: + svc_data: Parsed LogService JSON body. + + Returns: + ClearLogEndpoint if the service advertises ClearLog, else None. + """ + actions = svc_data.get("Actions") or {} + for key in _ACTION_CLEAR_LOG_KEYS: + action = actions.get(key) + if not isinstance(action, dict): + continue + target = action.get("target") or "" + if not target: + continue + return ClearLogEndpoint( + path=target.strip(), + overwrite_policy=svc_data.get("OverWritePolicy", ""), + ) + return None + + +def discover_clear_log_endpoints( + conn: RedfishConnection, + roots: Sequence[str] = ("Systems", "Managers"), + member_ids: Optional[dict[str, list[str]]] = None, + logger: Optional[logging.Logger] = None, +) -> list[ClearLogEndpoint]: + """Walk Redfish Systems/Managers LogServices to find ClearLog action endpoints. + + For each root, either walks the collection or uses explicit member IDs. + Follows the same discovery pattern as ARC's RedfishTool._discover_log_services. + + Args: + conn: Established Redfish connection. + roots: Root collections to search ("Systems", "Managers"). + member_ids: Optional map of root → list of member IDs to use directly + (bypasses the root collection GET). E.g. {"Managers": ["AMC"]}. + logger: Logger instance. + + Returns: + List of discovered ClearLogEndpoint objects. + """ + log = logger or _module_logger + api_root = (getattr(conn, "api_root", None) or "redfish/v1").strip("/") + base_url = getattr(conn, "base_url", "") + endpoints: list[ClearLogEndpoint] = [] + ids_map = member_ids or {} + + for root in roots: + explicit_ids = ids_map.get(root) + if explicit_ids is not None: + members = [{RF_ODATA_ID: f"/{api_root}/{root}/{mid}"} for mid in explicit_ids] + else: + try: + root_data = conn.get_response(f"/{api_root}/{root}") + if not root_data.ok: + log.debug( + "LogService discovery: GET %s/%s returned %s", + api_root, + root, + root_data.status_code, + ) + continue + try: + root_body = root_data.json() + except Exception: + continue + members = (root_body or {}).get(RF_MEMBERS) or [] + except RedfishConnectionError as exc: + log.debug("LogService discovery: failed to GET %s/%s: %s", api_root, root, exc) + continue + + for member in members: + if not isinstance(member, dict): + continue + odata_id = member.get(RF_ODATA_ID, "") + member_path = _odata_id_to_path(odata_id, base_url) + if not member_path: + continue + + ls_path = f"/{member_path}/{_LOG_SERVICES_SEGMENT}" + try: + ls_resp = conn.get_response(ls_path) + if not ls_resp.ok: + continue + try: + ls_body = ls_resp.json() + except Exception: + continue + except RedfishConnectionError: + continue + + for ls_member in (ls_body or {}).get(RF_MEMBERS) or []: + if not isinstance(ls_member, dict): + continue + ls_odata = ls_member.get(RF_ODATA_ID, "") + ls_svc_path = _odata_id_to_path(ls_odata, base_url) + if not ls_svc_path: + continue + + try: + svc_resp = conn.get_response(f"/{ls_svc_path}") + if not svc_resp.ok: + continue + try: + svc_data = svc_resp.json() + except Exception: + continue + except RedfishConnectionError: + continue + + endpoint = _parse_clear_endpoint(svc_data) + if endpoint is not None: + endpoints.append(endpoint) + log.debug( + "Discovered ClearLog endpoint: %s (OverWritePolicy=%s)", + endpoint.path, + endpoint.overwrite_policy, + ) + + if not endpoints: + log.debug("No ClearLog endpoints discovered in %s", list(roots)) + return endpoints + + +def clear_redfish_logs( + conn: RedfishConnection, + endpoints: list[ClearLogEndpoint], + logger: Optional[logging.Logger] = None, +) -> list[ClearLogResult]: + """POST LogService.ClearLog to each discovered endpoint, skipping NeverOverWrites. + + Mirrors ARC's RedfishTool.clear_log_stores: filters out NeverOverWrites + stores and continues on individual failures. + + Args: + conn: Established Redfish connection. + endpoints: Discovered clear-log endpoints from discover_clear_log_endpoints. + logger: Logger instance. + + Returns: + List of ClearLogResult, one per clearable endpoint. Endpoints with + OverWritePolicy=NeverOverWrites are omitted (not attempted). + """ + log = logger or _module_logger + clearable = [e for e in endpoints if e.overwrite_policy != _NEVER_OVERWRITE] + results: list[ClearLogResult] = [] + + for endpoint in clearable: + path = endpoint.path + try: + resp = conn.post(path, json={}) + status_code = resp.status_code + success = status_code in (200, 202, 204) + if success: + log.debug("Cleared log store: %s (status %s)", path, status_code) + results.append(ClearLogResult(path=path, success=True, status_code=status_code)) + else: + body_snippet = "" + try: + body_snippet = resp.text[:300] if resp.text else "" + except Exception: + pass + error = f"POST {path} returned {status_code}" + if body_snippet: + error = f"{error}: {body_snippet}" + log.warning("Failed to clear log store %s: %s", path, error) + results.append( + ClearLogResult(path=path, success=False, status_code=status_code, error=error) + ) + except RedfishConnectionError as exc: + log.warning("Failed to clear log store %s: %s", path, exc) + results.append(ClearLogResult(path=path, success=False, error=str(exc))) + + skipped = [e for e in endpoints if e.overwrite_policy == _NEVER_OVERWRITE] + for endpoint in skipped: + log.debug("Skipping ClearLog for %s: OverWritePolicy=NeverOverWrites", endpoint.path) + + return results diff --git a/nodescraper/connection/redfish/redfish_connection.py b/nodescraper/connection/redfish/redfish_connection.py index eabee716..2fa92236 100644 --- a/nodescraper/connection/redfish/redfish_connection.py +++ b/nodescraper/connection/redfish/redfish_connection.py @@ -1,325 +1,415 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from __future__ import annotations - -import json -import socket -from typing import Any, Callable, ClassVar, Optional, TypeVar, Union -from urllib.parse import urljoin, urlparse - -import requests -import urllib3 # type: ignore[import-untyped] -from pydantic import BaseModel -from requests import Response -from requests.auth import HTTPBasicAuth - -from .redfish_constants import RF_MEMBERS, RF_MEMBERS_COUNT, RF_MEMBERS_NEXT_LINK -from .redfish_path import RedfishPath - -DEFAULT_REDFISH_API_ROOT = "redfish/v1" - -_T = TypeVar("_T") - - -class RedfishGetResult(BaseModel): - """Artifact for the result of a Redfish GET request. - Logged under the same filename as inband command artifacts (command_artifacts.json). - """ - - ARTIFACT_LOG_BASENAME: ClassVar[str] = "command_artifacts" - - path: str - success: bool - data: Optional[dict[str, Any]] = None - error: Optional[str] = None - status_code: Optional[int] = None - log_html: bool = False - - def to_html_entry(self) -> dict: - """Return a dict suitable for HTML command artifact rendering.""" - stdout = json.dumps(self.data, indent=2, sort_keys=True) if self.data is not None else "" - stderr = self.error or "" - exit_code = 0 if self.success else (self.status_code if self.status_code is not None else 1) - return { - "command": f"GET {self.path}", - "stdout": stdout, - "stderr": stderr, - "exit_code": exit_code, - } - - -class RedfishConnectionError(Exception): - """Raised when a Redfish API request fails.""" - - def __init__(self, message: str, response: Optional[Response] = None): - super().__init__(message) - self.response = response - - -class RedfishConnection: - """Redfish REST client for GET requests.""" - - def __init__( - self, - base_url: str, - username: str, - password: Optional[str] = None, - timeout: float = 10.0, - use_session_auth: bool = True, - verify_ssl: bool = True, - api_root: Optional[str] = None, - ): - self.base_url = base_url.rstrip("/") - self.api_root = (api_root or DEFAULT_REDFISH_API_ROOT).strip("/") - self.username = username - self.password = password or "" - self.timeout = timeout - self.use_session_auth = use_session_auth - self.verify_ssl = verify_ssl - self._session: Optional[requests.Session] = None - self._session_token: Optional[str] = None - self._session_uri: Optional[str] = None # For logout DELETE - - def _host_label(self) -> str: - return urlparse(self.base_url).hostname or self.base_url - - @staticmethod - def _is_name_resolution_error(exc: BaseException) -> bool: - current: Optional[BaseException] = exc - seen: set[int] = set() - while current is not None and id(current) not in seen: - if isinstance(current, socket.gaierror): - return True - if type(current).__name__ == "NameResolutionError": - return True - seen.add(id(current)) - current = current.__cause__ or current.__context__ - message = str(exc) - return "Failed to resolve" in message or "No address associated with hostname" in message - - def _transport_error_message(self, exc: Exception) -> str: - host = self._host_label() - if isinstance(exc, socket.gaierror) or self._is_name_resolution_error(exc): - return f"Redfish hostname could not be resolved: {host}" - if isinstance(exc, requests.exceptions.Timeout): - return f"Redfish connection timed out: {host}" - if isinstance(exc, requests.exceptions.ConnectionError): - return f"Redfish connection failed: {host}" - return f"Redfish connection failed: {exc}" - - def _execute_request(self, request_fn: Callable[[], _T]) -> _T: - try: - return request_fn() - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exc: - raise RedfishConnectionError(self._transport_error_message(exc)) from exc - except socket.gaierror as exc: - raise RedfishConnectionError(self._transport_error_message(exc)) from exc - - def _ensure_session(self) -> requests.Session: - if self._session is None: - if not self.verify_ssl: - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - self._session = requests.Session() - self._session.verify = self.verify_ssl - if not self.verify_ssl: - self._session.trust_env = False - self._session.headers["Content-Type"] = "application/json" - self._session.headers["Accept"] = "application/json" - if self.use_session_auth and self.password: - self._login_session() - elif self.password: - self._session.auth = HTTPBasicAuth(self.username, self.password) - return self._session - - def _login_session(self) -> None: - """Create a Redfish session and set X-Auth-Token.""" - session = self._session - assert session is not None - sess_url = urljoin(self.base_url + "/", f"{self.api_root}/SessionService/Sessions") - payload = {"UserName": self.username, "Password": self.password} - resp = self._execute_request( - lambda: session.post( - sess_url, - json=payload, - timeout=self.timeout, - ) - ) - if not resp.ok: - raise RedfishConnectionError( - f"Session login failed: {resp.status_code} {resp.reason}", response=resp - ) - self._session_token = resp.headers.get("X-Auth-Token") - location = resp.headers.get("Location") - if location: - self._session_uri = ( - location - if location.startswith("http") - else urljoin(self.base_url + "/", location.lstrip("/")) - ) - if self._session_token: - session.headers["X-Auth-Token"] = self._session_token - else: - session.auth = HTTPBasicAuth(self.username, self.password) - - def get(self, path: RedfishPath) -> dict[str, Any]: - """GET a Redfish path and return the JSON body. path must be a RedfishPath.""" - path_str = str(path) - resp = self.get_response(path_str) - if not resp.ok: - raise RedfishConnectionError( - f"GET {path_str} failed: {resp.status_code} {resp.reason}", - response=resp, - ) - return resp.json() - - def get_response(self, path: Union[str, "RedfishPath"]) -> Response: - """GET a Redfish path and return the raw Response. path may be a string or RedfishPath.""" - path = str(path) - session = self._ensure_session() - url = path if path.startswith("http") else urljoin(self.base_url + "/", path.lstrip("/")) - return self._execute_request(lambda: session.get(url, timeout=self.timeout)) - - def post( - self, path: Union[str, "RedfishPath"], json: Optional[dict[str, Any]] = None - ) -> Response: - """POST to a Redfish path and return the raw Response. path may be a string or RedfishPath.""" - path = str(path) - session = self._ensure_session() - url = path if path.startswith("http") else urljoin(self.base_url + "/", path.lstrip("/")) - return self._execute_request( - lambda: session.post(url, json=json or {}, timeout=self.timeout) - ) - - def run_get(self, path: Union[str, RedfishPath]) -> RedfishGetResult: - """Run a Redfish GET request and return a result object. path may be a string or RedfishPath.""" - path_norm = str(path).strip() - if not path_norm.startswith("/"): - path_norm = "/" + path_norm - path_obj = RedfishPath(path_norm.strip("/")) if isinstance(path, str) else path - try: - data = self.get(path_obj) - return RedfishGetResult( - path=path_norm, - success=True, - data=data, - status_code=200, - ) - except RedfishConnectionError as e: - status = e.response.status_code if e.response is not None else None - return RedfishGetResult( - path=path_norm, - success=False, - error=str(e), - status_code=status, - ) - except Exception as e: - return RedfishGetResult( - path=path_norm, - success=False, - error=str(e), - status_code=None, - ) - - def run_get_paged( - self, - path: Union[str, RedfishPath], - max_pages: int = 200, - ) -> RedfishGetResult: - """Run a Redfish GET and transparently follow Members@odata.nextLink pagination. - - Each subsequent page's Members list is appended to the first page's Members list - so the caller receives a single merged response body. The Members@odata.nextLink key - and Members@odata.count are updated to reflect the merged result. If there is no - Members@odata.nextLink in the first response this behaves identically to run_get. - max_pages is a safety cap on the number of pages to fetch (default 200). - """ - first = self.run_get(path) - if not first.success or first.data is None: - return first - - # Short-circuit when there is nothing to page through. - if RF_MEMBERS_NEXT_LINK not in first.data: - return first - - merged_members: list = list(first.data.get(RF_MEMBERS) or []) - merged_data: dict = dict(first.data) - pages_fetched = 1 - next_link: Optional[str] = first.data.get(RF_MEMBERS_NEXT_LINK) - last_status_code = first.status_code - - while next_link and pages_fetched < max_pages: - page_result = self.run_get(next_link) - last_status_code = page_result.status_code - if not page_result.success or page_result.data is None: - break - merged_members.extend(page_result.data.get(RF_MEMBERS) or []) - next_link = page_result.data.get(RF_MEMBERS_NEXT_LINK) - pages_fetched += 1 - - merged_data[RF_MEMBERS] = merged_members - merged_data[RF_MEMBERS_COUNT] = len(merged_members) - merged_data.pop(RF_MEMBERS_NEXT_LINK, None) - - return RedfishGetResult( - path=first.path, - success=True, - data=merged_data, - status_code=last_status_code, - ) - - def copy(self) -> "RedfishConnection": - """Return a new connection with the same config and its own session (for concurrent use).""" - return RedfishConnection( - base_url=self.base_url, - username=self.username, - password=self.password, - timeout=self.timeout, - use_session_auth=self.use_session_auth, - verify_ssl=self.verify_ssl, - api_root=self.api_root, - ) - - def get_service_root(self) -> dict[str, Any]: - """GET service root (e.g. /redfish/v1/).""" - return self.get(RedfishPath(self.api_root)) - - def close(self) -> None: - """Release session and logout if session auth was used.""" - if self._session and self._session_uri: - try: - self._session.delete(self._session_uri, timeout=self.timeout) - except Exception: - pass - self._session = None - self._session_token = None - self._session_uri = None - - def __enter__(self) -> RedfishConnection: - self._ensure_session() - return self - - def __exit__(self, *args: Any) -> None: - self.close() +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import json +import socket +from typing import Any, Callable, ClassVar, Optional, Protocol, TypeVar, Union +from urllib.parse import urljoin, urlparse + +import requests +import urllib3 # type: ignore[import-untyped] +from pydantic import BaseModel +from requests.auth import HTTPBasicAuth + +from .redfish_constants import RF_MEMBERS, RF_MEMBERS_COUNT, RF_MEMBERS_NEXT_LINK +from .redfish_path import RedfishPath + +DEFAULT_REDFISH_API_ROOT = "redfish/v1" + +_T = TypeVar("_T") + + +class RedfishGetResult(BaseModel): + """Artifact for the result of a Redfish GET request. + Logged under the same filename as inband command artifacts (command_artifacts.json). + """ + + ARTIFACT_LOG_BASENAME: ClassVar[str] = "command_artifacts" + + path: str + success: bool + data: Optional[dict[str, Any]] = None + error: Optional[str] = None + status_code: Optional[int] = None + log_html: bool = False + + def to_html_entry(self) -> dict: + """Return a dict suitable for HTML command artifact rendering.""" + stdout = json.dumps(self.data, indent=2, sort_keys=True) if self.data is not None else "" + stderr = self.error or "" + exit_code = 0 if self.success else (self.status_code if self.status_code is not None else 1) + return { + "command": f"GET {self.path}", + "stdout": stdout, + "stderr": stderr, + "exit_code": exit_code, + } + + +class RedfishPostResult(BaseModel): + """Artifact for the result of a Redfish POST request.""" + + ARTIFACT_LOG_BASENAME: ClassVar[str] = "command_artifacts" + + path: str + success: bool + data: Optional[dict[str, Any]] = None + error: Optional[str] = None + status_code: Optional[int] = None + + def to_html_entry(self) -> dict: + """Return a dict suitable for HTML command artifact rendering.""" + stdout = json.dumps(self.data, indent=2, sort_keys=True) if self.data is not None else "" + stderr = self.error or "" + exit_code = 0 if self.success else (self.status_code if self.status_code is not None else 1) + return { + "command": f"POST {self.path}", + "stdout": stdout, + "stderr": stderr, + "exit_code": exit_code, + } + + +class RedfishHttpResponse(Protocol): + """Status, headers, and body from a Redfish GET or POST.""" + + status_code: int + headers: Any + + @property + def content(self) -> bytes: ... + + @property + def ok(self) -> bool: ... + + @property + def reason(self) -> str: ... + + @property + def text(self) -> str: ... + + def json(self) -> Any: ... + + +class RedfishConnectionError(Exception): + """Raised when a Redfish API request fails.""" + + def __init__(self, message: str, response: Optional[RedfishHttpResponse] = None): + super().__init__(message) + self.response = response + + +class RedfishConnection: + """Redfish REST client for GET requests.""" + + def __init__( + self, + base_url: str, + username: str, + password: Optional[str] = None, + timeout: float = 10.0, + use_session_auth: bool = True, + verify_ssl: bool = True, + api_root: Optional[str] = None, + ): + """Creates a RedfishConnection instance. + + Args: + base_url (str): The base url for the redfish service typically this is /redfish/v1 + username (str): The username for the redfish service. + password (Optional[str], optional): The password for the redfish service. Defaults to None. + timeout (float, optional): The timeout for redfish requests in seconds. Defaults to 10.0. + use_session_auth (bool, optional): Whether to use session-based authentication. Defaults to True. + verify_ssl (bool, optional): Whether to verify the SSL certificate. Defaults to True. + api_root (Optional[str], optional): The API root for the redfish services. Defaults to None. + """ + self.base_url: str = base_url.rstrip("/") + self.api_root: str = (api_root or DEFAULT_REDFISH_API_ROOT).strip("/") + self.username: str = username + self.password: str = password or "" + self.timeout: float = timeout + self.use_session_auth: bool = use_session_auth + self.verify_ssl: bool = verify_ssl + self._session: Optional[requests.Session] = None + self._session_token: Optional[str] = None + self._session_uri: Optional[str] = None # For logout DELETE + + def _host_label(self) -> str: + return urlparse(self.base_url).hostname or self.base_url + + @staticmethod + def _is_name_resolution_error(exc: BaseException) -> bool: + current: Optional[BaseException] = exc + seen: set[int] = set() + while current is not None and id(current) not in seen: + if isinstance(current, socket.gaierror): + return True + if type(current).__name__ == "NameResolutionError": + return True + seen.add(id(current)) + current = current.__cause__ or current.__context__ + message = str(exc) + return "Failed to resolve" in message or "No address associated with hostname" in message + + def _transport_error_message(self, exc: Exception) -> str: + host = self._host_label() + if isinstance(exc, socket.gaierror) or self._is_name_resolution_error(exc): + return f"Redfish hostname could not be resolved: {host}" + if isinstance(exc, requests.exceptions.Timeout): + return f"Redfish connection timed out: {host}" + if isinstance(exc, requests.exceptions.ConnectionError): + return f"Redfish connection failed: {host}" + return f"Redfish connection failed: {exc}" + + def _execute_request(self, request_fn: Callable[[], _T]) -> _T: + try: + return request_fn() + except requests.exceptions.RequestException as exc: + raise RedfishConnectionError(self._transport_error_message(exc)) from exc + except socket.gaierror as exc: + raise RedfishConnectionError(self._transport_error_message(exc)) from exc + + def _ensure_session(self) -> requests.Session: + if self._session is None: + if not self.verify_ssl: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + self._session = requests.Session() + self._session.verify = self.verify_ssl + if not self.verify_ssl: + self._session.trust_env = False + self._session.headers["Content-Type"] = "application/json" + self._session.headers["Accept"] = "application/json" + if self.use_session_auth and self.password: + self._login_session() + elif self.password: + self._session.auth = HTTPBasicAuth(self.username, self.password) + return self._session + + def _login_session(self) -> None: + """Create a Redfish session and set X-Auth-Token.""" + session = self._session + assert session is not None + sess_url = urljoin(self.base_url + "/", f"{self.api_root}/SessionService/Sessions") + payload = {"UserName": self.username, "Password": self.password} + resp = self._execute_request( + lambda: session.post( + sess_url, + json=payload, + timeout=self.timeout, + ) + ) + if not resp.ok: + raise RedfishConnectionError( + f"Session login failed: {resp.status_code} {resp.reason}", response=resp + ) + self._session_token = resp.headers.get("X-Auth-Token") + location = resp.headers.get("Location") + if location: + self._session_uri = ( + location + if location.startswith("http") + else urljoin(self.base_url + "/", location.lstrip("/")) + ) + if self._session_token: + session.headers["X-Auth-Token"] = self._session_token + else: + session.auth = HTTPBasicAuth(self.username, self.password) + + def _drop_session(self) -> None: + """Discard the HTTP session without logging out of Redfish. + + Args: + None + + Returns: + None + """ + if self._session is not None: + self._session.close() + self._session = None + self._session_token = None + self._session_uri = None + + def get(self, path: RedfishPath) -> dict[str, Any]: + """GET a Redfish path and return the JSON body. path must be a RedfishPath.""" + path_str = str(path) + resp = self.get_response(path_str) + if not resp.ok: + raise RedfishConnectionError( + f"GET {path_str} failed: {resp.status_code} {resp.reason}", + response=resp, + ) + try: + return resp.json() + except ValueError as exc: + raise RedfishConnectionError( + f"GET {path_str} returned a body that is not valid JSON", response=resp + ) from exc + + def get_response(self, path: Union[str, "RedfishPath"]) -> RedfishHttpResponse: + """GET a Redfish path and refresh the session once after HTTP 401. + + Args: + path: Redfish path or absolute URL. + + Returns: + The HTTP response, including a second 401 when reauthentication fails. + """ + path = str(path) + session = self._ensure_session() + url = path if path.startswith("http") else urljoin(self.base_url + "/", path.lstrip("/")) + response = self._execute_request(lambda: session.get(url, timeout=self.timeout)) + if response.status_code != 401 or not self.use_session_auth or not self.password: + return response + self._drop_session() + session = self._ensure_session() + return self._execute_request(lambda: session.get(url, timeout=self.timeout)) + + def post( + self, path: Union[str, "RedfishPath"], json: Optional[dict[str, Any]] = None + ) -> RedfishHttpResponse: + """POST to a Redfish path and return the raw Response. path may be a string or RedfishPath.""" + path = str(path) + session = self._ensure_session() + url = path if path.startswith("http") else urljoin(self.base_url + "/", path.lstrip("/")) + return self._execute_request( + lambda: session.post(url, json=json or {}, timeout=self.timeout) + ) + + def run_get(self, path: Union[str, RedfishPath]) -> RedfishGetResult: + """Run a Redfish GET request and return a result object. path may be a string or RedfishPath.""" + path_norm = str(path).strip() + if not path_norm.startswith("/"): + path_norm = "/" + path_norm + path_obj = RedfishPath(path_norm.strip("/")) if isinstance(path, str) else path + try: + data = self.get(path_obj) + return RedfishGetResult( + path=path_norm, + success=True, + data=data, + status_code=200, + ) + except RedfishConnectionError as e: + status = e.response.status_code if e.response is not None else None + return RedfishGetResult( + path=path_norm, + success=False, + error=str(e), + status_code=status, + ) + except Exception as e: + return RedfishGetResult( + path=path_norm, + success=False, + error=str(e), + status_code=None, + ) + + def run_get_paged( + self, + path: Union[str, RedfishPath], + max_pages: int = 200, + ) -> RedfishGetResult: + """Run a Redfish GET and transparently follow Members@odata.nextLink pagination. + + Each subsequent page's Members list is appended to the first page's Members list + so the caller receives a single merged response body. The Members@odata.nextLink key + and Members@odata.count are updated to reflect the merged result. If there is no + Members@odata.nextLink in the first response this behaves identically to run_get. + max_pages is a safety cap on the number of pages to fetch (default 200). + """ + first = self.run_get(path) + if not first.success or first.data is None: + return first + + # Short-circuit when there is nothing to page through. + if RF_MEMBERS_NEXT_LINK not in first.data: + return first + + merged_members: list = list(first.data.get(RF_MEMBERS) or []) + merged_data: dict = dict(first.data) + pages_fetched = 1 + next_link: Optional[str] = first.data.get(RF_MEMBERS_NEXT_LINK) + last_status_code = first.status_code + + while next_link and pages_fetched < max_pages: + page_result = self.run_get(next_link) + last_status_code = page_result.status_code + if not page_result.success or page_result.data is None: + break + merged_members.extend(page_result.data.get(RF_MEMBERS) or []) + next_link = page_result.data.get(RF_MEMBERS_NEXT_LINK) + pages_fetched += 1 + + merged_data[RF_MEMBERS] = merged_members + merged_data[RF_MEMBERS_COUNT] = len(merged_members) + merged_data.pop(RF_MEMBERS_NEXT_LINK, None) + + return RedfishGetResult( + path=first.path, + success=True, + data=merged_data, + status_code=last_status_code, + ) + + def copy(self) -> "RedfishConnection": + """Return a new connection with the same config and its own session (for concurrent use).""" + return RedfishConnection( + base_url=self.base_url, + username=self.username, + password=self.password, + timeout=self.timeout, + use_session_auth=self.use_session_auth, + verify_ssl=self.verify_ssl, + api_root=self.api_root, + ) + + def get_service_root(self) -> dict[str, Any]: + """GET service root (e.g. /redfish/v1/).""" + return self.get(RedfishPath(self.api_root)) + + def close(self) -> None: + """Release session and logout if session auth was used.""" + if self._session and self._session_uri: + try: + self._session.delete(self._session_uri, timeout=self.timeout) + except Exception: + pass + + if self._session: + self._session.close() + self._session = None + self._session_token = None + self._session_uri = None + + def __enter__(self) -> RedfishConnection: + self._ensure_session() + return self + + def __exit__(self, *args: Any) -> None: + self.close() diff --git a/nodescraper/connection/redfish/redfish_manager.py b/nodescraper/connection/redfish/redfish_manager.py index bc5b37d2..0b43dee7 100644 --- a/nodescraper/connection/redfish/redfish_manager.py +++ b/nodescraper/connection/redfish/redfish_manager.py @@ -1,139 +1,293 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from __future__ import annotations - -from logging import Logger -from typing import Optional, Union - -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.interfaces.connectionmanager import ConnectionManager -from nodescraper.interfaces.taskresulthook import TaskResultHook -from nodescraper.models import SystemInfo, TaskResult - -from .redfish_connection import RedfishConnection, RedfishConnectionError -from .redfish_params import RedfishConnectionParams - - -def _build_base_url(host: str, port: Optional[int], use_https: bool) -> str: - scheme = "https" if use_https else "http" - host_str = str(host) - if port is not None: - return f"{scheme}://{host_str}:{port}" - return f"{scheme}://{host_str}" - - -class RedfishConnectionManager(ConnectionManager[RedfishConnection, RedfishConnectionParams]): - """Connection manager for Redfish (BMC) API.""" - - def __init__( - self, - system_info: SystemInfo, - logger: Optional[Logger] = None, - max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, - parent: Optional[str] = None, - task_result_hooks: Optional[list[TaskResultHook]] = None, - connection_args: Optional[RedfishConnectionParams] = None, - **kwargs, - ): - super().__init__( - system_info, - logger, - max_event_priority_level, - parent, - task_result_hooks, - connection_args, - **kwargs, - ) - - def connect(self) -> TaskResult: - """Connect to the Redfish service and perform a simple GET to verify.""" - if not self.connection_args: - self._log_event( - category=EventCategory.RUNTIME, - description="No Redfish connection parameters provided", - priority=EventPriority.CRITICAL, - console_log=True, - ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - return self.result - - # Accept dict from JSON config; convert to RedfishConnectionParams - raw = self.connection_args - if isinstance(raw, dict): - params = RedfishConnectionParams.model_validate(raw) - elif isinstance(raw, RedfishConnectionParams): - params = raw - else: - self._log_event( - category=EventCategory.RUNTIME, - description="Redfish connection_args must be dict or RedfishConnectionParams", - priority=EventPriority.CRITICAL, - console_log=True, - ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - return self.result - - password = params.password.get_secret_value() if params.password else None - base_url = _build_base_url(str(params.host), params.port, params.use_https) - - try: - self.logger.info("Connecting to Redfish at %s", base_url) - self.connection = RedfishConnection( - base_url=base_url, - username=params.username, - password=password, - timeout=params.timeout_seconds, - use_session_auth=params.use_session_auth, - verify_ssl=params.verify_ssl, - api_root=params.api_root, - ) - self.connection._ensure_session() - self.connection.get_service_root() - except RedfishConnectionError as exc: - self._log_event( - category=EventCategory.RUNTIME, - description=str(exc), - priority=EventPriority.CRITICAL, - console_log=True, - ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - self.connection = None - except Exception as exc: - self._log_event( - category=EventCategory.RUNTIME, - description=f"Redfish connection failed: {exc}", - priority=EventPriority.CRITICAL, - console_log=True, - ) - self.result.status = ExecutionStatus.EXECUTION_FAILURE - self.connection = None - return self.result - - def disconnect(self) -> None: - """Disconnect and release the Redfish session.""" - if self.connection is not None: - self.connection.close() - super().disconnect() +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from logging import Logger +from typing import Any, ClassVar, Mapping, Optional, Union + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces.connectionmanager import ConnectionManager +from nodescraper.interfaces.taskresulthook import TaskResultHook +from nodescraper.models import SystemInfo, TaskResult + +from .redfish_connection import RedfishConnection, RedfishConnectionError +from .redfish_params import RedfishConnectionParams + + +def _build_base_url(host: str, port: Optional[int], use_https: bool) -> str: + scheme = "https" if use_https else "http" + host_str = str(host) + if port is not None: + return f"{scheme}://{host_str}:{port}" + return f"{scheme}://{host_str}" + + +class MultiTargetRedfishConnection: + """Sentinel placed on RedfishConnectionManager.connection in multi-target mode. + + Carrying this object (rather than leaving connection=None) allows the standard + DataPlugin.collect() guard to pass, while RedfishDataCollector's __init_subclass__ + wrapper intercepts collect_data and iterates the individual target connections. + """ + + def __init__( + self, + target_connections: Mapping[str, RedfishConnection], + max_workers: Optional[int] = None, + failed_targets: Optional[dict[str, str]] = None, + ) -> None: + self.target_connections = dict(target_connections) + self.multi_target_data: dict[str, Any] = {} + self.max_workers = max_workers + self.failed_targets = dict(failed_targets or {}) + + +def collected_multi_target_data(connection_manager: Any) -> dict[str, Any]: + """Return per-target data from an open connection or from the copy made at disconnect. + + Args: + connection_manager: Redfish connection manager for this plugin. + + Returns: + dict[str, Any]: Target key to collected data. Empty for single-target runs. + """ + conn = getattr(connection_manager, "connection", None) + if isinstance(conn, MultiTargetRedfishConnection) and conn.multi_target_data: + return conn.multi_target_data + return getattr(connection_manager, "_multi_target_data", None) or {} + + +class RedfishConnectionManager(ConnectionManager[RedfishConnection, RedfishConnectionParams]): + """Connection manager for Redfish (BMC) API.""" + + CONNECTION_ARGS_MODEL: ClassVar[type[RedfishConnectionParams]] = RedfishConnectionParams + + def __init__( + self, + system_info: SystemInfo, + logger: Optional[Logger] = None, + max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, + parent: Optional[str] = None, + task_result_hooks: Optional[list[TaskResultHook]] = None, + connection_args: Optional[RedfishConnectionParams] = None, + **kwargs, + ): + """Creates a RedfishConnectionManager Task instance. + + Args: + system_info (SystemInfo): System information for the connection manager task. + logger (Optional[logging.Logger], optional): _description_. Defaults to None. + max_event_priority_level (Union[EventPriority, str], optional): _description_. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): _description_. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook], None], optional): _description_. Defaults to None. + connection_args (Optional[Union[TConnectArg, dict]], optional): _description_. Defaults to None. + kwargs (dict, optional): Additional keyword arguments passed to the parent class. + + Raises: + ValueError: Will raise a ValueError when the connection_args cannot be mapped to the expected model. + """ + super().__init__( + system_info, + logger, + max_event_priority_level, + parent, + task_result_hooks, + connection_args, + **kwargs, + ) + + def connect(self) -> TaskResult: + """Connect to Redfish (single-target or multi-target).""" + if not self.connection_args: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to Redfish: no connection details were provided.", + "No Redfish connection parameters provided", + {"connection_args": None}, + ) + + raw = self.connection_args + if isinstance(raw, dict): + try: + params = RedfishConnectionParams.model_validate(raw) + except ValueError as exc: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to Redfish: the connection details are not valid.", + f"Invalid Redfish connection parameters: {exc}", + {"exception": str(exc)}, + ) + elif isinstance(raw, RedfishConnectionParams): + params = raw + else: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect to Redfish: the connection details are not valid.", + "Redfish connection_args must be dict or RedfishConnectionParams " + f"(got {type(raw).__name__})", + {"connection_args_type": type(raw).__name__}, + ) + + if params.is_multi_target: + return self._connect_multi(params) + return self._connect_single(params) + + def _connect_multi(self, params: RedfishConnectionParams) -> TaskResult: + """Connect each Redfish target. One failure does not drop the others. + + Args: + params: Parameters whose targets list is set. + + Returns: + TaskResult: Warning when some targets connect, execution failure when none do. + """ + self.target_connections: dict[str, RedfishConnection] = {} + failed_targets: dict[str, str] = {} + for target in params.targets or []: + key, conn, error = self._connect_target(target) + if conn is None: + failed_targets[key] = error or f"Redfish connection failed for target {key!r}" + continue + if key in self.target_connections: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Duplicate Redfish target key {key!r}; keeping the first connection", + priority=EventPriority.WARNING, + console_log=True, + ) + conn.close() + continue + self.target_connections[key] = conn + + lines = "\n".join(f"[{k}] {v}" for k, v in failed_targets.items()) + if not self.target_connections: + summary = "Could not connect to any of the Redfish targets." + self._fail_connect( + EventCategory.RUNTIME, + summary, + f"Redfish connection failed for every target ({len(failed_targets)} tried)\n{lines}", + {"failed_targets": dict(failed_targets)}, + ) + return self.result + + if failed_targets: + self.result.status = ExecutionStatus.WARNING + self.result.message = f"Some Redfish targets failed to connect\n{lines}" + self.connection = MultiTargetRedfishConnection( # type: ignore[assignment] + self.target_connections, + max_workers=params.max_workers, + failed_targets=failed_targets, + ) + return self.result + + def _connect_target( + self, target: RedfishConnectionParams + ) -> tuple[str, Optional[RedfishConnection], Optional[str]]: + """Connect one target. + + Args: + target: Connection parameters for a single BMC. + + Returns: + tuple[str, Optional[RedfishConnection], Optional[str]]: Target key, connection, and error text. + """ + key = target.target_key or str(target.host) + password = target.password.get_secret_value() if target.password else None + base_url = _build_base_url(str(target.host), target.port, target.use_https) + self.logger.info("Connecting to Redfish at %s (target=%r)", base_url, key) + conn: Optional[RedfishConnection] = None + try: + conn = RedfishConnection( + base_url=base_url, + username=target.username or "", + password=password, + timeout=target.timeout_seconds, + use_session_auth=target.use_session_auth, + verify_ssl=target.verify_ssl, + api_root=target.api_root, + ) + conn._ensure_session() + conn.get_service_root() + except Exception as exc: # noqa: BLE001 - one bad target must not drop the others + if conn is not None: + conn.close() + description = f"Redfish connection failed for target {key!r}: {exc}" + self._log_event( + category=EventCategory.RUNTIME, + description=description, + priority=EventPriority.WARNING, + console_log=True, + ) + return key, None, description + return key, conn, None + + def _connect_single(self, params: RedfishConnectionParams) -> TaskResult: + """Connect in single-target mode.""" + password = params.password.get_secret_value() if params.password else None + base_url = _build_base_url(str(params.host), params.port, params.use_https) + self.logger.info("Connecting to Redfish at %s", base_url) + conn = RedfishConnection( + base_url=base_url, + username=params.username or "", + password=password, + timeout=params.timeout_seconds, + use_session_auth=params.use_session_auth, + verify_ssl=params.verify_ssl, + api_root=params.api_root, + ) + try: + conn._ensure_session() + conn.get_service_root() + except RedfishConnectionError as exc: + conn.close() + return self._fail_connect( + EventCategory.RUNTIME, + "Could not connect to Redfish.", + f"Redfish connection to {base_url} failed: {exc}", + { + "base_url": base_url, + "host": str(params.host), + "port": params.port, + "use_https": params.use_https, + "verify_ssl": params.verify_ssl, + "timeout_seconds": params.timeout_seconds, + "exception": str(exc), + }, + ) + + self.connection = conn + return self.result + + def disconnect(self) -> None: + """Disconnect Redfish sessions and keep collected multi-target data on the manager.""" + if isinstance(self.connection, MultiTargetRedfishConnection): + self._multi_target_data: dict[str, Any] = dict(self.connection.multi_target_data) + for conn in self.connection.target_connections.values(): + conn.close() + self.target_connections = {} + elif self.connection is not None: + self.connection.close() + super().disconnect() diff --git a/nodescraper/connection/redfish/redfish_oem_diag.py b/nodescraper/connection/redfish/redfish_oem_diag.py index affabf6e..67561170 100644 --- a/nodescraper/connection/redfish/redfish_oem_diag.py +++ b/nodescraper/connection/redfish/redfish_oem_diag.py @@ -1,379 +1,640 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from __future__ import annotations - -import json -import logging -import re -import time -from pathlib import Path -from typing import Any, Optional - -from requests import Response -from requests.status_codes import codes - -from nodescraper.enums import TaskState - -from .redfish_connection import RedfishConnection, RedfishConnectionError -from .redfish_constants import RF_ODATA_ID -from .redfish_path import RedfishPath - -_module_logger = logging.getLogger(__name__) - -_LOG_RESPONSE_BODY_LIMIT = 1500 - - -def _log_collect_diag_response( - log: logging.Logger, - status: int, - body: Any, - raw_text: str = "", -) -> None: - """Log CollectDiagnosticData response at INFO when no Location/TaskMonitor found.""" - if isinstance(body, dict): - snippet = json.dumps(body, indent=2) - else: - snippet = raw_text or str(body) - if len(snippet) > _LOG_RESPONSE_BODY_LIMIT: - snippet = snippet[:_LOG_RESPONSE_BODY_LIMIT] + "... (truncated)" - log.info( - "CollectDiagnosticData response (no Location/TaskMonitor): status=%s body=%s", - status, - snippet, - ) - - -# @Redfish.AllowableValues: Redfish annotation for the list of allowable values for a string -REDFISH_ANNOTATION_ALLOWABLE_VALUES = "Redfish.AllowableValues" - -# OEMDiagnosticDataType: LogService CollectDiagnosticData action parameter -OEM_DIAGNOSTIC_DATA_TYPE_PARAM = "OEMDiagnosticDataType" - -RF_ANNOTATION_ALLOWABLE = f"{OEM_DIAGNOSTIC_DATA_TYPE_PARAM}@{REDFISH_ANNOTATION_ALLOWABLE_VALUES}" - -# Default max wait for BMC task (seconds) -DEFAULT_TASK_TIMEOUT_S = 1800 - - -def get_oem_diagnostic_allowable_values( - conn: RedfishConnection, - log_service_path: str, -) -> Optional[list[str]]: - """GET the LogService and return OEMDiagnosticDataType@Redfish.AllowableValues if present. - - Args: - conn: Redfish connection (session established). - log_service_path: Path to the LogService (e.g. redfish/v1/Systems/UBB/LogServices/DiagLogs). - - Returns: - List of allowable type strings, or None if not found / GET failed. - """ - path = log_service_path.strip().strip("/") - try: - data = conn.get(RedfishPath(path)) - except RedfishConnectionError: - return None - if not isinstance(data, dict): - return None - actions = data.get("Actions") or {} - collect_action = actions.get("LogService.CollectDiagnosticData") or actions.get( - "#LogService.CollectDiagnosticData" - ) - if isinstance(collect_action, dict): - allow = collect_action.get(RF_ANNOTATION_ALLOWABLE) - if isinstance(allow, list) and all(isinstance(x, str) for x in allow): - return list(allow) - return None - - -def _resolve_path(conn: RedfishConnection, path: str) -> str: - """Return full URL for a path (relative to base_url).""" - if path.startswith("http"): - return path - path = path.lstrip("/") - base = conn.base_url.rstrip("/") - return f"{base}/{path}" - - -def _get_path_from_connection(conn: RedfishConnection, path: str) -> str: - """Return path relative to BMC (no host). For use with conn.get_response(path).""" - if path.startswith("http"): - # Strip base URL to get path under /redfish/v1/... - base = conn.base_url.rstrip("/") - if path.startswith(base + "/"): - return path[len(base) :].lstrip("/") - return path - return path.lstrip("/") - - -def _get_task_monitor_uri(body: dict, conn: RedfishConnection) -> Optional[str]: - """Extract task monitor URI from a Task-like body (DSP0266 or OEM variants). - - TaskMonitor may be a string URI or an object with @odata.id (e.g. TaskService/TaskMonitors/378). - """ - - def _resolve_uri(uri: str) -> str: - if not uri.startswith("http"): - uri = _resolve_path(conn, uri.strip().lstrip("/")) - return uri - - for key in ("TaskMonitor", "Monitor", "TaskMonitorUri"): - val = body.get(key) - if isinstance(val, str) and val.strip(): - return _resolve_uri(val) - if isinstance(val, dict): - odata_id = val.get(RF_ODATA_ID) - if isinstance(odata_id, str) and odata_id.strip(): - return _resolve_uri(odata_id) - oem = body.get("Oem") - if isinstance(oem, dict): - for vendor_dict in oem.values(): - if isinstance(vendor_dict, dict): - for k in ("TaskMonitor", "Monitor", "TaskMonitorUri"): - val = vendor_dict.get(k) - if isinstance(val, str) and val.strip(): - return _resolve_uri(val) - if isinstance(val, dict): - odata_id = val.get(RF_ODATA_ID) - if isinstance(odata_id, str) and odata_id.strip(): - return _resolve_uri(odata_id) - odata_id = body.get(RF_ODATA_ID) - if isinstance(odata_id, str) and odata_id.strip(): - base_uri = _resolve_uri(odata_id) - return f"{base_uri.rstrip('/')}/Monitor" - return None - - -# Workaround for LogEntry URL: some BMCs 404 when URL includes port -def _strip_port_from_url(url: str) -> Optional[str]: - """Return URL with port removed from authority (e.g. host:443 -> host).""" - if re.search(r"://[^/]+:\d+", url): - return re.sub(r"(://[^:/]+):\d+", r"\1", url, count=1) - return None - - -def _download_log_and_save( - conn: RedfishConnection, - log_entry_json: dict[str, Any], - oem_diagnostic_type: str, - output_dir: Optional[Path], - log: logging.Logger, -) -> Optional[bytes]: - # Download binary log if AdditionalDataURI present - log_bytes: Optional[bytes] = None - data_uri = log_entry_json.get("AdditionalDataURI") - if data_uri: - data_path = _get_path_from_connection(conn, data_uri) - data_resp = conn.get_response(data_path) - if data_resp.status_code == codes.ok: - log_bytes = data_resp.content - - if output_dir: - output_dir = Path(output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - if log_bytes is not None: - archive_path = output_dir / f"{oem_diagnostic_type}.tar.xz" - archive_path.write_bytes(log_bytes) - log.info("Log written to disk: %s -> %s", oem_diagnostic_type, archive_path.name) - metadata_file = output_dir / f"{oem_diagnostic_type}_log_entry.json" - try: - metadata_file.write_text(json.dumps(log_entry_json, indent=2), encoding="utf-8") - log.info( - "Log metadata written to disk: %s -> %s", oem_diagnostic_type, metadata_file.name - ) - except Exception as e: - log.exception("Failed to write log metadata to %s: %s", metadata_file, e) - - return log_bytes - - -def collect_oem_diagnostic_data( - conn: RedfishConnection, - log_service_path: str, - oem_diagnostic_type: Optional[str] = None, - task_timeout_s: int = DEFAULT_TASK_TIMEOUT_S, - output_dir: Optional[Path] = None, - validate_type: bool = False, - allowed_types: Optional[list[str]] = None, - logger: Optional[logging.Logger] = None, -) -> tuple[Optional[bytes], Optional[dict[str, Any]], Optional[str]]: - """ - Initiate OEM diagnostic collection, poll until done, download log and metadata. - - Args: - conn: Redfish connection (session already established). - log_service_path: Path to LogService under Systems, e.g. - "redfish/v1/Systems/UBB/LogServices/DiagLogs" (no leading slash). - oem_diagnostic_type: OEM type for DiagnosticDataType OEM (e.g. "JournalControl", "AllLogs"). Required. - task_timeout_s: Max seconds to wait for BMC task - output_dir: If set, save log archive and LogEntry JSON here. - validate_type: If True, require oem_diagnostic_type to be in allowed_types. - allowed_types: Allowable OEM diagnostic types for validation when validate_type is True. - logger: Logger - - Returns: - (log_bytes, log_entry_metadata_dict, error_message). - On success: (bytes, dict, None). On failure: (None, None, error_str). - """ - log = logger if logger is not None else _module_logger - if not oem_diagnostic_type or not oem_diagnostic_type.strip(): - return None, None, "oem_diagnostic_type is required" - if validate_type and allowed_types and oem_diagnostic_type not in allowed_types: - return ( - None, - None, - f"oem_diagnostic_type {oem_diagnostic_type!r} not in allowed types", - ) - path_prefix = log_service_path.rstrip("/") - action_path = f"{path_prefix}/Actions/LogService.CollectDiagnosticData" - payload = {"DiagnosticDataType": "OEM", "OEMDiagnosticDataType": oem_diagnostic_type} - - try: - resp: Response = conn.post(action_path, json=payload) - except RedfishConnectionError as e: - return None, None, str(e) - - if resp.status_code not in (codes.ok, codes.accepted): - return ( - None, - None, - f"Unexpected status {resp.status_code} for CollectDiagnosticData: {resp.text}", - ) - - # DSP0266 12.2: 202 shall include Location (task monitor URI), optionally Retry-After - location_header = resp.headers.get("Location") or resp.headers.get("Content-Location") - if location_header and not location_header.startswith("http"): - location_header = _resolve_path(conn, location_header) - sleep_s = int(resp.headers.get("Retry-After", 1) or 1) - try: - oem_response = resp.json() - except Exception: - oem_response = {} - - # 200 OK with TaskState=Completed: synchronous completion, body is the Task - task_json: Optional[dict[str, Any]] = None - if resp.status_code == codes.ok and isinstance(oem_response, dict): - if oem_response.get("TaskState") == TaskState.completed.value: - headers_list = oem_response.get("Payload", {}).get("HttpHeaders", []) or [] - if any(isinstance(h, str) and "Location:" in h for h in headers_list): - task_json = oem_response - - # When TaskMonitor is implemented - task_monitor: Optional[str] = None - task_path: Optional[str] = None - if task_json is None: - task_monitor = location_header or _get_task_monitor_uri(oem_response, conn) - if oem_response.get(RF_ODATA_ID): - task_path = _get_path_from_connection(conn, oem_response[RF_ODATA_ID]) - if not task_monitor and task_path: - task_resp = conn.get_response(task_path) - if task_resp.status_code == codes.ok: - fetched = task_resp.json() - task_monitor = _get_task_monitor_uri(fetched, conn) - if not task_monitor: - _log_collect_diag_response( - log, resp.status_code, oem_response, getattr(resp, "text", "") or "" - ) - return None, None, "No TaskMonitor in response and no Location header" - - if task_json is None: - assert task_monitor is not None - # Poll task monitor until no longer 202/404 (e.g. GET /redfish/v1/TaskService/TaskMonitors/378) - start = time.time() - poll_resp = None - while True: - if time.time() - start > task_timeout_s: - return None, None, f"Task did not complete within {task_timeout_s}s" - time.sleep(sleep_s) - monitor_path = _get_path_from_connection(conn, task_monitor) - poll_resp = conn.get_response(monitor_path) - if poll_resp.status_code not in (codes.accepted, codes.not_found): - break - - # TaskMonitor response body has @odata.id pointing to the Task (e.g. /redfish/v1/TaskService/Tasks/5) - try: - monitor_body = poll_resp.json() if poll_resp else {} - except Exception: - monitor_body = {} - task_uri_from_monitor = ( - monitor_body.get(RF_ODATA_ID) if isinstance(monitor_body, dict) else None - ) - if isinstance(task_uri_from_monitor, str) and task_uri_from_monitor.strip(): - task_path = _get_path_from_connection(conn, task_uri_from_monitor.strip()) - elif not task_path: - task_path = _get_path_from_connection(conn, task_monitor.rstrip("/").rsplit("/", 1)[0]) - task_resp = conn.get_response(task_path) - if task_resp.status_code != codes.ok: - return None, None, f"Task GET failed: {task_resp.status_code}" - task_json = task_resp.json() - if task_json.get("TaskState") != TaskState.completed.value: - return None, None, f"Task did not complete: TaskState={task_json.get('TaskState')}" - - # LogEntry location from Payload.HttpHeaders - headers_list = task_json.get("Payload", {}).get("HttpHeaders", []) or [] - location = None - for header in headers_list: - if isinstance(header, str) and "Location:" in header: - location = header.split("Location:", 1)[-1].strip() - break - if not location: - return None, None, "Location header missing in task Payload.HttpHeaders" - if location.startswith("http"): - log_entry_path = location - else: - log_entry_path = location.lstrip("/") - - # GET LogEntry (some BMCs 404 when URL includes explicit port; try without port first) - log_entry_alt = _strip_port_from_url(log_entry_path) - if log_entry_alt is None and not log_entry_path.startswith("http"): - log_entry_alt = _strip_port_from_url( - conn.base_url.rstrip("/") + "/" + log_entry_path.lstrip("/") - ) - paths_to_try = [log_entry_alt, log_entry_path] if log_entry_alt else [log_entry_path] - log_entry_json = None - first_status: Optional[int] = codes.not_found - first_error = "" - for try_path in paths_to_try: - if try_path is None: - continue - try: - log_entry_resp = conn.get_response(try_path) - if log_entry_resp.status_code == codes.ok: - log_entry_json = log_entry_resp.json() - break - if try_path == paths_to_try[0]: - first_status = log_entry_resp.status_code - except Exception as e: - if try_path == paths_to_try[0]: - first_status = None - first_error = str(e) - continue - if log_entry_json is None: - err = first_error if first_status is None else f"status {first_status}" - return None, None, f"LogEntry GET failed: {err} (GET {log_entry_path})" - - log_bytes = _download_log_and_save(conn, log_entry_json, oem_diagnostic_type, output_dir, log) - return log_bytes, log_entry_json, None +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import json +import logging +import re +import time +from pathlib import Path +from typing import Any, Optional + +from requests.status_codes import codes + +from nodescraper.enums import TaskState + +from .redfish_connection import ( + RedfishConnection, + RedfishConnectionError, + RedfishHttpResponse, +) +from .redfish_constants import RF_ODATA_ID +from .redfish_path import RedfishPath + +_module_logger = logging.getLogger(__name__) + +_LOG_RESPONSE_BODY_LIMIT = 1500 + + +def _log_collect_diag_response( + log: logging.Logger, + status: int, + body: Any, + raw_text: str = "", +) -> None: + """Log CollectDiagnosticData response at INFO when no Location/TaskMonitor found.""" + if isinstance(body, dict): + snippet = json.dumps(body, indent=2) + else: + snippet = raw_text or str(body) + if len(snippet) > _LOG_RESPONSE_BODY_LIMIT: + snippet = snippet[:_LOG_RESPONSE_BODY_LIMIT] + "... (truncated)" + log.info( + "CollectDiagnosticData response (no Location/TaskMonitor): status=%s body=%s", + status, + snippet, + ) + + +# @Redfish.AllowableValues: Redfish annotation for the list of allowable values for a string +REDFISH_ANNOTATION_ALLOWABLE_VALUES = "Redfish.AllowableValues" + +# OEMDiagnosticDataType: LogService CollectDiagnosticData action parameter +OEM_DIAGNOSTIC_DATA_TYPE_PARAM = "OEMDiagnosticDataType" + +RF_ANNOTATION_ALLOWABLE = f"{OEM_DIAGNOSTIC_DATA_TYPE_PARAM}@{REDFISH_ANNOTATION_ALLOWABLE_VALUES}" + +# Default max wait for BMC task (seconds) +DEFAULT_TASK_TIMEOUT_S = 1800 + + +def get_oem_diagnostic_allowable_values( + conn: RedfishConnection, + log_service_path: str, +) -> Optional[list[str]]: + """GET the LogService and return OEMDiagnosticDataType@Redfish.AllowableValues if present. + + Args: + conn: Redfish connection (session established). + log_service_path: Path to the LogService (e.g. redfish/v1/Systems/UBB/LogServices/DiagLogs). + + Returns: + List of allowable type strings, or None if not found / GET failed. + """ + path = log_service_path.strip().strip("/") + try: + data = conn.get(RedfishPath(path)) + except RedfishConnectionError: + return None + if not isinstance(data, dict): + return None + actions = data.get("Actions") or {} + collect_action = actions.get("LogService.CollectDiagnosticData") or actions.get( + "#LogService.CollectDiagnosticData" + ) + if isinstance(collect_action, dict): + allow = collect_action.get(RF_ANNOTATION_ALLOWABLE) + if isinstance(allow, list) and all(isinstance(x, str) for x in allow): + return list(allow) + return None + + +def _resolve_path(conn: RedfishConnection, path: str) -> str: + """Return full URL for a path (relative to base_url).""" + if path.startswith("http"): + return path + path = path.lstrip("/") + base = conn.base_url.rstrip("/") + return f"{base}/{path}" + + +def _get_path_from_connection(conn: RedfishConnection, path: str) -> str: + """Return path relative to BMC (no host). For use with conn.get_response(path).""" + if path.startswith("http"): + # Strip base URL to get path under /redfish/v1/... + base = conn.base_url.rstrip("/") + if path.startswith(base + "/"): + return path[len(base) :].lstrip("/") + return path + return path.lstrip("/") + + +def _get_task_monitor_uri(body: dict, conn: RedfishConnection) -> Optional[str]: + """Extract task monitor URI from a Task-like body (DSP0266 or OEM variants). + + TaskMonitor may be a string URI or an object with @odata.id (e.g. TaskService/TaskMonitors/378). + """ + + def _resolve_uri(uri: str) -> str: + if not uri.startswith("http"): + uri = _resolve_path(conn, uri.strip().lstrip("/")) + return uri + + for key in ("TaskMonitor", "Monitor", "TaskMonitorUri"): + val = body.get(key) + if isinstance(val, str) and val.strip(): + return _resolve_uri(val) + if isinstance(val, dict): + odata_id = val.get(RF_ODATA_ID) + if isinstance(odata_id, str) and odata_id.strip(): + return _resolve_uri(odata_id) + oem = body.get("Oem") + if isinstance(oem, dict): + for vendor_dict in oem.values(): + if isinstance(vendor_dict, dict): + for k in ("TaskMonitor", "Monitor", "TaskMonitorUri"): + val = vendor_dict.get(k) + if isinstance(val, str) and val.strip(): + return _resolve_uri(val) + if isinstance(val, dict): + odata_id = val.get(RF_ODATA_ID) + if isinstance(odata_id, str) and odata_id.strip(): + return _resolve_uri(odata_id) + odata_id = body.get(RF_ODATA_ID) + if isinstance(odata_id, str) and odata_id.strip(): + base_uri = _resolve_uri(odata_id) + return f"{base_uri.rstrip('/')}/Monitor" + return None + + +def _task_resource_path(path: str) -> Optional[str]: + """Return a TaskService/Tasks path when path is a Task member, else None. + + Args: + path: Absolute or relative Redfish URI. + + Returns: + Normalized Task path, or None. + """ + stripped = path.strip().lstrip("/") + if "TaskService/Tasks/" in stripped and "TaskMonitors" not in stripped: + return stripped + return None + + +def _poll_task_resource( + conn: RedfishConnection, + task_path: str, + timeout_s: int, + sleep_s: int, +) -> tuple[Optional[dict[str, Any]], Optional[str]]: + """GET a Task resource until TaskState is Completed or a terminal failure. + + Args: + conn: Redfish connection. + task_path: Path to a TaskService/Tasks member. + timeout_s: Max seconds to wait. + sleep_s: Seconds between GETs. + + Returns: + (task JSON, None) on success, or (None, error). + """ + start = time.time() + interval = max(int(sleep_s), 1) + while True: + if time.time() - start > timeout_s: + return None, f"Task did not complete within {timeout_s}s" + try: + poll_resp = conn.get_response(task_path) + except RedfishConnectionError as exc: + return None, str(exc) + if poll_resp.status_code == codes.ok: + try: + body = poll_resp.json() + except Exception: + body = {} + if isinstance(body, dict): + state = body.get("TaskState") + if state == TaskState.completed.value: + return body, None + if state in ( + TaskState.exception.value, + TaskState.cancelled.value, + TaskState.killed.value, + ): + return None, f"Task did not complete: TaskState={state}" + elif poll_resp.status_code != codes.accepted: + return None, f"Task GET failed: {poll_resp.status_code}" + time.sleep(interval) + + +# Workaround for LogEntry URL: some BMCs 404 when URL includes port +def _strip_port_from_url(url: str) -> Optional[str]: + """Return URL with port removed from authority (e.g. host:443 -> host).""" + if re.search(r"://[^/]+:\d+", url): + return re.sub(r"(://[^:/]+):\d+", r"\1", url, count=1) + return None + + +_LOG_ENTRY_GET_ATTEMPTS = 4 +_LOG_ENTRY_GET_RETRY_SLEEP_S = 2 + + +def _entries_collection_path(log_entry_path: str) -> Optional[str]: + """Return the Entries collection path for a LogEntry URI. + + Args: + log_entry_path (str): Task Location path or URL for a LogEntry. + + Returns: + Optional[str]: collection path such as redfish/v1/.../Dump/Entries, or None. + """ + path = log_entry_path.split("?", 1)[0] + if "://" in path: + path = path.split("://", 1)[1] + path = path.split("/", 1)[1] if "/" in path else path + path = path.lstrip("/") + if "/Entries/" not in f"/{path}": + return None + return path[: path.rfind("/Entries/") + len("/Entries")].lstrip("/") + + +def _entry_id_from_path(log_entry_path: str) -> Optional[str]: + """Return the LogEntry Id from a Location path. + + Args: + log_entry_path (str): Task Location path or URL. + + Returns: + Optional[str]: entry Id, or None. + """ + tail = log_entry_path.split("?", 1)[0].rstrip("/").rsplit("/", 1) + if len(tail) != 2 or tail[1] in ("", "Entries"): + return None + return tail[1] + + +def _pick_member_href(body: dict[str, Any], wanted_id: Optional[str]) -> Optional[str]: + """Pick a Members @odata.id, preferring wanted_id then the last member. + + Args: + body (dict[str, Any]): Entries collection JSON. + wanted_id (Optional[str]): LogEntry Id from the task Location. + + Returns: + Optional[str]: member href, or None. + """ + members = body.get("Members") or [] + hrefs: list[str] = [] + for member in members: + if not isinstance(member, dict): + continue + href = member.get(RF_ODATA_ID) + if isinstance(href, str) and href.strip(): + hrefs.append(href.strip()) + extra = member.get("Id") + if wanted_id and extra is not None and str(extra) == wanted_id: + return href.strip() + if wanted_id: + suffix = "/" + wanted_id + for href in hrefs: + if href.rstrip("/").endswith(suffix): + return href + return None + return hrefs[-1] if hrefs else None + + +def _get_json_if_ok(conn: RedfishConnection, path: str) -> tuple[Optional[dict[str, Any]], int]: + """GET path and return JSON when the status is 200. + + Args: + conn (RedfishConnection): Redfish connection. + path (str): relative Redfish path. + + Returns: + tuple[Optional[dict[str, Any]], int]: parsed object (or None) and status. + """ + resp = conn.get_response(path) + if resp.status_code != codes.ok: + return None, resp.status_code + try: + body = resp.json() + except Exception: + return None, resp.status_code + if isinstance(body, dict): + return body, resp.status_code + return None, resp.status_code + + +def _fetch_log_entry_json( + conn: RedfishConnection, + log_entry_path: str, + log: logging.Logger, +) -> tuple[Optional[dict[str, Any]], Optional[int], Optional[str]]: + """GET a LogEntry, retrying 404s and falling back to the Entries collection. + + Args: + conn (RedfishConnection): Redfish connection. + log_entry_path (str): path from the completed task Location header. + log (logging.Logger): logger. + + Returns: + tuple[Optional[dict[str, Any]], Optional[int], Optional[str]]: entry JSON, + last HTTP status, and last exception string. + """ + paths_to_try: list[str] = [] + log_entry_alt = _strip_port_from_url(log_entry_path) + if log_entry_alt is None and not log_entry_path.startswith("http"): + log_entry_alt = _strip_port_from_url( + conn.base_url.rstrip("/") + "/" + log_entry_path.lstrip("/") + ) + if log_entry_alt: + paths_to_try.append(_get_path_from_connection(conn, log_entry_alt)) + rel_path = _get_path_from_connection(conn, log_entry_path) + if rel_path not in paths_to_try: + paths_to_try.append(rel_path) + + last_status: Optional[int] = codes.not_found + last_error = "" + for attempt in range(_LOG_ENTRY_GET_ATTEMPTS): + for try_path in paths_to_try: + if not try_path: + continue + try: + body, last_status = _get_json_if_ok(conn, try_path) + if body is not None: + return body, codes.ok, None + except Exception as e: + last_status = None + last_error = str(e) + if attempt < _LOG_ENTRY_GET_ATTEMPTS - 1: + time.sleep(_LOG_ENTRY_GET_RETRY_SLEEP_S) + + coll_path = _entries_collection_path(rel_path) + wanted_id = _entry_id_from_path(rel_path) + if coll_path: + log.info("LogEntry GET 404; listing %s for Id %s", coll_path, wanted_id) + try: + coll, coll_status = _get_json_if_ok(conn, coll_path) + except Exception as e: + last_error = str(e) + coll = None + coll_status = None + if coll is not None: + href = _pick_member_href(coll, wanted_id) + if href: + member_path = _get_path_from_connection(conn, href) + try: + body, member_status = _get_json_if_ok(conn, member_path) + if body is not None: + return body, codes.ok, None + extra = None + for member in coll.get("Members") or []: + if isinstance(member, dict) and member.get(RF_ODATA_ID) == href: + extra = member + break + if extra and extra.get("AdditionalDataURI"): + return extra, codes.ok, None + last_error = f"member GET status {member_status} for {member_path}" + except Exception as e: + last_error = str(e) + else: + last_error = f"Id {wanted_id} not in {coll_path}" + elif coll_status is not None: + last_error = f"Entries collection GET status {coll_status}" + + return None, last_status, last_error or None + + +def _download_log_and_save( + conn: RedfishConnection, + log_entry_json: dict[str, Any], + oem_diagnostic_type: str, + output_dir: Optional[Path], + log: logging.Logger, +) -> Optional[bytes]: + # Download binary log if AdditionalDataURI present + log_bytes: Optional[bytes] = None + data_uri = log_entry_json.get("AdditionalDataURI") + if data_uri: + data_path = _get_path_from_connection(conn, data_uri) + data_resp = conn.get_response(data_path) + if data_resp.status_code == codes.ok: + log_bytes = data_resp.content + + if output_dir: + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + if log_bytes is not None: + archive_path = output_dir / f"{oem_diagnostic_type}.tar.xz" + archive_path.write_bytes(log_bytes) + log.info("Log written to disk: %s -> %s", oem_diagnostic_type, archive_path.name) + metadata_file = output_dir / f"{oem_diagnostic_type}_log_entry.json" + try: + metadata_file.write_text(json.dumps(log_entry_json, indent=2), encoding="utf-8") + log.info( + "Log metadata written to disk: %s -> %s", + oem_diagnostic_type, + metadata_file.name, + ) + except Exception as e: + log.exception("Failed to write log metadata to %s: %s", metadata_file, e) + + return log_bytes + + +def collect_oem_diagnostic_data( + conn: RedfishConnection, + log_service_path: str, + oem_diagnostic_type: Optional[str] = None, + task_timeout_s: int = DEFAULT_TASK_TIMEOUT_S, + output_dir: Optional[Path] = None, + validate_type: bool = False, + allowed_types: Optional[list[str]] = None, + logger: Optional[logging.Logger] = None, + diagnostic_data_type: str = "OEM", +) -> tuple[Optional[bytes], Optional[dict[str, Any]], Optional[str]]: + """ + Initiate CollectDiagnosticData, poll until done, download log and metadata. + + Args: + conn: Redfish connection (session already established). + log_service_path: Path to LogService under Systems, e.g. + "redfish/v1/Systems/UBB/LogServices/DiagLogs" (no leading slash). + oem_diagnostic_type: OEM type when diagnostic_data_type is OEM (e.g. JournalControl, AllLogs). + task_timeout_s: Max seconds to wait for BMC task + output_dir: If set, save log archive and LogEntry JSON here. + validate_type: If True, require oem_diagnostic_type to be in allowed_types. + allowed_types: Allowable OEM diagnostic types for validation when validate_type is True. + logger: Logger + diagnostic_data_type: DMTF DiagnosticDataType (OEM, Manager, and similar). + + Returns: + (log_bytes, log_entry_metadata_dict, error_message). + On success: (bytes, dict, None). On failure: (None, None, error_str). + """ + SLEEP_S_DEFAULT = 1 + log = logger if logger is not None else _module_logger + diag_type = (diagnostic_data_type or "OEM").strip() or "OEM" + oem_type = (oem_diagnostic_type or "").strip() + if diag_type == "OEM" and not oem_type: + return None, None, "oem_diagnostic_type is required" + if validate_type and allowed_types and oem_type and oem_type not in allowed_types: + return ( + None, + None, + f"oem_diagnostic_type {oem_type!r} not in allowed types", + ) + path_prefix = log_service_path.rstrip("/") + action_path = f"{path_prefix}/Actions/LogService.CollectDiagnosticData" + payload: dict[str, Any] = {"DiagnosticDataType": diag_type} + if oem_type: + payload["OEMDiagnosticDataType"] = oem_type + + try: + resp: RedfishHttpResponse = conn.post(action_path, json=payload) + except RedfishConnectionError as e: + return None, None, str(e) + + if resp.status_code not in (codes.ok, codes.accepted): + return ( + None, + None, + f"Unexpected status {resp.status_code} for CollectDiagnosticData: {resp.text}", + ) + + # DSP0266 12.2: 202 shall include Location (task monitor URI), optionally Retry-After + location_header = resp.headers.get("Location") or resp.headers.get("Content-Location") + if location_header and not location_header.startswith("http"): + location_header = _resolve_path(conn, location_header) + try: + sleep_s = int(resp.headers.get("Retry-After", SLEEP_S_DEFAULT) or SLEEP_S_DEFAULT) + except ValueError: + sleep_s = SLEEP_S_DEFAULT + try: + oem_response = resp.json() + except Exception: + oem_response = {} + + # 200 OK with TaskState=Completed: synchronous completion, body is the Task + task_json: Optional[dict[str, Any]] = None + if resp.status_code == codes.ok and isinstance(oem_response, dict): + if oem_response.get("TaskState") == TaskState.completed.value: + headers_list = oem_response.get("Payload", {}).get("HttpHeaders", []) or [] + if any(isinstance(h, str) and "Location:" in h for h in headers_list): + task_json = oem_response + + task_monitor: Optional[str] = None + task_path: Optional[str] = None + if task_json is None: + if isinstance(oem_response, dict) and oem_response.get(RF_ODATA_ID): + task_path = _task_resource_path( + _get_path_from_connection(conn, oem_response[RF_ODATA_ID]) + ) + if location_header: + loc_path = _get_path_from_connection(conn, location_header) + loc_task = _task_resource_path(loc_path) + if loc_task: + task_path = loc_task + else: + task_monitor = location_header + if not task_monitor and not task_path and isinstance(oem_response, dict): + task_monitor = _get_task_monitor_uri(oem_response, conn) + if task_path: + task_json, poll_err = _poll_task_resource(conn, task_path, task_timeout_s, sleep_s) + if poll_err: + return None, None, poll_err + elif task_monitor: + start = time.time() + poll_resp = None + while True: + if time.time() - start > task_timeout_s: + return None, None, f"Task did not complete within {task_timeout_s}s" + monitor_path = _get_path_from_connection(conn, task_monitor) + try: + poll_resp = conn.get_response(monitor_path) + except RedfishConnectionError as exc: + return None, None, str(exc) + if poll_resp.status_code == codes.not_found: + return None, None, f"TaskMonitor GET failed: status {codes.not_found}" + if poll_resp.status_code != codes.accepted: + break + time.sleep(max(int(sleep_s), 1)) + try: + monitor_body = poll_resp.json() if poll_resp else {} + except Exception: + monitor_body = {} + task_uri_from_monitor = ( + monitor_body.get(RF_ODATA_ID) if isinstance(monitor_body, dict) else None + ) + if isinstance(task_uri_from_monitor, str) and task_uri_from_monitor.strip(): + follow_path = _get_path_from_connection(conn, task_uri_from_monitor.strip()) + else: + follow_path = _get_path_from_connection( + conn, task_monitor.rstrip("/").rsplit("/", 1)[0] + ) + follow_task = _task_resource_path(follow_path) + if follow_task: + task_json, poll_err = _poll_task_resource( + conn, follow_task, task_timeout_s, sleep_s + ) + if poll_err: + return None, None, poll_err + else: + try: + task_resp = conn.get_response(follow_path) + except RedfishConnectionError as exc: + return None, None, str(exc) + if task_resp.status_code != codes.ok: + return None, None, f"Task GET failed: {task_resp.status_code}" + task_json = task_resp.json() + if task_json.get("TaskState") != TaskState.completed.value: + return ( + None, + None, + f"Task did not complete: TaskState={task_json.get('TaskState')}", + ) + else: + _log_collect_diag_response( + log, resp.status_code, oem_response, getattr(resp, "text", "") or "" + ) + return None, None, "No TaskMonitor in response and no Location header" + + if not isinstance(task_json, dict): + return None, None, "Task did not complete: missing task body" + + # LogEntry location from Payload.HttpHeaders + headers_list = task_json.get("Payload", {}).get("HttpHeaders", []) or [] + location = None + for header in headers_list: + if isinstance(header, str) and "Location:" in header: + location = header.split("Location:", 1)[-1].strip() + break + if not location: + return None, None, "Location header missing in task Payload.HttpHeaders" + if location.startswith("http"): + log_entry_path = location + else: + log_entry_path = location.lstrip("/") + + log_entry_json, first_status, first_error = _fetch_log_entry_json(conn, log_entry_path, log) + if log_entry_json is None: + if first_error: + err = first_error + if first_status is not None: + err = f"{first_error} (status {first_status})" + elif first_status is not None: + err = f"status {first_status}" + else: + err = "unknown" + return None, None, f"LogEntry GET failed: {err} (GET {log_entry_path})" + + file_stem = oem_type or diag_type + try: + log_bytes = _download_log_and_save(conn, log_entry_json, file_stem, output_dir, log) + except RedfishConnectionError as exc: + return None, None, str(exc) + return log_bytes, log_entry_json, None diff --git a/nodescraper/connection/redfish/redfish_params.py b/nodescraper/connection/redfish/redfish_params.py index 4eb70a96..90783903 100644 --- a/nodescraper/connection/redfish/redfish_params.py +++ b/nodescraper/connection/redfish/redfish_params.py @@ -1,82 +1,130 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from __future__ import annotations - -from typing import Optional, Union - -from pydantic import BaseModel, ConfigDict, Field, SecretStr -from pydantic.networks import IPvAnyAddress - -from nodescraper.connection.inband.sshparams import SSHConnectionParams - -from .redfish_connection import DEFAULT_REDFISH_API_ROOT - - -class RedfishConnectionParams(BaseModel): - """Connection parameters for a Redfish (BMC) API endpoint.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - host: Union[IPvAnyAddress, str] - username: str - password: Optional[SecretStr] = None - port: Optional[int] = Field(default=None, ge=1, le=65535) - use_https: bool = True - verify_ssl: bool = Field( - default=True, - description="Verify HTTPS server certificate. Set False for BMCs with self-signed certs.", - ) - timeout_seconds: float = Field(default=10.0, gt=0, le=300) - use_session_auth: bool = True - api_root: str = Field( - default=DEFAULT_REDFISH_API_ROOT, - description="Redfish API path (e.g. 'redfish/v1'). Override for a different API version.", - ) - - -def redfish_params_to_ssh( - params: Union[RedfishConnectionParams, dict], - *, - ssh_port: Optional[int] = None, - key_filename: Optional[str] = None, -) -> SSHConnectionParams: - """Map Redfish BMC credentials to SSH connection params for shell access.""" - if isinstance(params, dict): - data = dict(params) - ssh_port = data.pop("ssh_port", ssh_port if ssh_port is not None else 22) - key_filename = data.pop("key_filename", key_filename) - params = RedfishConnectionParams.model_validate(data) - else: - if ssh_port is None: - ssh_port = 22 - - return SSHConnectionParams( - hostname=str(params.host), - username=params.username, - password=params.password, - port=ssh_port, - key_filename=key_filename, - ) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator + +# RedfishTargetParams is an alias kept for callers that import it by name. +# Internally, both the top-level config and per-target entries use the same model. +from pydantic.networks import IPvAnyAddress + +from nodescraper.connection.inband.sshparams import SSHConnectionParams + +from .redfish_connection import DEFAULT_REDFISH_API_ROOT + + +class RedfishConnectionParams(BaseModel): + """Connection parameters for a Redfish (BMC) API endpoint. + + Single-target mode: supply ``host`` (and optionally ``username``, ``password``, etc.). + Multi-target mode: supply ``targets`` — a list of ``RedfishConnectionParams`` entries, + each with ``host`` set and an optional ``target_key`` identifier. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + target_key: Optional[str] = Field( + default=None, + description="Identifier used when this entry appears inside a 'targets' list.", + ) + name: Optional[str] = Field(default=None) + targets: Optional[list[RedfishConnectionParams]] = Field( + default=None, description="List of OOB Redfish targets (multi-target mode)." + ) + max_workers: Optional[int] = Field( + default=None, + ge=1, + description=( + "Max concurrent threads for multi-target collection. " + "Defaults to min(len(targets), 32) when not set." + ), + ) + host: Optional[Union[IPvAnyAddress, str]] = None + username: Optional[str] = None + password: Optional[SecretStr] = None + port: Optional[int] = Field(default=None, ge=1, le=65535) + use_https: bool = True + verify_ssl: bool = Field( + default=True, + description="Verify HTTPS server certificate. Set False for BMCs with self-signed certs.", + ) + timeout_seconds: float = Field(default=10.0, gt=0, le=300) + use_session_auth: bool = True + api_root: str = Field( + default=DEFAULT_REDFISH_API_ROOT, + description="Redfish API path (e.g. 'redfish/v1'). Override for a different API version.", + ) + + @model_validator(mode="after") + def _validate_target_config(self) -> "RedfishConnectionParams": + if not self.targets and self.host is None: + raise ValueError( + "Either 'targets' (multi-target mode) or 'host' (single-target mode) must be provided." + ) + return self + + @property + def is_multi_target(self) -> bool: + """True when one or more targets are configured via the ``targets`` list.""" + return bool(self.targets) + + +RedfishConnectionParams.model_rebuild() + +# Backward-compatible alias so existing callers of RedfishTargetParams still work. +RedfishTargetParams = RedfishConnectionParams + + +def redfish_params_to_ssh( + params: Union[RedfishConnectionParams, dict], + *, + ssh_port: Optional[int] = None, + key_filename: Optional[str] = None, +) -> SSHConnectionParams: + """Map Redfish BMC credentials to SSH connection params for shell access.""" + if isinstance(params, dict): + data = dict(params) + ssh_port = data.pop("ssh_port", ssh_port if ssh_port is not None else 22) + key_filename = data.pop("key_filename", key_filename) + params = RedfishConnectionParams.model_validate(data) + else: + if ssh_port is None: + ssh_port = 22 + + if params.host is None: + raise ValueError( + "SSH mapping requires a Redfish host. Multi-target configs apply only to Redfish plugins." + ) + + return SSHConnectionParams( + hostname=str(params.host), + username=params.username, + password=params.password, + port=ssh_port, + key_filename=key_filename, + ) diff --git a/nodescraper/connection/redfish/redfish_path.py b/nodescraper/connection/redfish/redfish_path.py index 0cd14024..f5dec6f3 100644 --- a/nodescraper/connection/redfish/redfish_path.py +++ b/nodescraper/connection/redfish/redfish_path.py @@ -1,61 +1,61 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Fluent Redfish path builder with pathlib-like / syntax and optional parameter substitution.""" -from __future__ import annotations - -from typing import Dict - - -class RedfishPath: - """Fluent interface for building Redfish URI paths.""" - - def __init__(self, base: str = "") -> None: - self._path = (base or "").strip().strip("/") - self._params: Dict[str, str] = {} - - def __truediv__(self, segment: str) -> RedfishPath: - """Allow path / \"segment\" syntax. Leading/trailing slashes on segment are stripped.""" - seg = (segment or "").strip().strip("/") - if not seg: - return RedfishPath(self._path) - new_path = RedfishPath() - new_path._path = f"{self._path}/{seg}" if self._path else seg - new_path._params = dict(self._params) - return new_path - - def __call__(self, **params: str) -> str: - """Substitute placeholders in the path and return the final path string. - - Placeholders use {key}; e.g. path \"Systems/{id}/LogServices/DiagLogs\" with (id=\"UBB\") - returns \"Systems/UBB/LogServices/DiagLogs\". - """ - result = self._path - for key, value in params.items(): - result = result.replace(f"{{{key}}}", value) - return result - - def __str__(self) -> str: - return self._path +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Fluent Redfish path builder with pathlib-like / syntax and optional parameter substitution.""" +from __future__ import annotations + +from typing import Dict + + +class RedfishPath: + """Fluent interface for building Redfish URI paths.""" + + def __init__(self, base: str = "") -> None: + self._path = (base or "").strip().strip("/") + self._params: Dict[str, str] = {} + + def __truediv__(self, segment: str) -> RedfishPath: + """Allow path / \"segment\" syntax. Leading/trailing slashes on segment are stripped.""" + seg = (segment or "").strip().strip("/") + if not seg: + return RedfishPath(self._path) + new_path = RedfishPath() + new_path._path = f"{self._path}/{seg}" if self._path else seg + new_path._params = dict(self._params) + return new_path + + def __call__(self, **params: str) -> str: + """Substitute placeholders in the path and return the final path string. + + Placeholders use {key}; e.g. path \"Systems/{id}/LogServices/DiagLogs\" with (id=\"UBB\") + returns \"Systems/UBB/LogServices/DiagLogs\". + """ + result = self._path + for key, value in params.items(): + result = result.replace(f"{{{key}}}", value) + return result + + def __str__(self) -> str: + return self._path diff --git a/nodescraper/connection/redfish/ssh_proxy_connection.py b/nodescraper/connection/redfish/ssh_proxy_connection.py new file mode 100644 index 00000000..1d47a963 --- /dev/null +++ b/nodescraper/connection/redfish/ssh_proxy_connection.py @@ -0,0 +1,368 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import json as json_lib +import logging +import secrets +import shlex +from http import HTTPStatus +from typing import Any, Optional, Union +from urllib.parse import urljoin, urlparse + +from requests.structures import CaseInsensitiveDict + +from nodescraper.connection.inband.inband import CommandArtifact +from nodescraper.connection.inband.inbandremote import RemoteShell, SSHConnectionError + +from .redfish_connection import RedfishConnection, RedfishConnectionError +from .redfish_path import RedfishPath + + +def parse_curl_headers(header_text: str) -> CaseInsensitiveDict: + """Parse curl -D header blocks into a case-insensitive map (last hop wins). + + Args: + header_text: Raw HTTP header dump from curl -D. + + Returns: + CaseInsensitiveDict of header name to value. + """ + headers: CaseInsensitiveDict = CaseInsensitiveDict() + for block in header_text.replace("\r\n", "\n").split("\n\n"): + for line in block.split("\n"): + if not line or line.lower().startswith("http/"): + continue + if ":" not in line: + continue + name, value = line.split(":", 1) + headers[name.strip()] = value.strip() + return headers + + +_DEAD_SESSION_MARKERS = ( + "server connection dropped", + "ssh session not active", + "session not active", + "timeout opening channel", + "socket is closed", + "connection reset", + "not connected", + "ssh channel failed", +) + + +def _is_dead_session(detail: str) -> bool: + """Return whether an SSH error means the BMC session is gone. + + Args: + detail: Error text from the SSH client. + + Returns: + True when the session should be opened again. + """ + text = detail.lower() + return any(marker in text for marker in _DEAD_SESSION_MARKERS) + + +def split_curl_stdout(raw: str, marker: str) -> tuple[int, str, bytes]: + """Split curl stdout into an HTTP status, headers, and body. + + Args: + raw: Combined curl stdout. + marker: Trailer token written by curl -w. + + Returns: + Status code, header text, and body bytes. + """ + payload, found, tail = raw.rpartition(marker) + if not found: + raise RedfishConnectionError("curl stdout is missing the status trailer") + if payload.endswith("\n"): + payload = payload[:-1] + try: + status_code = int(tail.strip()) + except (TypeError, ValueError) as exc: + raise RedfishConnectionError(f"curl status is not numeric: {tail.strip()}") from exc + header_text, body_text = _split_header_and_body(payload) + return status_code, header_text, body_text.encode("utf-8") + + +def _split_header_and_body(payload: str) -> tuple[str, str]: + """Split a curl -D - payload into header text and body text. + + Args: + payload: Headers and body with the status trailer removed. + + Returns: + Header text and body text. Header text is empty when no header block is present. + """ + for separator in ("\r\n\r\n", "\n\n"): + index = payload.find(separator) + if index != -1: + return payload[:index], payload[index + len(separator) :] + return "", payload + + +def _url_path(url: str) -> str: + """Return the URL path for log messages (no host). + + Args: + url: Absolute URL. + + Returns: + Path component or the original string. + """ + return urlparse(url).path or url + + +class CurlResponse: + """Minimal requests.Response stand-in for curl over SSH.""" + + def __init__( + self, + status_code: int, + content: bytes, + headers: Optional[CaseInsensitiveDict] = None, + ): + self.status_code = status_code + self.content = content + self.headers = headers or CaseInsensitiveDict() + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 400 + + @property + def reason(self) -> str: + try: + return HTTPStatus(self.status_code).phrase + except ValueError: + return "" + + @property + def text(self) -> str: + return self.content.decode("utf-8", errors="replace") + + def json(self) -> Any: + """Parse the body as JSON. + + Returns: + Parsed object, or empty dict when the body is empty. + """ + if not self.content or not self.text.strip(): + return {} + return json_lib.loads(self.text) + + +class SshProxyRedfishConnection(RedfishConnection): + """Redfish client that runs curl on a remote BMC to reach an internal AMC URL.""" + + def __init__( + self, + shell: RemoteShell, + base_url: str, + timeout: float = 60.0, + api_root: Optional[str] = None, + ): + super().__init__( + base_url=base_url, + username="", + password=None, + timeout=timeout, + use_session_auth=False, + verify_ssl=False, + api_root=api_root, + ) + self._shell = shell + + def _cmd_timeout(self) -> int: + return max(int(self.timeout) + 15, 30) + + def _curl(self, url: str, extra_flags: str = "") -> CurlResponse: + """Run curl over SSH, and open the session again once if it has dropped. + + Args: + url: Absolute URL to request. + extra_flags: Extra curl arguments, including a trailing space when set. + + Returns: + CurlResponse with status, headers, and body. + """ + if self._shell._proxy_ssh_down: + raise RedfishConnectionError("SSH session not active") + try: + return self._curl_once(url, extra_flags) + except RedfishConnectionError as exc: + if not _is_dead_session(str(exc)): + raise + logging.getLogger("nodescraper").info( + "SSH proxy session dropped; reconnecting and retrying %s", + _url_path(url), + ) + try: + self._shell.reconnect_ssh() + except (SSHConnectionError, OSError) as reconnect_exc: + self._shell._proxy_ssh_down = True + raise RedfishConnectionError( + f"SSH proxy reconnect failed: {reconnect_exc}" + ) from reconnect_exc + try: + return self._curl_once(url, extra_flags) + except RedfishConnectionError as retry_exc: + if _is_dead_session(str(retry_exc)): + self._shell._proxy_ssh_down = True + raise + + def _curl_once(self, url: str, extra_flags: str = "") -> CurlResponse: + """Run one curl on the BMC and read the response from that command. + + Args: + url: Absolute URL to request. + extra_flags: Extra curl arguments, including a trailing space when set. + + Returns: + CurlResponse with status, headers, and body. + """ + marker = f"NODESCRAPER{secrets.token_hex(16)}" + max_time = max(int(self.timeout), 1) + trailer = f"\n{marker}%{{http_code}}\n" + cmd = ( + f"curl -sS --max-time {max_time} -D - -o - " + f"-w {shlex.quote(trailer)} {extra_flags}{shlex.quote(url)}" + ) + result: CommandArtifact = self._shell.run_command( + cmd, timeout=self._cmd_timeout(), strip=False + ) + if _is_dead_session(result.stderr or ""): + raise RedfishConnectionError((result.stderr or "").strip() or "SSH session not active") + raw = result.stdout or "" + detail = (result.stderr or result.stdout or "").strip() + if marker not in raw: + raise RedfishConnectionError(f"curl to {_url_path(url)} failed: {detail}") + try: + status_code, header_text, body = split_curl_stdout(raw, marker) + except RedfishConnectionError as exc: + raise RedfishConnectionError(f"curl to {_url_path(url)} failed: {exc}") from exc + if result.exit_code != 0 and status_code == 0: + raise RedfishConnectionError(f"curl to {_url_path(url)} failed: {detail}") + return CurlResponse( + status_code=status_code, + content=body, + headers=parse_curl_headers(header_text), + ) + + def _ensure_session(self): + return None + + def _login_session(self) -> None: + return None + + def get_response(self, path: Union[str, "RedfishPath"]) -> CurlResponse: + """GET a Redfish path via curl on the SSH host. + + Args: + path: Redfish URI or RedfishPath. + + Returns: + CurlResponse with status, headers, and body. + """ + return self._curl(self._abs_url(str(path))) + + def post( + self, path: Union[str, "RedfishPath"], json: Optional[dict[str, Any]] = None + ) -> CurlResponse: + """POST JSON to a Redfish path via curl on the SSH host. + + Args: + path: Redfish URI or RedfishPath. + json: JSON body. + + Returns: + CurlResponse with status, headers, and body. + """ + payload = json_lib.dumps(json or {}) + extra = ( + f"-X POST -H {shlex.quote('Content-Type: application/json')} " + f"-d {shlex.quote(payload)} " + ) + return self._curl(self._abs_url(str(path)), extra_flags=extra) + + def get(self, path: RedfishPath) -> dict[str, Any]: + """GET a Redfish path and return the JSON body. + + Args: + path: Redfish path object. + + Returns: + Parsed JSON object. + + Raises: + RedfishConnectionError: When the GET fails or the body is not JSON. + """ + path_str = str(path) + resp = self.get_response(path_str) + if not resp.ok: + raise RedfishConnectionError( + f"GET {path_str} failed: {resp.status_code} {resp.reason}", + response=resp, + ) + try: + data = resp.json() + except (ValueError, json_lib.JSONDecodeError) as exc: + raise RedfishConnectionError( + f"GET {path_str} returned invalid JSON: {exc}", + response=resp, + ) from exc + if not isinstance(data, dict): + raise RedfishConnectionError( + f"GET {path_str} did not return a JSON object", + response=resp, + ) + return data + + def copy(self) -> "SshProxyRedfishConnection": + """Return a wrapper that shares the same SSH session. + + Returns: + SshProxyRedfishConnection using the same RemoteShell. + """ + return SshProxyRedfishConnection( + shell=self._shell, + base_url=self.base_url, + timeout=self.timeout, + api_root=self.api_root, + ) + + def close(self) -> None: + self._session = None + self._session_token = None + self._session_uri = None + + def _abs_url(self, path: str) -> str: + if path.startswith("http"): + return path + return urljoin(self.base_url + "/", path.lstrip("/")) diff --git a/nodescraper/connection/redfish/ssh_proxy_manager.py b/nodescraper/connection/redfish/ssh_proxy_manager.py new file mode 100644 index 00000000..334dfa2c --- /dev/null +++ b/nodescraper/connection/redfish/ssh_proxy_manager.py @@ -0,0 +1,342 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from logging import Logger +from typing import Any, ClassVar, Optional, Union + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces.connectionmanager import ConnectionManager +from nodescraper.interfaces.taskresulthook import TaskResultHook +from nodescraper.models import SystemInfo, TaskResult +from nodescraper.utils import get_exception_traceback + +from ..inband.inbandremote import RemoteShell, SSHConnectionError +from .redfish_connection import RedfishConnectionError +from .redfish_manager import MultiTargetRedfishConnection, _build_base_url +from .ssh_proxy_connection import SshProxyRedfishConnection +from .ssh_proxy_params import RedfishSshProxyConnectionParams + + +class RedfishSshProxyConnectionManager( + ConnectionManager[SshProxyRedfishConnection, RedfishSshProxyConnectionParams] +): + """SSH to a BMC and curl Redfish on an address reachable only from that host (AMC).""" + + CONNECTION_ARGS_MODEL: ClassVar[type[RedfishSshProxyConnectionParams]] = ( + RedfishSshProxyConnectionParams + ) + + def __init__( + self, + system_info: SystemInfo, + logger: Optional[Logger] = None, + max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, + parent: Optional[str] = None, + task_result_hooks: Optional[list[TaskResultHook]] = None, + connection_args: Optional[Union[RedfishSshProxyConnectionParams, dict[str, Any]]] = None, + **kwargs, + ): + super().__init__( + system_info, + logger, + max_event_priority_level, + parent, + task_result_hooks, + connection_args, + **kwargs, + ) + + def connect(self) -> TaskResult: + """Open BMC SSH and verify AMC Redfish via curl. + + Returns: + TaskResult: Connection result. One failed target is a warning when another connects. + """ + params = self._validated_params() + if params is None: + return self.result + if params.is_multi_target: + return self._connect_multi(params) + return self._connect_single(params) + + def _validated_params(self) -> Optional[RedfishSshProxyConnectionParams]: + """Return connection params or record a failure when they are missing or invalid. + + Returns: + Optional[RedfishSshProxyConnectionParams]: Parsed params, or None when invalid. + """ + if not self.connection_args: + self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect through the BMC: no connection details were provided.", + "No Redfish SSH-proxy connection parameters provided", + {"connection_args": None}, + ) + return None + + raw = self.connection_args + if isinstance(raw, dict): + try: + return RedfishSshProxyConnectionParams.model_validate(raw) + except Exception as exc: + self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect through the BMC: the connection details are not valid.", + f"Invalid Redfish SSH-proxy connection parameters: {exc}", + {"exception": str(exc)}, + ) + return None + if isinstance(raw, RedfishSshProxyConnectionParams): + return raw + self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect through the BMC: the connection details are not valid.", + "Redfish SSH-proxy connection_args must be dict or RedfishSshProxyConnectionParams " + f"(got {type(raw).__name__})", + {"connection_args_type": type(raw).__name__}, + ) + return None + + def _connect_single(self, params: RedfishSshProxyConnectionParams) -> TaskResult: + """Connect one BMC SSH proxy. + + Args: + params: Single-target SSH-proxy parameters. + + Returns: + TaskResult: Connection result. + """ + debug_data = { + "ssh_host": str(params.ssh.hostname) if params.ssh else None, + "ssh_port": params.ssh.port if params.ssh else None, + "ssh_username": params.ssh.username if params.ssh else None, + "redfish_host": str(params.host), + "redfish_port": params.port, + "use_https": params.use_https, + "timeout_seconds": params.timeout_seconds, + } + try: + self.connection = self._open_proxy(params) + except SSHConnectionError as exc: + return self._fail_connect( + EventCategory.SSH, + "Could not connect to the BMC over SSH.", + f"SSH connection to BMC '{debug_data['ssh_host']}' failed: {exc}", + {**debug_data, "exception": str(exc)}, + ) + except RedfishConnectionError as exc: + return self._fail_connect( + EventCategory.RUNTIME, + "Connected to the BMC, but could not reach Redfish through it.", + f"Redfish check through BMC SSH proxy failed: {exc}", + {**debug_data, "exception": str(exc)}, + ) + except ValueError as exc: + return self._fail_connect( + EventCategory.RUNTIME, + "Cannot connect through the BMC: the connection details are not valid.", + f"Invalid Redfish SSH-proxy connection parameters: {exc}", + {**debug_data, "exception": str(exc)}, + ) + except Exception as exc: + return self._fail_connect( + EventCategory.RUNTIME, + "Connecting through the BMC failed unexpectedly.", + f"Unexpected error during SSH connection: {exc}", + {**debug_data, "exception": str(exc), **get_exception_traceback(exc)}, + ) + return self.result + + def _connect_multi(self, params: RedfishSshProxyConnectionParams) -> TaskResult: + """Connect each SSH-proxy target. One failure does not drop the others. + + Args: + params: Parameters whose targets list is set. + + Returns: + TaskResult: Warning when any target connects, execution failure when none do. + """ + self.target_connections: dict[str, SshProxyRedfishConnection] = {} + failed_targets: dict[str, str] = {} + for target in params.targets or []: + key, conn, error = self._connect_target(target) + if conn is None: + failed_targets[key] = error or f"SSH-proxy connection failed for target {key!r}" + continue + if key in self.target_connections: + self._log_event( + category=EventCategory.RUNTIME, + description=( + f"Duplicate SSH-proxy target key {key!r}; keeping the first connection" + ), + priority=EventPriority.WARNING, + console_log=True, + ) + self._close_proxy(conn) + continue + self.target_connections[key] = conn + + lines = "\n".join(f"[{k}] {v}" for k, v in failed_targets.items()) + if not self.target_connections: + summary = "Could not connect to any of the BMC targets." + self._fail_connect( + EventCategory.RUNTIME, + summary, + f"SSH-proxy Redfish connection failed for every target ({len(failed_targets)} tried)\n{lines}", + {"failed_targets": dict(failed_targets)}, + ) + return self.result + + if failed_targets: + self.result.status = ExecutionStatus.WARNING + self.result.message = f"Some SSH-proxy Redfish targets failed to connect\n{lines}" + self.connection = MultiTargetRedfishConnection( # type: ignore[assignment] + self.target_connections, + max_workers=params.max_workers, + failed_targets=failed_targets, + ) + return self.result + + def _connect_target( + self, target: RedfishSshProxyConnectionParams + ) -> tuple[str, Optional[SshProxyRedfishConnection], Optional[str]]: + """Connect one target and keep going when it fails. + + Args: + target: SSH-proxy parameters for a single BMC. + + Returns: + tuple[str, Optional[SshProxyRedfishConnection], Optional[str]]: Key, connection, and error. + """ + key = _proxy_target_key(target) + try: + return key, self._open_proxy(target), None + except Exception as exc: + if isinstance(exc, (SSHConnectionError, RedfishConnectionError)): + description = f"SSH-proxy Redfish connection failed for target {key!r}: {exc}" + else: + description = f"Unknown SSH-proxy Redfish error for target {key!r}: {exc}" + self._log_event( + category=( + EventCategory.SSH + if isinstance(exc, SSHConnectionError) + else EventCategory.RUNTIME + ), + description=description, + priority=EventPriority.WARNING, + console_log=True, + ) + return key, None, description + + def _open_proxy(self, params: RedfishSshProxyConnectionParams) -> SshProxyRedfishConnection: + """Open one BMC SSH session and verify the AMC Redfish root. + + Args: + params: Single-target SSH-proxy parameters. + + Returns: + SshProxyRedfishConnection: Open connection. + + Raises: + ValueError: When host or ssh is missing. + """ + if params.host is None or params.ssh is None: + raise ValueError("SSH-proxy target requires host and ssh") + base_url = _build_base_url(str(params.host), params.port, params.use_https) + self.logger.info( + "Connecting SSH proxy Redfish: ssh=%s curl=%s", + params.ssh.hostname, + base_url, + ) + shell = RemoteShell(params.ssh) + conn: Optional[SshProxyRedfishConnection] = None + try: + shell.connect_ssh() + conn = SshProxyRedfishConnection( + shell=shell, + base_url=base_url, + timeout=params.timeout_seconds, + api_root=params.api_root, + ) + conn.get_service_root() + return conn + except Exception: + if conn is not None: + self._close_proxy(conn) + else: + try: + shell.client.close() + except Exception: + pass + raise + + @staticmethod + def _close_proxy(conn: SshProxyRedfishConnection) -> None: + """Close the curl wrapper and the BMC SSH session. + + Args: + conn: SSH-proxy Redfish connection to close. + + Returns: + None. + """ + conn.close() + try: + conn._shell.client.close() + except Exception: + pass + + def disconnect(self) -> None: + """Close every SSH-proxy session and keep collected multi-target data.""" + conn = self.connection + if isinstance(conn, MultiTargetRedfishConnection): + self._multi_target_data: dict = dict(conn.multi_target_data) + for target_conn in conn.target_connections.values(): + if isinstance(target_conn, SshProxyRedfishConnection): + self._close_proxy(target_conn) + else: + target_conn.close() + self.target_connections = {} + elif isinstance(conn, SshProxyRedfishConnection): + self._close_proxy(conn) + super().disconnect() + + +def _proxy_target_key(target: RedfishSshProxyConnectionParams) -> str: + """Return the target key, falling back to the SSH hostname. + + Args: + target: One SSH-proxy target. + + Returns: + str: target_key, otherwise the SSH hostname, otherwise the AMC host. + """ + if target.target_key: + return target.target_key + if target.ssh is not None: + return str(target.ssh.hostname) + return str(target.host) diff --git a/nodescraper/connection/redfish/ssh_proxy_params.py b/nodescraper/connection/redfish/ssh_proxy_params.py new file mode 100644 index 00000000..426275b5 --- /dev/null +++ b/nodescraper/connection/redfish/ssh_proxy_params.py @@ -0,0 +1,104 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic.networks import IPvAnyAddress + +from nodescraper.connection.inband.sshparams import SSHConnectionParams + +from .redfish_connection import DEFAULT_REDFISH_API_ROOT + + +class RedfishSshProxyConnectionParams(BaseModel): + """Redfish over SSH: curl on the BMC to an AMC address reachable from that host. + + Single-target mode supplies host and ssh. Multi-target mode supplies targets, + and each entry has its own ssh block and AMC host. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + target_key: Optional[str] = Field( + default=None, + description="Identifier used when this entry appears inside a targets list.", + ) + targets: Optional[list["RedfishSshProxyConnectionParams"]] = Field( + default=None, + description="BMC SSH-proxy targets. Each entry has its own ssh block and AMC host.", + ) + max_workers: Optional[int] = Field( + default=None, + ge=1, + description=( + "Max concurrent collection threads. Defaults to the target count, capped at 32." + ), + ) + host: Optional[Union[IPvAnyAddress, str]] = Field( + default=None, + description="Redfish host as seen from the SSH target (internal AMC address).", + ) + ssh: Optional[SSHConnectionParams] = Field( + default=None, + description="SSH parameters for the BMC that can reach host.", + ) + port: Optional[int] = Field(default=80, ge=1, le=65535) + use_https: bool = Field( + default=False, + description="Use https when curling Redfish from the SSH target.", + ) + timeout_seconds: float = Field(default=60.0, gt=0, le=3600) + api_root: str = Field( + default=DEFAULT_REDFISH_API_ROOT, + description="Redfish API path (e.g. redfish/v1).", + ) + + @model_validator(mode="after") + def _validate_target_config(self) -> "RedfishSshProxyConnectionParams": + """Require host and ssh unless a targets list is set. + + Returns: + RedfishSshProxyConnectionParams: This params object. + """ + if self.targets: + return self + if self.host is None or self.ssh is None: + raise ValueError("Either targets or both host and ssh must be provided.") + return self + + @property + def is_multi_target(self) -> bool: + """True when one or more targets are configured via the targets list. + + Returns: + bool: True when targets is non-empty. + """ + return bool(self.targets) + + +RedfishSshProxyConnectionParams.model_rebuild() diff --git a/nodescraper/enums/eventcategory.py b/nodescraper/enums/eventcategory.py index 2c5d5514..bc45726c 100644 --- a/nodescraper/enums/eventcategory.py +++ b/nodescraper/enums/eventcategory.py @@ -41,6 +41,8 @@ class EventCategory(AutoNameStrEnum): - OS Generic Operating System events. Does not include specific events from OS which point to another category + - PROCESS_READ + Successful CPU process collection events. - PLATFORM Generic Platform Errors e.g. topo enumeration Platform-specific errors which do not fall under other categories (e.g. BMC, SMC, UBB) @@ -79,6 +81,7 @@ class EventCategory(AutoNameStrEnum): RAS = auto() IO = auto() OS = auto() + PROCESS_READ = auto() PLATFORM = auto() APPLICATION = auto() MEMORY = auto() diff --git a/nodescraper/enums/osfamily.py b/nodescraper/enums/osfamily.py index bce34550..81024631 100644 --- a/nodescraper/enums/osfamily.py +++ b/nodescraper/enums/osfamily.py @@ -32,5 +32,6 @@ class OSFamily(enum.Enum): WINDOWS = enum.auto() UNKNOWN = enum.auto() LINUX = enum.auto() + ESXI = enum.auto() EOS = enum.auto() SONIC = enum.auto() diff --git a/nodescraper/enums/taskstate.py b/nodescraper/enums/taskstate.py index 46d9d3fa..7735d049 100644 --- a/nodescraper/enums/taskstate.py +++ b/nodescraper/enums/taskstate.py @@ -1,44 +1,44 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import enum - - -class TaskState(enum.Enum): - """Redfish Task resource TaskState.""" - - new = "New" - starting = "Starting" - running = "Running" - suspended = "Suspended" - interrupted = "Interrupted" - pending = "Pending" - stopping = "Stopping" - completed = "Completed" - killed = "Killed" - exception = "Exception" - service = "Service" - cancelling = "Cancelling" - cancelled = "Cancelled" +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import enum + + +class TaskState(enum.Enum): + """Redfish Task resource TaskState.""" + + new = "New" + starting = "Starting" + running = "Running" + suspended = "Suspended" + interrupted = "Interrupted" + pending = "Pending" + stopping = "Stopping" + completed = "Completed" + killed = "Killed" + exception = "Exception" + service = "Service" + cancelling = "Cancelling" + cancelled = "Cancelled" diff --git a/nodescraper/helpers/__init__.py b/nodescraper/helpers/__init__.py index 2cca9561..69d0ff0d 100644 --- a/nodescraper/helpers/__init__.py +++ b/nodescraper/helpers/__init__.py @@ -1,38 +1,38 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from nodescraper.plugins.inband.dmesg import ( - compare_dmesg_runs, - compute_extracted_errors, - find_dmesg_datamodel_path, - load_dmesg_data, -) - -__all__ = [ - "compare_dmesg_runs", - "compute_extracted_errors", - "find_dmesg_datamodel_path", - "load_dmesg_data", -] +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.plugins.inband.dmesg import ( + compare_dmesg_runs, + compute_extracted_errors, + find_dmesg_datamodel_path, + load_dmesg_data, +) + +__all__ = [ + "compare_dmesg_runs", + "compute_extracted_errors", + "find_dmesg_datamodel_path", + "load_dmesg_data", +] diff --git a/nodescraper/helpers/plugin_execution_target.py b/nodescraper/helpers/plugin_execution_target.py new file mode 100644 index 00000000..4e4cd3e4 --- /dev/null +++ b/nodescraper/helpers/plugin_execution_target.py @@ -0,0 +1,119 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Any, Optional, Union + +from pydantic import BaseModel + +from nodescraper.connection.inband.inbandmanager import InBandConnectionManager +from nodescraper.connection.redfish.redfish_manager import RedfishConnectionManager +from nodescraper.enums import SystemLocation +from nodescraper.interfaces import ConnectionManager, PluginInterface +from nodescraper.models import SystemInfo + + +def _host_from_connection_args(raw: Optional[Union[dict[str, Any], BaseModel]]) -> Optional[str]: + if raw is None: + return None + if isinstance(raw, dict): + targets = raw.get("targets") + if isinstance(targets, list) and targets: + hosts = [] + for item in targets: + if isinstance(item, dict): + host = item.get("target_key") or item.get("host") + if host: + hosts.append(str(host)) + if hosts: + return ", ".join(hosts) + for key in ("host", "hostname", "ip"): + value = raw.get(key) + if value: + return str(value) + return None + for key in ("host", "hostname", "ip"): + value = getattr(raw, key, None) + if value is not None and str(value): + return str(value) + return None + + +def format_in_band_target_summary( + system_info: SystemInfo, + connection_configs: Optional[dict[str, Union[dict[str, Any], BaseModel]]] = None, +) -> str: + """Return a short summary of the default in-band execution target.""" + if system_info.location == SystemLocation.REMOTE: + host = _host_from_connection_args((connection_configs or {}).get("InBandConnectionManager")) + if host: + return f"In-band default: remote host via SSH ({host})" + return "In-band default: remote host via SSH" + host_name = system_info.name or "local host" + return f"In-band default: local host ({host_name})" + + +def format_plugin_execution_target( + plugin_class: type[PluginInterface], + *, + system_info: SystemInfo, + connection_manager: Optional[ConnectionManager] = None, + connection_configs: Optional[dict[str, Union[dict[str, Any], BaseModel]]] = None, +) -> Optional[str]: + """Return a one-line description of where a plugin collects data from.""" + connection_type = getattr(plugin_class, "CONNECTION_TYPE", None) + if connection_type is None: + return None + + configs = connection_configs or {} + manager_args = ( + getattr(connection_manager, "connection_args", None) if connection_manager else None + ) + + if connection_type is RedfishConnectionManager or issubclass( + connection_type, RedfishConnectionManager + ): + host = _host_from_connection_args(manager_args) or _host_from_connection_args( + configs.get("RedfishConnectionManager") + ) + if host: + return f"Execution target: BMC via Redfish OOB ({host})" + return "Execution target: BMC via Redfish OOB" + + if connection_type is InBandConnectionManager or issubclass( + connection_type, InBandConnectionManager + ): + if system_info.location == SystemLocation.REMOTE: + host = _host_from_connection_args(manager_args) or _host_from_connection_args( + configs.get("InBandConnectionManager") + ) + if host: + return f"Execution target: remote host via SSH ({host})" + return "Execution target: remote host via SSH" + host_name = system_info.name or "local host" + return f"Execution target: local host ({host_name})" + + return f"Execution target: {connection_type.__name__}" diff --git a/nodescraper/interfaces/connectionmanager.py b/nodescraper/interfaces/connectionmanager.py index d413ffaf..60937f8d 100644 --- a/nodescraper/interfaces/connectionmanager.py +++ b/nodescraper/interfaces/connectionmanager.py @@ -29,14 +29,13 @@ import logging import types from functools import wraps -from typing import Callable, Generic, Optional, TypeVar, Union +from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union from pydantic import BaseModel from nodescraper.constants import DEFAULT_EVENT_REPORTER from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.models import SystemInfo, TaskResult -from nodescraper.typeutils import TypeUtils from nodescraper.utils import get_exception_traceback from .task import Task @@ -68,8 +67,25 @@ def wrapper( "Exception connecting with %s", connection_manager.__class__.__name__ ) connection_manager.result.status = ExecutionStatus.EXECUTION_FAILURE + connection_manager.result.message = f"Connection failed: {str(exception)}" result = connection_manager.result + connection_manager.result = result + + if result.status == ExecutionStatus.UNSET and connection_manager.connection is None: + # Force a fail if conncetion manager failed to give us a connection + result.status = ExecutionStatus.EXECUTION_FAILURE + + if result.status >= ExecutionStatus.ERROR and not result.events: + result.message = result.message or "Connection could not be established" + # Push an event if the connection manager didn't + connection_manager._log_event( + category=EventCategory.RUNTIME, + description=result.message, + priority=EventPriority.CRITICAL, + console_log=True, + ) + result.finalize() connection_manager._run_hooks(result) @@ -89,18 +105,37 @@ class ConnectionManager(Task, Generic[TConnection, TConnectArg]): TASK_TYPE = "CONNECTION_MANAGER" + CONNECTION_ARGS_MODEL: ClassVar[Optional[type[BaseModel]]] + def __init__( self, system_info: SystemInfo, logger: Optional[logging.Logger] = None, max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, parent: Optional[str] = None, - task_result_hooks: Optional[list[TaskResultHook], None] = None, - connection_args: Optional[Union[TConnectArg, dict]] = None, + task_result_hooks: Optional[list[TaskResultHook]] = None, + connection_args: Optional[Union[TConnectArg, dict[str, Any]]] = None, event_reporter: str = DEFAULT_EVENT_REPORTER, session_id: Optional[str] = None, **kwargs, ): + """Creates a Connection Manager Task instance. + + Args: + system_info (SystemInfo): System information for the connection manager task. + logger (Optional[logging.Logger], optional): Logger instance for the connection manager task. Defaults to None. + max_event_priority_level (Union[EventPriority, str], optional): Maximum priority level for events. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): Parent task identifier. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook], None], optional): List of task result hooks to be executed. Defaults to None. + connection_args (Optional[Union[TConnectArg, dict]], optional): Connection arguments for the connection manager. + If a dictionary is provided, it will be transformed into a BaseModel instance of the expected type. Defaults to None. + When it is None then connection args will be None and it will not be built into the default connection model. + event_reporter (str, optional): Event reporter identifier. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): Session identifier for the connection manager task. Defaults to None. + + Raises: + ValueError: Will raise a ValueError when the connection_args cannot be mapped to the expected model. + """ super().__init__( system_info=system_info, logger=logger, @@ -113,19 +148,19 @@ def __init__( ) if isinstance(connection_args, dict): - generic_map = TypeUtils.get_generic_map(self.__class__) - connection_arg_model = generic_map.get(TConnectArg) - if not connection_arg_model: - raise ValueError("No model defined for connection args") - - connection_args = connection_arg_model(**connection_args) + connection_arg_model = self.CONNECTION_ARGS_MODEL + if connection_arg_model is not None: + connection_args = connection_arg_model(**connection_args) + elif connection_args: + self.logger.warning("Connection args provided but no model defined") self.connection_args = connection_args self.connection: Optional[TConnection] = None def __init_subclass__(cls, **kwargs) -> None: super().__init_subclass__(**kwargs) - + if not hasattr(cls, "CONNECTION_ARGS_MODEL"): + raise ValueError("Subclasses of ConnectionManager must define CONNECTION_ARGS_MODEL") if "connect" in cls.__dict__: cls.connect = connect_decorator(cls.connect) @@ -145,7 +180,51 @@ def __exit__( def connect(self) -> TaskResult: """initialize connection""" + def _fail_connect( + self, + category: EventCategory, + message: str, + event_description: str | None = None, + event_data: Optional[dict[Any, Any]] = None, + ) -> TaskResult: + """Record a failed connection attempt: one CRITICAL event, nullifying the connection, and marking the result as failure. + + Call this for every expected failure inside connect() and return its result. Exceptions + that connect() does not expect are not caught there; connect_decorator reports them. + + Args: + category (EventCategory): Event category describing where the failure happened. + message (str): Short, plain-language summary for the end user (becomes the task result message), + e.g. "Could not connect to the BMC over SSH". No exception text or jargon. + event_description (str): Event description with a little more debugging detail than the message, e.g. the + host and the underlying error string. When None or empty the message is used. + event_data (Optional[dict], optional): Full debug information for the event, e.g. host, port, exception type, + exception string and traceback. Never include secrets. This data must be JSON-serializable. + Defaults to None. + + Returns: + TaskResult: The failed connection result. + """ + self._log_event( + category=category, + description=event_description or message, + data=event_data, + priority=EventPriority.CRITICAL, + console_log=True, + ) + self.connection = None + self.result.message = message + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result + + def safe_disconnect(self) -> None: + """Disconnect without raising; a failure is logged.""" + try: + self.disconnect() + except Exception: + self.logger.exception("Failed to disconnect %s", self.__class__.__name__) + def disconnect(self): - """disconnect connection (Optional)""" + """disconnect connection (Optional). The result of the connection attempt is kept.""" + self.logger.info(f"Connection {self.__class__.__name__} disconnected.") self.connection = None - self.result.status = ExecutionStatus.UNSET diff --git a/nodescraper/interfaces/dataanalyzertask.py b/nodescraper/interfaces/dataanalyzertask.py index c91fb251..36a46e55 100644 --- a/nodescraper/interfaces/dataanalyzertask.py +++ b/nodescraper/interfaces/dataanalyzertask.py @@ -118,8 +118,10 @@ class DataAnalyzer(Task, abc.ABC, Generic[TDataModel, TAnalyzeArg]): def __init_subclass__(cls, **kwargs: dict[str, Any]) -> None: super().__init_subclass__(**kwargs) - if not inspect.isabstract(cls) and cls.DATA_MODEL is None: + if not inspect.isabstract(cls) and not getattr(cls, "DATA_MODEL", None): raise TypeError(f"No data model set for {cls.__name__}") + if not hasattr(cls, "analyze_data") or not callable(cls.analyze_data): + raise TypeError(f"No analyze_data method defined for {cls.__name__}") if "analyze_data" in vars(cls): setattr(cls, "analyze_data", analyze_decorator(cls.analyze_data)) # noqa diff --git a/nodescraper/interfaces/datacollectortask.py b/nodescraper/interfaces/datacollectortask.py index 034303bd..cc96e974 100644 --- a/nodescraper/interfaces/datacollectortask.py +++ b/nodescraper/interfaces/datacollectortask.py @@ -49,7 +49,9 @@ from .taskresulthook import TaskResultHook -def _supported_sku_name_set(supported: Optional[set[Union[str, Enum]]]) -> Optional[set[str]]: +def _supported_sku_name_set( + supported: Optional[set[Union[str, Enum]]], +) -> Optional[set[str]]: """Map ``SUPPORTED_SKUS`` to string names for comparison with ``SystemInfo.sku``.""" if not supported: return None @@ -118,7 +120,11 @@ def wrapper( result = collector.result data = None - if data is None and not result.status: + if data is None and result.status in {ExecutionStatus.OK, ExecutionStatus.UNSET}: + # If the collector doesn't set the result status but no data was collected, mark it as a failure + # If a collector does not want to return data it should be under NOT_RAN, OK must return a data model. + if result.message == "": + result.message = "Data Model was not collected and collection failed without specific error message" result.status = ExecutionStatus.EXECUTION_FAILURE result.finalize(collector.logger) @@ -156,15 +162,22 @@ def __init__( task_result_hooks: Optional[list[TaskResultHook]] = None, event_reporter: str = DEFAULT_EVENT_REPORTER, session_id: Optional[str] = None, + log_path: Optional[str] = None, **kwargs, ): """data collector init function Args: system_info (SystemInfo): system info object for target system for data collection - system_interaction (SystemInteraction): enum to indicate the type of actions that can be performed when interacting with the system - event_reporter (str, optional): Reporter string stored on emitted events. Defaults to DEFAULT_EVENT_REPORTER. + connection (TConnection): connection object for the data collector logger (Optional[logging.Logger], optional): python logger object. Defaults to None. + system_interaction_level (SystemInteractionLevel | str): The interaction level which the collector will use + determine which commands it can run and how invasive the interactions can be when running those commands.Defaults to SystemInteractionLevel.INTERACTIVE. + max_event_priority_level (Union[EventPriority, str], optional): priority limit for events. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): parent task identifier. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook]], optional): list of task result hooks. Defaults to None. + event_reporter (str, optional): Reporter string stored on emitted events. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): session identifier. Defaults to None. log_path (Optional[str], optional): file system log path. Defaults to None. """ super().__init__( @@ -176,13 +189,13 @@ def __init__( event_reporter=event_reporter, session_id=session_id, ) - if isinstance(system_interaction_level, str): system_interaction_level = getattr(SystemInteractionLevel, system_interaction_level) - self.system_interaction_level = system_interaction_level - self.connection = connection - self._html_view = False + self.log_path: str | None = log_path + self.system_interaction_level: SystemInteractionLevel = system_interaction_level + self.connection: TConnection = connection + self._html_view: bool = False allowed_skus = _supported_sku_name_set(self.SUPPORTED_SKUS) if ( @@ -230,6 +243,10 @@ def collect_data( ) -> tuple[TaskResult, Optional[TDataModel]]: """Collect data from a target system + Args: + args (Optional[TCollectArg], optional): collection arguments for this data collection run. Defaults to None. + Returns: - tuple[TaskResult, DataModel]: tuple containing result and data model + tuple[TaskResult, Optional[TDataModel]]: tuple containing result and data model. No DataModel for collection failure cases therefore + it is optional. """ diff --git a/nodescraper/interfaces/dataplugin.py b/nodescraper/interfaces/dataplugin.py index 1b18df28..758d5424 100644 --- a/nodescraper/interfaces/dataplugin.py +++ b/nodescraper/interfaces/dataplugin.py @@ -46,7 +46,8 @@ SystemInfo, TaskResult, ) -from nodescraper.utils import pascal_to_snake +from nodescraper.taskresulthooks.filesystemloghook import hooks_for_fixed_directory +from nodescraper.utils import resolve_log_dir_name from .connectionmanager import TConnectArg, TConnectionManager from .task import SystemCompatibilityError @@ -63,6 +64,9 @@ dict[str, Type[CollectorArgs]], ] +NO_TASK_STR = "No {task_name} configured for {plugin_name}" +TASK_NOT_RAN_STR = "Data {task_name} not ran for {plugin_name}" + class DataPlugin( PluginInterface, @@ -94,6 +98,19 @@ def __init__( session_id: Optional[str] = None, **kwargs, ): + """_summary_ + + Args: + system_info (Optional[SystemInfo], optional): system info object. Defaults to None. + logger (Optional[logging.Logger], optional): python logger instance. Defaults to None. + connection_manager (Optional[TConnectionManager], optional): connection manager instance. Defaults to None. + connection_args (Optional[Union[TConnectArg , dict]], optional): connection args. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook]], optional): list of task result hooks. Defaults to None. + log_path (Optional[str], optional): path for file system logs. Defaults to None. + event_reporter (str, optional): Reporter string stored on emitted events. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): session identifier. Defaults to None. + kwargs (optional): additional keyword arguments. These given to the baseclass as well. + """ super().__init__( system_info, logger, @@ -106,13 +123,26 @@ def __init__( **kwargs, ) self._validate_class_var() + collector = self.get_collector_classes() self.collection_result: TaskResult = TaskResult( status=ExecutionStatus.NOT_RAN, - message=f"Data collection not ran for {self.__class__.__name__}", + message=( + TASK_NOT_RAN_STR.format(task_name="collection", plugin_name=self.__class__.__name__) + if collector + else NO_TASK_STR.format(task_name="collection", plugin_name=self.__class__.__name__) + ), + parent=self.__class__.__name__, + task=collector[0].__name__ if collector else None, ) self.analysis_result: TaskResult = TaskResult( status=ExecutionStatus.NOT_RAN, - message=f"Data analysis not ran for {self.__class__.__name__}", + message=( + TASK_NOT_RAN_STR.format(task_name="analysis", plugin_name=self.__class__.__name__) + if self.ANALYZER is not None + else NO_TASK_STR.format(task_name="analysis", plugin_name=self.__class__.__name__) + ), + parent=self.__class__.__name__, + task=self.ANALYZER.__name__ if self.ANALYZER is not None else None, ) self._data: Optional[TDataModel] = None @@ -126,6 +156,23 @@ def get_collector_classes(cls) -> tuple[Type[DataCollector], ...]: return tuple(collector) return (collector,) + def _connection_manager_hooks(self) -> list[TaskResultHook]: + """Hooks for a connection manager created by this plugin. + + Its result is saved in ``///connection`` so it is kept + apart from the collector's own result and events files. + """ + if not self.log_path: + return self.task_result_hooks + parts = [resolve_log_dir_name(self.__class__.__name__)] + collectors = self.get_collector_classes() + if collectors: + parts.append(resolve_log_dir_name(collectors[0].__name__)) + parts.append("connection") + return hooks_for_fixed_directory( + self.task_result_hooks, str(Path(self.log_path).joinpath(*parts)) + ) + @classmethod def _collector_args_class( cls, collector_cls: Type[DataCollector] @@ -279,7 +326,7 @@ def data(self, data: Optional[Union[str, dict, TDataModel]]): if isinstance(data, (str, dict)): self._data = self.DATA_MODEL.import_model(data) elif not isinstance(data, self.DATA_MODEL): - raise ValueError(f"data is invalid type, expected {self.DATA_MODEL.__class__.__name__}") + raise ValueError(f"data is invalid type, expected {self.DATA_MODEL.__name__}") else: self._data = data @@ -289,15 +336,22 @@ def collect( system_interaction_level: Optional[ Union[SystemInteractionLevel, str] ] = SystemInteractionLevel.INTERACTIVE, - preserve_connection: bool = False, + preserve_connection: bool = True, collection_args: Optional[TCollectArg] = None, ) -> TaskResult: """Run data collector task + A connection manager created by this plugin is always closed (and its result saved) when + collection finishes. A connection manager passed in by the caller is left open by default + so the caller, who owns it, can reuse it across plugins and close it when done; pass + ``preserve_connection=False`` to have the plugin close it after this collection. + Args: max_event_priority_level (Union[EventPriority, str], optional): priority limit for events. Defaults to EventPriority.CRITICAL. system_interaction_level (Union[SystemInteractionLevel, str], optional): system interaction level. Defaults to SystemInteractionLevel.INTERACTIVE. - preserve_connection (bool, optional): whether we should close the connection after data collection. Defaults to False. + preserve_connection (bool, optional): keep a connection manager passed in by the caller open after + collection. Set False to close it. Has no effect on a manager the plugin created itself, + which is always closed. Defaults to True. collection_args (Optional[TCollectArg], optional): args for data collection (validated model). Defaults to None. Returns: @@ -317,27 +371,37 @@ def collect( try: if not self.connection_manager: if not self.CONNECTION_TYPE: - self.collection_result = TaskResult( + return TaskResult( task=primary_collector.__name__, parent=self.__class__.__name__, status=ExecutionStatus.NOT_RAN, message=f"No connection manager type provided for {self.__class__.__name__}", ) - return self.collection_result self.logger.info("No connection manager provide, initializing connection manager") + self._owns_connection_manager = True self.connection_manager = self.CONNECTION_TYPE( system_info=self.system_info.model_copy(), logger=self.logger, parent=self.__class__.__name__, - task_result_hooks=self.task_result_hooks, + task_result_hooks=self._connection_manager_hooks(), event_reporter=self.event_reporter, session_id=self.session_id, ) + # Connect if never attempted or previously connected then disconnected; + # If the connection instance has previously been known to be in a failed Status + # then it should not attempt to reconnect automatically. if ( not self.connection_manager.connection - and self.connection_manager.result.status == ExecutionStatus.UNSET + and self.connection_manager.result.status + in ( + ExecutionStatus.UNSET, + ExecutionStatus.OK, + ExecutionStatus.WARNING, + ) ): + # Set the parent to us + self.connection_manager.parent = self.__class__.__name__ self.connection_manager.connect() # Proceed as long as a connection was established. @@ -357,7 +421,17 @@ def collect( for collector_cls in collector_classes: collector_args = self._resolve_collector_args(collector_cls, collection_args) - collection_task = collector_cls( + collector_log_path = None + if self.log_path: + # Place the log path of the connection in the collector's directory + _log_path = ( + Path(self.log_path) + / resolve_log_dir_name(self.__class__.__name__) + / resolve_log_dir_name(collector_cls.__name__) + ) + _log_path.mkdir(parents=True, exist_ok=True) + collector_log_path = str(_log_path) + collection_task: DataCollector[Any, Any, Any] = collector_cls( system_info=self.system_info.model_copy(), logger=self.logger, system_interaction_level=system_interaction_level, @@ -365,7 +439,7 @@ def collect( max_event_priority_level=max_event_priority_level, parent=self.__class__.__name__, task_result_hooks=self.task_result_hooks, - log_path=self.log_path, + log_path=collector_log_path, event_reporter=self.event_reporter, session_id=self.session_id, ) @@ -398,8 +472,10 @@ def collect( message=f"Unhandled exception running data collector: {str(e)}", ) finally: - if not preserve_connection and self.connection_manager: - self.connection_manager.disconnect() + if self.connection_manager and ( + self._owns_connection_manager or not preserve_connection + ): + self.connection_manager.safe_disconnect() return self.collection_result @@ -460,6 +536,15 @@ def analyze( self.analysis_result = analyzer_task.analyze_data(self.data, analysis_args) return self.analysis_result + def _get_connection_result(self) -> Optional[TaskResult]: + """Snapshot of the connection result, or None if no connection was initiated.""" + if self.connection_manager is None: + return None + result = self.connection_manager.result + if result is None or result.status == ExecutionStatus.UNSET: + return None + return result.model_copy(deep=True) + def run( self, collection: Annotated[ @@ -475,7 +560,7 @@ def run( Union[SystemInteractionLevel, str], "System interaction level (e.g. PASSIVE, INTERACTIVE, DISRUPTIVE).", ] = SystemInteractionLevel.INTERACTIVE, - preserve_connection: bool = False, + preserve_connection: bool = True, data: Annotated[ Optional[Union[str, dict, TDataModel]], Field( @@ -495,7 +580,8 @@ def run( analysis (bool, optional): Enable data analysis. Defaults to True. max_event_priority_level (Union[EventPriority, str], optional): Max priority level to assign to events. Defaults to EventPriority.CRITICAL. system_interaction_level (Union[SystemInteractionLevel, str], optional): System interaction level. Defaults to SystemInteractionLevel.INTERACTIVE. - preserve_connection (bool, optional): Whether to close the connection when data collection is complete. Defaults to False. + preserve_connection (bool, optional): Keep a connection manager passed in by the caller open after collection. + Set False to close it. A manager the plugin created itself is always closed. Defaults to True. data (Optional[Union[str, dict, TDataModel]], optional): Input data. Defaults to None. collection_args (Optional[Union[TCollectArg , dict]], optional): Arguments for data collection. Defaults to None. analysis_args (Optional[Union[TAnalyzeArg , dict]], optional): Arguments for data analysis. Defaults to None. @@ -508,8 +594,8 @@ def run( self.collect( max_event_priority_level=max_event_priority_level, system_interaction_level=system_interaction_level, - collection_args=collection_args, preserve_connection=preserve_connection, + collection_args=collection_args, ) if analysis: @@ -520,6 +606,10 @@ def run( ) status = max(self.collection_result.status, self.analysis_result.status) + collection_message = self.collection_result.message or "" + show_targets = bool( + self.collection_result.details.get("per_target_messages") and collection_message + ) message = "" if status == ExecutionStatus.NOT_RAN: @@ -530,11 +620,19 @@ def run( ExecutionStatus.WARNING, ]: failure_parts: list[str] = [] + if show_targets and self.collection_result.status == ExecutionStatus.OK: + failure_parts.append(collection_message) for label, task_result in ( ("Collection", self.collection_result), ("Analysis", self.analysis_result), ): - if task_result.status == ExecutionStatus.WARNING: + if ( + show_targets + and label == "Collection" + and task_result.status == ExecutionStatus.WARNING + ): + failure_parts.append(collection_message) + elif task_result.status == ExecutionStatus.WARNING: failure_parts.append(f"{label} warning: {task_result.message}") elif task_result.status in ( ExecutionStatus.ERROR, @@ -542,6 +640,8 @@ def run( ): failure_parts.append(f"{label} error: {task_result.message}") message = "; ".join(failure_parts) + elif show_targets: + message = collection_message else: message = "Plugin tasks completed successfully" @@ -553,6 +653,7 @@ def run( system_data=self.data, collection_result=self.collection_result, analysis_result=self.analysis_result, + connection_result=self._get_connection_result(), ), ) @@ -572,30 +673,64 @@ def find_datamodel_path_in_run(cls, run_path: str) -> Optional[str]: data_model_cls = getattr(cls, "DATA_MODEL", None) if not data_model_cls: return None + plugin_dir = resolve_log_dir_name(cls.__name__) + plugin_roots = [ + entry + for entry in os.listdir(run_path) + if entry == plugin_dir or entry.startswith(plugin_dir + "[") + ] for collector_cls in cls.get_collector_classes(): - collector_dir = os.path.join( - run_path, - pascal_to_snake(cls.__name__), - pascal_to_snake(collector_cls.__name__), - ) - if not os.path.isdir(collector_dir): - continue - result_path = os.path.join(collector_dir, "result.json") + collector_name = resolve_log_dir_name(collector_cls.__name__) + for plugin_root in plugin_roots: + collector_dir = os.path.join(run_path, plugin_root, collector_name) + found = cls._datamodel_file_in_collector_tree( + collector_dir, data_model_cls.__name__ + ) + if found: + return found + return None + + @classmethod + def _datamodel_file_in_collector_tree( + cls, collector_dir: str, data_model_name: str + ) -> Optional[str]: + """Return a datamodel file in a collector directory or one target subdirectory. + + Args: + collector_dir: Plugin collector directory, which may contain target folders. + data_model_name: DATA_MODEL class name used to match the JSON file. + + Returns: + Optional[str]: Absolute path to the datamodel file, or None. + """ + if not os.path.isdir(collector_dir): + return None + search_dirs = [collector_dir] + search_dirs.extend( + os.path.join(collector_dir, child) + for child in sorted(os.listdir(collector_dir)) + if os.path.isdir(os.path.join(collector_dir, child)) + ) + want_json = data_model_name.lower() + ".json" + for search_dir in search_dirs: + result_path = os.path.join(search_dir, "result.json") if not os.path.isfile(result_path): continue - try: - res_payload = json.loads(Path(result_path).read_text(encoding="utf-8")) - if res_payload.get("parent") != cls.__name__: + if search_dir == collector_dir: + try: + res_payload = json.loads(Path(result_path).read_text(encoding="utf-8")) + parent = res_payload.get("parent") or "" + if parent != cls.__name__ and not str(parent).startswith(cls.__name__ + "["): + continue + except (json.JSONDecodeError, OSError): continue - except (json.JSONDecodeError, OSError): - continue - want_json = data_model_cls.__name__.lower() + ".json" - for fname in os.listdir(collector_dir): + for fname in os.listdir(search_dir): low = fname.lower() - if low.endswith("datamodel.json") or low == want_json: - return os.path.join(collector_dir, fname) - if low.endswith(".log"): - return os.path.join(collector_dir, fname) + if low.endswith(f"{data_model_name.lower()}.json") or low == want_json: + return os.path.join(search_dir, fname) + for fname in os.listdir(search_dir): + if fname.lower().endswith(".log"): + return os.path.join(search_dir, fname) return None @classmethod diff --git a/nodescraper/interfaces/plugin.py b/nodescraper/interfaces/plugin.py index 9fe998d9..22658ef4 100644 --- a/nodescraper/interfaces/plugin.py +++ b/nodescraper/interfaces/plugin.py @@ -64,6 +64,9 @@ def __init__( task_result_hooks (Optional[list[TaskResultHook]], optional): list of task result hooks. Defaults to None. log_path (Optional[str], optional): path for file system logs. Defaults to None. queue_callback (Optional[Callable], optional): function to add additional plugins to plugin executor queue. Defaults to None. + event_reporter (str, optional): Reporter string stored on emitted events. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): session identifier. Defaults to None. + kwargs (optional): additional keyword arguments. These are currently not used by the base plugin interface. """ if logger is None: logger = logging.getLogger(DEFAULT_LOGGER) @@ -71,15 +74,15 @@ def __init__( if system_info is None: system_info = SystemInfo() - self.system_info = system_info + self.system_info: SystemInfo = system_info - if not task_result_hooks: - task_result_hooks = [] - self.task_result_hooks = task_result_hooks + # copy the hook list so that hooks added here are not leaked back to the caller + # If the copy is not performed then any modifications to the hook list here would affect the caller's list as well. + self.task_result_hooks = list(task_result_hooks) if task_result_hooks else [] if log_path: for hook in self.task_result_hooks: - if isinstance(hook, FileSystemLogHook): + if isinstance(hook, FileSystemLogHook) and hook.log_base_path == log_path: break else: self.task_result_hooks.append(FileSystemLogHook(log_base_path=log_path)) @@ -92,18 +95,26 @@ def __init__( self.session_id = session_id self.connection_manager = connection_manager + # True only when this plugin built its connection manager itself; a manager passed in + # belongs to the caller, who is responsible for closing it. + self._owns_connection_manager = False if connection_args and self.CONNECTION_TYPE and not self.connection_manager: + self._owns_connection_manager = True self.connection_manager = self.CONNECTION_TYPE( system_info=self.system_info.model_copy(), logger=logger, connection_args=connection_args, parent=self.__class__.__name__, - task_result_hooks=self.task_result_hooks, + task_result_hooks=self._connection_manager_hooks(), event_reporter=event_reporter, session_id=self.session_id, ) + def _connection_manager_hooks(self) -> list: + """Result hooks for a connection manager this plugin creates itself.""" + return self.task_result_hooks + @classmethod def is_valid(cls) -> bool: """Check if plugin class is valid and can be instantiated diff --git a/nodescraper/interfaces/resultcollator.py b/nodescraper/interfaces/resultcollator.py index b61fe313..77cb1e45 100644 --- a/nodescraper/interfaces/resultcollator.py +++ b/nodescraper/interfaces/resultcollator.py @@ -25,7 +25,8 @@ ############################################################################### import abc import logging -from typing import Optional +from logging import Logger +from typing import Optional, Union from nodescraper.constants import DEFAULT_LOGGER from nodescraper.models import PluginResult, TaskResult @@ -39,14 +40,23 @@ def __init__( logger: Optional[logging.Logger] = None, log_path: Optional[str] = None, ): + """Creates a PluginResultCollator instance. + + Args: + logger (Optional[logging.Logger], optional): logger instance for the collator. Defaults to None. + log_path (Optional[str], optional): path to the log file. Defaults to None. + """ if logger is None: logger = logging.getLogger(DEFAULT_LOGGER) - self.logger = logger - self.log_path = log_path + self.logger: Logger = logger + self.log_path: Union[str, None] = log_path @abc.abstractmethod def collate_results( - self, plugin_results: list[PluginResult], connection_results: list[TaskResult], **kwargs + self, + plugin_results: list[PluginResult], + connection_results: list[TaskResult], + **kwargs, ): """Function to process the result of a plugin executor run diff --git a/nodescraper/interfaces/task.py b/nodescraper/interfaces/task.py index aa19d56f..557c3034 100644 --- a/nodescraper/interfaces/task.py +++ b/nodescraper/interfaces/task.py @@ -26,6 +26,7 @@ import abc import copy import datetime +import inspect import logging import uuid from typing import Any, Optional, Union @@ -59,6 +60,20 @@ def __init__( session_id: Optional[str] = None, **kwargs: dict[str, Any], ): + """Creates a Task instance. + + Args: + system_info (SystemInfo): system info object for target system for data collection + logger (Optional[logging.Logger], optional): python logger object. Defaults to None. + max_event_priority_level (Union[EventPriority, str], optional): priority limit for events. Defaults to EventPriority.CRITICAL. + parent (Optional[str], optional): parent task identifier. Defaults to None. + task_result_hooks (Optional[list[TaskResultHook]], optional): list of task result hooks. Defaults to None. + event_reporter (str, optional): Reporter string stored on emitted events. Defaults to DEFAULT_EVENT_REPORTER. + session_id (Optional[str], optional): session identifier. Defaults to None. + + Raises: + ValueError: Will raise a ValueError when the session_id is not a Valid UUID string. + """ if logger is None: logger = logging.getLogger(DEFAULT_LOGGER) self.system_info = system_info @@ -106,8 +121,11 @@ def max_event_priority_level(self, input_value: Union[str, EventPriority]): self._max_event_priority_level = value def __init_subclass__(cls, **kwargs) -> None: + """Validates that the subclass contains a TASK_TYPE attribute which is not None.""" super().__init_subclass__(**kwargs) - if cls.TASK_TYPE is None: + if not inspect.isabstract(cls) and ( + (getattr(cls, "TASK_TYPE", None) is None) or (cls.TASK_TYPE is None) + ): raise TypeError(f"No value provided for TASK_TYPE in task class {cls.__name__}") def _build_event( @@ -118,13 +136,27 @@ def _build_event( data: Optional[dict] = None, timestamp: Optional[datetime.datetime] = None, ) -> Event: + """This will build an event + + Args: + category (Union[EventCategory, str]): The category of the event. + description (str): The description of the event, typically a human-readable message. + priority (EventPriority): The priority level of the event. + data (Optional[dict], optional): Additional data associated with the event. Defaults to None. + timestamp (Optional[datetime.datetime], optional): The timestamp of the event. Defaults to None. + Returns: + Event: The constructed event object. + """ if data is None: - data = {"task_name": self.__class__.__name__, "task_type": self.TASK_TYPE} + data: dict[Any, Any] = { + "task_name": self.__class__.__name__, + "task_type": self.TASK_TYPE, + } else: # Copy to avoid mutating the caller's dict - data = copy.copy(data) + data: dict[Any, Any] = copy.copy(data) data["task_name"] = self.__class__.__name__ data["task_type"] = self.TASK_TYPE @@ -162,6 +194,16 @@ def _log_event( timestamp: Optional[datetime.datetime] = None, console_log: bool = False, ): + """Log an Event + + Args: + category (Union[EventCategory, str]): The category of the event. + description (str): The description of the event, typically a human-readable message. + priority (EventPriority): The priority level of the event. + data (Optional[dict], optional): Additional data associated with the event. Defaults to None. + timestamp (Optional[datetime.datetime], optional): The timestamp of the event. Defaults to None. + console_log (bool, optional): Whether to also log the event to the console. Defaults to False. + """ event = self._build_event( category=category, description=description, diff --git a/nodescraper/models/__init__.py b/nodescraper/models/__init__.py index 6b7ebb00..cde27594 100644 --- a/nodescraper/models/__init__.py +++ b/nodescraper/models/__init__.py @@ -30,6 +30,8 @@ from .event import Event from .pluginconfig import PluginConfig from .pluginresult import PluginResult +from .postactioncondition import PostActionCondition +from .postactionpluginconfig import PostActionPluginConfig from .priority_override import ( NO_CHANGE, PriorityOverrideRule, @@ -51,6 +53,8 @@ "PluginResult", "DataPluginResult", "PluginConfig", + "PostActionCondition", + "PostActionPluginConfig", "NO_CHANGE", "PriorityOverrideRule", "apply_priority_override_rules", diff --git a/nodescraper/models/collectorargs.py b/nodescraper/models/collectorargs.py index 02ba532f..56447c5d 100644 --- a/nodescraper/models/collectorargs.py +++ b/nodescraper/models/collectorargs.py @@ -23,10 +23,14 @@ # SOFTWARE. # ############################################################################### -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field class CollectorArgs(BaseModel): + """The base for all collector args in node-scraper. By default disallow extra args, they will raise Validation Error.""" + + model_config = ConfigDict(extra="forbid") + html_view: bool = Field( default=False, description=( @@ -34,5 +38,3 @@ class CollectorArgs(BaseModel): "using human-readable output." ), ) - - model_config = {"extra": "forbid", "exclude_none": True} diff --git a/nodescraper/models/datamodel.py b/nodescraper/models/datamodel.py index c310c810..1d6c0826 100644 --- a/nodescraper/models/datamodel.py +++ b/nodescraper/models/datamodel.py @@ -24,12 +24,11 @@ # ############################################################################### import io -import json import os import tarfile -from typing import TypeVar, Union +from typing import Any, TypeVar, Union -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator from nodescraper.utils import get_unique_filename @@ -37,16 +36,24 @@ class FileModel(BaseModel): + """Binary file payload. JSON uses URL-safe base64 for ``file_contents``.""" + + model_config = ConfigDict(ser_json_bytes="base64", val_json_bytes="base64") + file_contents: bytes file_name: str @field_validator("file_contents", mode="before") @classmethod - def file_contents_conformer(cls, value: Union[io.BytesIO, str, bytes]) -> bytes: + def file_contents_conformer( + cls, value: Union[io.BytesIO, str, bytes], info: ValidationInfo + ) -> Union[str, bytes]: if isinstance(value, io.BytesIO): return value.getvalue() - if isinstance(value, str): + if isinstance(value, str) and info.mode != "json": + # Constructor/Python input is text. return value.encode("utf-8") + # JSON strings stay base64 for val_json_bytes. return value def log_model(self, log_path: str) -> None: @@ -77,22 +84,28 @@ def log_model(self, log_path: str): get_unique_filename(log_path, f"{self.__class__.__name__.lower()}.json"), ) - exlude_fields = set() - for key in self.model_fields: + # Write binary FileModel payloads as files; keep them in JSON as base64. + for key in self.__class__.model_fields: data = getattr(self, key) if isinstance(data, FileModel): data.log_model(log_path) - exlude_fields.add(key) + elif ( + isinstance(data, list) + and data + and all(isinstance(item, FileModel) for item in data) + ): + for item in data: + item.log_model(log_path) with open(log_name, "w", encoding="utf-8") as log_file: - log_file.write(self.model_dump_json(indent=2, exclude=exlude_fields)) + log_file.write(self.model_dump_json(indent=2)) def merge_data(self, input_data: "DataModel") -> None: """Merge data into current data""" pass @classmethod - def import_model(cls: type[TDataModel], model_input: Union[dict, str]) -> TDataModel: + def import_model(cls: type[TDataModel], model_input: Union[dict[str, Any], str]) -> TDataModel: """import a data model if the input is a string attempt to read data from file using the string as a file name if input is a dict, pass key value pairs directly to init function @@ -100,7 +113,7 @@ def import_model(cls: type[TDataModel], model_input: Union[dict, str]) -> TDataM Args: cls (type[DataModel]): Data model class - model_input (Union[dict, str]): model data input + model_input (Union[dict[str, Any], str]): model data input Raises: ValueError: if model_input has an invalid type @@ -113,18 +126,16 @@ def import_model(cls: type[TDataModel], model_input: Union[dict, str]) -> TDataM return cls(**model_input) if isinstance(model_input, str): - # Build from tarfile if supported - if tarfile.is_tarfile(model_input): - return cls.build_from_tar(model_input) # Build from folder if supported if os.path.isdir(model_input): return cls.build_from_folder(model_input) - + # Build from tarfile if supported + elif tarfile.is_tarfile(model_input): + return cls.build_from_tar(model_input) # Build from json file - with open(model_input, "r", encoding="utf-8") as input_file: - data = json.load(input_file) - - return cls(**data) + else: + with open(model_input, "r", encoding="utf-8") as input_file: + return cls.model_validate_json(input_file.read()) raise ValueError("Invalid input for model data") diff --git a/nodescraper/models/datapluginresult.py b/nodescraper/models/datapluginresult.py index 2ae73742..2f990d63 100644 --- a/nodescraper/models/datapluginresult.py +++ b/nodescraper/models/datapluginresult.py @@ -37,3 +37,4 @@ class DataPluginResult(BaseModel): system_data: Optional[DataModel] = None collection_result: Optional[TaskResult] = None analysis_result: Optional[TaskResult] = None + connection_result: Optional[TaskResult] = None diff --git a/nodescraper/models/event.py b/nodescraper/models/event.py index de2ecc64..7e3851db 100644 --- a/nodescraper/models/event.py +++ b/nodescraper/models/event.py @@ -31,24 +31,15 @@ from typing import Any, Optional, Union, cast from pydantic import BaseModel, Field, field_serializer, field_validator +from pydantic_core import to_jsonable_python from nodescraper.constants import DEFAULT_EVENT_REPORTER from nodescraper.enums import EventPriority def _data_to_json_safe(obj: Any) -> Any: - """Recursively convert event data to JSON-serializable form (e.g. exceptions -> str).""" - if isinstance(obj, BaseException): - return str(obj) - if isinstance(obj, dict): - return {k: _data_to_json_safe(v) for k, v in obj.items()} - if isinstance(obj, (list, tuple)): - return [_data_to_json_safe(v) for v in obj] - if isinstance(obj, (str, int, float, bool, type(None))): - return obj - if isinstance(obj, (Enum, datetime.datetime, uuid.UUID)): - return str(obj) - return str(obj) + """Convert data to JSON-serializable form using Pydantic's fast Rust core.""" + return to_jsonable_python(obj, fallback=lambda x: str(x)) LOG_LEVEL_MAP = { @@ -178,7 +169,9 @@ def validate_data(cls, data: dict) -> dict: Returns: dict: data output """ - if len(str(data).encode("utf-8")) >= (1024 * 100): + # Use the same serialization that will happen during model_dump() + json_safe_data = _data_to_json_safe(data) + if len(str(json_safe_data).encode("utf-8")) >= (1024 * 100): raise ValueError("Data must be below 100KB in size") return data diff --git a/nodescraper/models/pluginconfig.py b/nodescraper/models/pluginconfig.py index a060530f..76618898 100644 --- a/nodescraper/models/pluginconfig.py +++ b/nodescraper/models/pluginconfig.py @@ -29,6 +29,8 @@ from pydantic import BaseModel, Field +from nodescraper.models.postactionpluginconfig import PostActionPluginConfig + class PluginConfig(BaseModel): """Model for preset configuration of plugins and result collators""" @@ -36,6 +38,7 @@ class PluginConfig(BaseModel): global_args: dict = Field(default_factory=dict) plugins: dict[str, dict] = Field(default_factory=dict) result_collators: dict[str, dict] = Field(default_factory=dict) + post_action_plugins: list[PostActionPluginConfig] = Field(default_factory=list) name: Optional[str] = None desc: Optional[str] = None @@ -51,13 +54,16 @@ def merge(cls, *configs: PluginConfig | dict[str, Any]) -> PluginConfig: """Merge recipe plugin configs. Plugin entries from later configs overwrite earlier ones with the same name. + ``post_action_plugins`` are concatenated from all configs in order. ``name``, ``desc``, ``global_args``, and ``result_collators`` come from the first config. """ normalized = [cls.coerce(config) for config in configs] merged_plugins: dict[str, dict[str, Any]] = {} + merged_post_actions = [] for config in normalized: merged_plugins.update(config.plugins) + merged_post_actions.extend(config.post_action_plugins) first = normalized[0] if normalized else cls() return cls( name=first.name, @@ -65,4 +71,5 @@ def merge(cls, *configs: PluginConfig | dict[str, Any]) -> PluginConfig: global_args=first.global_args, plugins=merged_plugins, result_collators=first.result_collators, + post_action_plugins=merged_post_actions, ) diff --git a/nodescraper/models/postactioncondition.py b/nodescraper/models/postactioncondition.py new file mode 100644 index 00000000..da2bed63 --- /dev/null +++ b/nodescraper/models/postactioncondition.py @@ -0,0 +1,173 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any, Optional + +from pydantic import BaseModel, field_validator + +from nodescraper.enums import EventPriority, ExecutionStatus + +if TYPE_CHECKING: + from nodescraper.models.event import Event + from nodescraper.models.pluginresult import PluginResult + + +class PostActionCondition(BaseModel): + """A single condition that, if matched, causes a post-action plugin to run. + + All specified (non-None) fields are AND'd together within one condition. + Unspecified fields are ignored and never prevent a match. A list of + ``PostActionCondition`` objects is OR'd by the containing + :class:`PostActionPluginConfig`. + """ + + plugin: Optional[str] = None + """If set, only inspect the PluginResult whose ``source`` matches this name. + If None, all results are candidates.""" + + status: Optional[ExecutionStatus] = None + """If set, the result's ExecutionStatus must be >= this value. + Accepts an :class:`~nodescraper.enums.ExecutionStatus` member or its name as + a string (e.g. ``"WARNING"``, ``"ERROR"``, ``"EXECUTION_FAILURE"``).""" + + event_category: Optional[str] = None + """If set, at least one event from analysis_result or collection_result must + have a category equal to this value (matched after the same normalisation + applied to event categories: strip, upper, spaces/hyphens → underscores).""" + + event_priority: Optional[EventPriority] = None + """If set, at least one event's priority must be >= this value. + Accepts an :class:`~nodescraper.enums.EventPriority` member or its name as + a string (e.g. ``"WARNING"``, ``"ERROR"``, ``"CRITICAL"``).""" + + event_description_contains: Optional[str] = None + """If set, at least one event's description must contain this substring + (case-sensitive).""" + + # ------------------------------------------------------------------ + # Field validators — allow string names from JSON configs + # ------------------------------------------------------------------ + + @field_validator("status", mode="before") + @classmethod + def validate_status(cls, v: Any) -> Optional[ExecutionStatus]: + """Accept an ExecutionStatus member or its name string.""" + if v is None or isinstance(v, ExecutionStatus): + return v + if isinstance(v, str): + try: + return ExecutionStatus[v.upper()] + except KeyError as e: + raise ValueError(f"Unknown ExecutionStatus name: {v!r}") from e + return v + + @field_validator("event_priority", mode="before") + @classmethod + def validate_event_priority(cls, v: Any) -> Optional[EventPriority]: + """Accept an EventPriority member or its name string.""" + if v is None or isinstance(v, EventPriority): + return v + if isinstance(v, str): + try: + return EventPriority[v.upper()] + except KeyError as e: + raise ValueError(f"Unknown EventPriority name: {v!r}") from e + return v + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _normalise_category(raw: str) -> str: + """Apply the same normalisation used by :class:`~nodescraper.models.event.Event`.""" + normalised = str(raw).strip().upper() + return re.sub(r"[\s-]", "_", normalised) + + def _get_all_events(self, result: PluginResult) -> list[Event]: + """Collect events from both collection and analysis task results.""" + events: list[Event] = [] + rd = result.result_data + if rd is None: + return events + if hasattr(rd, "collection_result") and rd.collection_result is not None: + events.extend(rd.collection_result.events) + if hasattr(rd, "analysis_result") and rd.analysis_result is not None: + events.extend(rd.analysis_result.events) + return events + + def _matches_result(self, result: PluginResult) -> bool: + """Return True if *result* satisfies all specified fields (AND logic). + + Each field that is not None must be satisfied; unset fields are skipped. + """ + # --- status check --- + if self.status is not None: + if result.status < self.status: + return False + + # Remaining checks all operate on events; collect them once. + events = self._get_all_events(result) + + # --- event_category check --- + if self.event_category is not None: + normalised = self._normalise_category(self.event_category) + if not any(e.category == normalised for e in events): + return False + + # --- event_priority check --- + if self.event_priority is not None: + if not any(e.priority >= self.event_priority for e in events): + return False + + # --- event_description_contains check --- + if self.event_description_contains is not None: + if not any(self.event_description_contains in e.description for e in events): + return False + + return True + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def is_met(self, plugin_results: list[PluginResult]) -> bool: + """Return True if this condition is satisfied by any of the provided results. + + If ``plugin`` is set only that plugin's result is checked; otherwise all + results are candidates. + + Args: + plugin_results: List of :class:`~nodescraper.models.pluginresult.PluginResult` + objects from the primary plugin run. + + Returns: + bool: True if at least one candidate result satisfies all specified fields. + """ + candidates = [r for r in plugin_results if self.plugin is None or r.source == self.plugin] + return any(self._matches_result(r) for r in candidates) diff --git a/nodescraper/models/postactionpluginconfig.py b/nodescraper/models/postactionpluginconfig.py new file mode 100644 index 00000000..87d3732f --- /dev/null +++ b/nodescraper/models/postactionpluginconfig.py @@ -0,0 +1,105 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pydantic import BaseModel, Field, field_validator + +from nodescraper.models.postactioncondition import PostActionCondition + +if TYPE_CHECKING: + from nodescraper.models.pluginresult import PluginResult + + +class PostActionPluginConfig(BaseModel): + """Configuration for a single post-action plugin. + + A post-action plugin runs after all primary plugins have completed, but only + if at least one condition in ``conditions`` is satisfied by the primary results + (OR semantics across conditions). + + The ``plugin`` and ``plugin_args`` fields mirror the structure of a regular + entry in :attr:`~nodescraper.models.PluginConfig.plugins` — the plugin is + looked up by name in the registry and run identically to a primary plugin. + + Example JSON config entry:: + + { + "plugin": "SomeRemediationPlugin", + "plugin_args": {"collection": true, "analysis": false}, + "conditions": [ + {"plugin": "DmesgPlugin", "status": "ERROR"}, + {"event_priority": "CRITICAL", "event_description_contains": "GPU reset"} + ] + } + """ + + plugin: str + """Name of the plugin to run — must be registered in the plugin registry. + Must be a non-empty, non-whitespace-only string.""" + + @field_validator("plugin") + @classmethod + def validate_plugin(cls, v: str) -> str: + """Reject empty or whitespace-only plugin names at construction time.""" + if not v or not v.strip(): + raise ValueError("plugin name must not be empty or whitespace-only") + return v + + plugin_args: dict = Field(default_factory=dict) + """Arguments forwarded verbatim to ``plugin.run()``. Same shape as entries + in :attr:`~nodescraper.models.PluginConfig.plugins`, e.g.:: + + { + "collection": True, + "analysis": False, + "collection_args": {"some_arg": "value"} + } + """ + + conditions: list[PostActionCondition] = Field(default_factory=list) + """List of conditions (OR'd). If any one condition is met by the primary + plugin results this post-action plugin will be executed.""" + + def should_run(self, plugin_results: list[PluginResult]) -> bool: + """Return True if at least one condition is satisfied by *plugin_results*. + + An empty ``conditions`` list is treated as *never run* (returns False), + which prevents post-action plugins from accidentally firing unconditionally + when a config omits conditions. + + Args: + plugin_results: The list of + :class:`~nodescraper.models.pluginresult.PluginResult` objects + produced by the primary plugin run. + + Returns: + bool: True if this post-action plugin should be executed. + """ + if not self.conditions: + return False + return any(condition.is_met(plugin_results) for condition in self.conditions) diff --git a/nodescraper/models/systeminfo.py b/nodescraper/models/systeminfo.py index d91a68cf..825276e7 100644 --- a/nodescraper/models/systeminfo.py +++ b/nodescraper/models/systeminfo.py @@ -41,6 +41,6 @@ class SystemInfo(BaseModel): platform: Optional[str] = None gpu_count: Optional[int] = None cpu_count: Optional[int] = None - metadata: Optional[dict] = Field(default_factory=dict) + metadata: dict = Field(default_factory=dict) location: Optional[SystemLocation] = SystemLocation.LOCAL vendorid_ep: int = 0x1002 diff --git a/nodescraper/models/taskresult.py b/nodescraper/models/taskresult.py index be406cfa..33fac415 100644 --- a/nodescraper/models/taskresult.py +++ b/nodescraper/models/taskresult.py @@ -162,13 +162,22 @@ def summary_str(self) -> str: """Message string for display.""" return self.message or "" - def log_result(self, log_path: str) -> None: - """Write result, artifacts, and events to log_path. Events are merged into a single events.json.""" + def log_result(self, log_path: str, file_suffix: str = "") -> None: + """Write result, artifacts, and events to log_path. + + Args: + log_path (str): Directory to write to. + file_suffix (str, optional): Appended to the ``result`` and ``events`` file names, e.g. + ``"(1)"`` writes ``result(1).json`` and ``events(1).json``. Events are merged into an + existing events file of the same name. Defaults to "". + """ from nodescraper.connection.inband import BaseFileArtifact os.makedirs(log_path, exist_ok=True) - with open(os.path.join(log_path, "result.json"), "w", encoding="utf-8") as log_file: + with open( + os.path.join(log_path, f"result{file_suffix}.json"), "w", encoding="utf-8" + ) as log_file: log_file.write(self.model_dump_json(exclude={"artifacts", "events"}, indent=2)) artifact_map: dict[str, list[dict[str, Any]]] = {} @@ -208,7 +217,7 @@ def log_result(self, log_path: str) -> None: html_file.write(render_command_artifacts_html(html_entries, title)) if self.events: - event_log = os.path.join(log_path, "events.json") + event_log = os.path.join(log_path, f"events{file_suffix}.json") new_events = [e.model_dump(mode="json", exclude_none=True) for e in self.events] existing_events = [] if os.path.isfile(event_log): @@ -235,40 +244,52 @@ def _event_occurrence_count(event: Event) -> int: return 1 return max(1, n) + @staticmethod + def _format_limited_descriptions(msg_counts: dict[str, int], prefix: str) -> str: + """Format the 3 most frequent descriptions, noting how many were left out.""" + items = sorted(msg_counts.items(), key=lambda kv: kv[1], reverse=True) + details = [f"{msg} (x{count})" if count > 1 else msg for msg, count in items[:3]] + summary = f"{prefix}: {', '.join(details)}" + omitted = len(items) - 3 + if omitted > 0: + summary += f" (omitted {omitted} descriptions)" + return summary + def _get_event_summary(self) -> str: """Get summary string for events Returns: - str: event summary with counts and descriptions + str: warning count, error count with the 3 most frequent error + descriptions, and the 3 most frequent critical descriptions """ + total_warnings = 0 + total_errors = 0 error_msg_counts: dict[str, int] = {} - warning_msg_counts: dict[str, int] = {} - + critical_msg_counts: dict[str, int] = {} for event in self.events: n = self._event_occurrence_count(event) - if event.priority == EventPriority.WARNING: - warning_msg_counts[event.description] = ( - warning_msg_counts.get(event.description, 0) + n + if event.priority == EventPriority.CRITICAL: + critical_msg_counts[event.description] = ( + critical_msg_counts.get(event.description, 0) + n ) - elif event.priority >= EventPriority.ERROR: + elif event.priority == EventPriority.ERROR: error_msg_counts[event.description] = error_msg_counts.get(event.description, 0) + n + if event.priority == EventPriority.WARNING: + total_warnings += n + elif event.priority >= EventPriority.ERROR: + total_errors += n summary_parts = [] - - if warning_msg_counts: - total_warnings = sum(warning_msg_counts.values()) - warning_details = [ - f"{msg} (x{count})" if count > 1 else msg - for msg, count in warning_msg_counts.items() - ] - summary_parts.append(f"{total_warnings} warnings: {', '.join(warning_details)}") - - if error_msg_counts: - total_errors = sum(error_msg_counts.values()) - error_details = [ - f"{msg} (x{count})" if count > 1 else msg for msg, count in error_msg_counts.items() - ] - summary_parts.append(f"{total_errors} errors: {', '.join(error_details)}") + if total_warnings: + summary_parts.append(f"{total_warnings} warnings") + if total_errors: + error_summary = f"{total_errors} errors" + if error_msg_counts: + error_summary = self._format_limited_descriptions(error_msg_counts, error_summary) + summary_parts.append(error_summary) + + if critical_msg_counts: + summary_parts.append(self._format_limited_descriptions(critical_msg_counts, "critical")) return "; ".join(summary_parts) diff --git a/nodescraper/pluginexecutor.py b/nodescraper/pluginexecutor.py index 772f3662..b79c9df6 100644 --- a/nodescraper/pluginexecutor.py +++ b/nodescraper/pluginexecutor.py @@ -28,24 +28,39 @@ import copy import inspect import logging +import os import uuid from collections import deque from collections.abc import Callable, Sequence -from typing import Optional, Type, Union +from typing import Any, Optional, Type, Union from pydantic import BaseModel from nodescraper.base.oobsshdataplugin import OOBSSHDataPlugin +from nodescraper.connection.inband import InBandConnectionManager +from nodescraper.connection.inband.osdetection import discover_and_write_os_family from nodescraper.connection.oob_ssh import OobSshConnectionManager +from nodescraper.connection.redfish import ( + RedfishConnectionManager, + RedfishSshProxyConnectionManager, +) from nodescraper.constants import DEFAULT_LOGGER +from nodescraper.enums import ExecutionStatus, OSFamily +from nodescraper.helpers.plugin_execution_target import ( + format_in_band_target_summary, +) from nodescraper.interfaces import ConnectionManager, DataPlugin, PluginInterface from nodescraper.interfaces.taskresulthook import TaskResultHook -from nodescraper.models import PluginConfig, SystemInfo +from nodescraper.models import PluginConfig, SystemInfo, TaskResult from nodescraper.models.pluginresult import PluginResult -from nodescraper.pluginregistry import PluginRegistry +from nodescraper.models.taskresult import STATUS_LOG_LEVEL_MAP +from nodescraper.pluginregistry import PluginRegistry, default_registry from nodescraper.taskresulthooks import FileSystemLogHook +from nodescraper.taskresulthooks.filesystemloghook import hooks_for_fixed_directory from nodescraper.typeutils import TypeUtils +OS_DISCOVERY_DIR = "os_discovery" + class PluginExecutor: """Class to manage execution of data collectors and error detectors""" @@ -61,12 +76,40 @@ def __init__( session_id: Optional[str] = None, plugin_run_result_hooks: Optional[Sequence[Callable[[PluginResult], None]]] = None, ): + """Initialize the PluginExecutor instance. + Args: + plugin_configs (list[PluginConfig]): This is a list of the PluginConfig Object, when this list is greater than a single PluginConfig + then the plugin_configs will be merged into a single PluginConfig. The single PluginConfig post merge will be used to control + all of the execution that is done by this executor. It will run all `plugins` defined in the merged PluginConfig. + connections (Optional[dict[str, Union[dict, BaseModel]]], optional): Connections is a dictionary where the + keys represent connection names and the values are either dictionaries or BaseModel instances containing the connection details + Optionally the user can provide just the dict[str, dict[Any,Any]] In this case the dictionary attributed to that connection name + will be transformed into a BaseModel instance and it will raise an error when this model fails to validate. + Any key in this dict will be promptly built using the args during `__init__` even if the connection is not used by any of the tasks + defined in the plugin_configs. If the connection isn't defined for a particular Plugin but that plugin requires the connection then + that connection will be built anyway but the arguments will not be provided which may lead to connection to not being properly established. + It is recommended that in-band connection arg should always be provided for remote connections. Defaults to None. + system_info (Optional[SystemInfo], optional): System information for the plugin executor, this is passed to the connection, and plugins, The executor will + only give out copies to other components. If the original system_info passed here is OSFamily.UNKOWN then the executor will attempt to detect the correct OS family + by starting a in-band connection and then determining the correct OS family. Defaults to None. + logger (Optional[logging.Logger], optional): Logger instance for the plugin executor. Defaults to None. + plugin_registry (Optional[PluginRegistry], optional): Plugin registry instance for the plugin executor, when this is None then the PluginRegistry will be + assigned a default `PluginRegistry()`. Defaults to None. + log_path (Optional[str], optional): Path to the log file for the plugin executor. When this is defined then the FileSystemLogHook will be automatically + added to the connection results hooks resulting in it logging to the defined folder. Defaults to None. + session_id (Optional[str], optional): Session identifier for the plugin executor. Defaults to None. + plugin_run_result_hooks (Optional[Sequence[Callable[[PluginResult], None]]], optional): Sequence of callables to be executed with the result of each plugin run. When this is + None then then this will be made empty list [] . Defaults to None. + + Raises: + ValueError: If the provided session_id is not a valid UUID string. + """ if logger is None: logger = logging.getLogger(DEFAULT_LOGGER) self.logger = logger - self.plugin_registry = plugin_registry or PluginRegistry() + self.plugin_registry = plugin_registry or default_registry() if system_info is None: system_info = SystemInfo() @@ -100,30 +143,46 @@ def __init__( if log_path: self.connection_result_hooks.append(FileSystemLogHook(log_base_path=log_path)) - if connections: - for connection, connection_args in connections.items(): - if connection not in self.plugin_registry.connection_managers: - self.logger.error( - "Unable to find registered connection manager class for %s", connection - ) - continue - - connection_manager = self.plugin_registry.connection_managers[connection] - - self.connection_library[connection_manager] = connection_manager( - system_info=self.system_info.model_copy(), - logger=self.logger, - connection_args=connection_args, - task_result_hooks=self.connection_result_hooks, - session_id=self.session_id, - ) + self._populate_connection_library(connections) self.logger.info("System Name: %s", self.system_info.name) if self.system_info.sku: self.logger.info("System SKU: %s", self.system_info.sku) if self.system_info.platform: self.logger.info("System Platform: %s", self.system_info.platform) - self.logger.info("System location: %s", self.system_info.location) + self.logger.info( + "%s", + format_in_band_target_summary(self.system_info, self.connection_configs), + ) + + def _populate_connection_library( + self, connections: dict[str, dict[str, Any] | BaseModel] | None + ) -> None: + """Init the connection library with the provided connections. + + Args: + connections (dict[str, dict[str, Any]]): A dictionary mapping connection names to their arguments. + where the first level of the dict is always a name of the connection class and then its arguments. + """ + if connections is None: + return + for connection, connection_args in connections.items(): + if connection not in self.plugin_registry.connection_managers: + self.logger.error( + "Unable to find registered connection manager class for %s", + connection, + ) + continue + + connection_manager = self.plugin_registry.connection_managers[connection] + + self.connection_library[connection_manager] = connection_manager( + system_info=self.system_info, + logger=self.logger, + connection_args=connection_args, + task_result_hooks=self.connection_result_hooks, + session_id=self.session_id, + ) @staticmethod def _deep_merge_plugin_args(existing: dict, incoming: dict) -> dict: @@ -145,6 +204,14 @@ def _deep_merge_plugin_args(existing: dict, incoming: dict) -> dict: @staticmethod def merge_configs(plugin_configs: list[PluginConfig]) -> PluginConfig: + """Merge multiple PluginConfig instances into a single PluginConfig. + + Args: + plugin_configs (list[PluginConfig]): A list of PluginConfig instances to merge. + + Returns: + PluginConfig: A single PluginConfig instance containing the merged configurations. + """ merged_config = PluginConfig() for config in plugin_configs: merged_config.global_args.update(config.global_args) @@ -158,9 +225,234 @@ def merge_configs(plugin_configs: list[PluginConfig]) -> PluginConfig: else: merged_config.plugins[plugin_name] = dict(plugin_args) merged_config.result_collators.update(config.result_collators) + merged_config.post_action_plugins.extend(config.post_action_plugins) return merged_config + def discover_os_info(self) -> None: + """Use a dedicated InBandConnectionManager to discover OS info and update system_info. + + The discovery connection is separate from the connection library and its result is + saved under ``/os_discovery`` so it does not mix with the results of the + connections used by plugins. self.system_info will be updated with the discovered OS info. + """ + if self.log_path: + hooks = hooks_for_fixed_directory( + self.connection_result_hooks, os.path.join(self.log_path, OS_DISCOVERY_DIR) + ) + else: + hooks = list(self.connection_result_hooks) + try: + inband_connection = InBandConnectionManager( + system_info=self.system_info, + logger=self.logger, + connection_args=self.connection_configs.get(InBandConnectionManager.__name__), + task_result_hooks=hooks, + parent=OS_DISCOVERY_DIR, + session_id=self.session_id, + ) + except Exception as e: + # Raised for missing or invalid connection args or an invalid session_id. + self.logger.error( + "Error occurred while initializing InBandConnectionManager: %s", str(e) + ) + return + try: + result = inband_connection.connect() if inband_connection else None + if (not inband_connection) or (not result) or (result.status != ExecutionStatus.OK): + self.logger.info( + "InBandConnectionManager not available or failed to connect for OS discovery. Skipping OS discovery." + ) + return + discover_and_write_os_family(inband_connection, self.system_info, self.logger) + except Exception as e: + self.logger.error( + "Error occurred during OS discovery with InBandConnectionManager: %s", str(e) + ) + finally: + inband_connection.safe_disconnect() + + def _resolve_redfish_manager(self, plugin_name: str) -> Optional[ConnectionManager]: + """Pick the Redfish transport for a plugin declaring ``RedfishConnectionManager``. + + The direct ``RedfishConnectionManager`` wins when configured; otherwise + ``RedfishSshProxyConnectionManager`` is used. With neither configured the plugin is skipped. + + Args: + plugin_name: Registry key for the plugin (for logging). + + Returns: + The connection manager instance, or None if no Redfish connection is configured. + """ + for name in ( + RedfishConnectionManager.__name__, + RedfishSshProxyConnectionManager.__name__, + ): + if name not in self.connection_configs: + continue + mgr_impl = self.plugin_registry.connection_managers.get(name) + conn_mgr = self.connection_library.get(mgr_impl) if mgr_impl else None + if conn_mgr is not None: + return conn_mgr + self.logger.error( + "%s requires RedfishConnectionManager or RedfishSshProxyConnectionManager in the " + "connection config", + plugin_name, + ) + return None + + def _get_connection_manager_for_plugin( + self, + plugin_class: type, + plugin_name: str, + ) -> Optional[ConnectionManager]: + """Resolve and (if needed) initialise the connection manager for *plugin_class*. + + Returns the :class:`~nodescraper.interfaces.ConnectionManager` instance to + use, or ``None`` if one cannot be obtained (errors are logged and the caller + should skip the plugin). + + The resolved manager is stored in ``self.connection_library`` so that it can + be shared across plugins that require the same type. + + Plugins declaring ``RedfishConnectionManager`` fall back to + ``RedfishSshProxyConnectionManager`` when only the proxy is configured + (see :meth:`_resolve_redfish_manager`). Plugins declaring + ``RedfishSshProxyConnectionManager`` always use the proxy. + """ + if not plugin_class.CONNECTION_TYPE: + return None + + if issubclass(plugin_class, OOBSSHDataPlugin): + mgr_impl = OobSshConnectionManager + connection_args = self.connection_configs.get("RedfishConnectionManager") + if connection_args is None: + self.logger.error( + "%s requires RedfishConnectionManager in the connection config", + plugin_name, + ) + return None + else: + connection_manager_class: Type[ConnectionManager] = plugin_class.CONNECTION_TYPE + if connection_manager_class is RedfishConnectionManager: + return self._resolve_redfish_manager(plugin_name) + if connection_manager_class.__name__ in self.plugin_registry.connection_managers: + mgr_impl = self.plugin_registry.connection_managers[ + connection_manager_class.__name__ + ] + elif ( + inspect.isclass(connection_manager_class) + and issubclass(connection_manager_class, ConnectionManager) + and not inspect.isabstract(connection_manager_class) + ): + # External packages set CONNECTION_TYPE on the plugin; use it when + # not listed under nodescraper.connection_managers entry points. + mgr_impl = connection_manager_class + else: + self.logger.error( + "Unable to find registered connection manager class for %s that is required by %s", + getattr(connection_manager_class, "__name__", connection_manager_class), + plugin_name, + ) + return None + connection_args = None + + if mgr_impl not in self.connection_library: + self.logger.info("Initializing connection manager for %s", mgr_impl.__name__) + init_kwargs = { + "system_info": self.system_info, + "logger": self.logger, + "task_result_hooks": self.connection_result_hooks, + "session_id": self.session_id, + } + if connection_args is not None: + init_kwargs["connection_args"] = connection_args + try: + self.connection_library[mgr_impl] = mgr_impl(**init_kwargs) + except Exception as e: + self.logger.error( + "Failed to initialize %s for %s: %s", mgr_impl.__name__, plugin_name, e + ) + return None + + return self.connection_library[mgr_impl] + + def _run_single_plugin( + self, + plugin_name: str, + plugin_args: dict, + plugin_results: list[PluginResult], + *, + queue_callback: Optional[Callable], + ) -> bool: + """Instantiate and run one plugin, appending its result to *plugin_results*. + + Args: + plugin_name: Registry key for the plugin to run. + plugin_args: Arguments forwarded to ``plugin_inst.run()``. + plugin_results: Accumulated results list; mutated in-place on success. + queue_callback: Passed to the plugin constructor so it can enqueue + additional plugins. Pass ``None`` for post-action + plugins that must not extend the primary queue. + + Returns: + ``True`` if the plugin ran (or raised an unexpected exception that was + logged), ``False`` if it was skipped due to a registry miss, missing + connection manager, or invalid global args. + """ + # With a lazy registry this is where the plugin module is imported. + plugin_class = self.plugin_registry.plugins.get(plugin_name) + if plugin_class is None: + self.logger.error("Unable to find registered plugin for name %s", plugin_name) + return False + + init_payload = { + "system_info": self.system_info, + "logger": self.logger, + "queue_callback": queue_callback, + "log_path": self.log_path, + "session_id": self.session_id, + } + + conn_mgr: Optional[ConnectionManager] = None + if plugin_class.CONNECTION_TYPE: + conn_mgr = self._get_connection_manager_for_plugin(plugin_class, plugin_name) + connection_config = self.connection_configs.get(plugin_class.CONNECTION_TYPE.__name__) + init_payload["connection_args"] = connection_config + if conn_mgr is None: + return False + init_payload["connection_manager"] = conn_mgr + try: + plugin_inst = plugin_class(**init_payload) + + plugin_run_args = copy.deepcopy(plugin_args) + + try: + global_run_args = self.apply_global_args_to_plugin( + plugin_inst, plugin_class, self.plugin_config.global_args + ) + + run_payload = self.merge_plugin_run_args(plugin_run_args, global_run_args) + except ValueError as ve: + self.logger.error( + "Invalid global_args for plugin %s: %s. Skipping plugin.", + plugin_name, + str(ve), + ) + return False + + plugin_result = plugin_inst.run(**run_payload) + plugin_results.append(plugin_result) + for hook in self.plugin_run_result_hooks: + hook(plugin_result) + + except Exception as e: + self.logger.exception("Unexpected exception when running plugin %s: %s", plugin_name, e) + if conn_mgr is not None: + conn_mgr.safe_disconnect() + + return True + def run_queue(self) -> list[PluginResult]: """Run the plugin queue and return results @@ -168,146 +460,140 @@ def run_queue(self) -> list[PluginResult]: list[PluginResult]: List of results from running the plugins in the queue """ plugin_results = [] + # For Plugins discover OS Family + if self.system_info.os_family is None or self.system_info.os_family == OSFamily.UNKNOWN: + self.discover_os_info() plugin_queue = deque(self.plugin_config.plugins.items()) try: while len(plugin_queue) > 0: plugin_name, plugin_args = plugin_queue.popleft() - if plugin_name not in self.plugin_registry.plugins: - self.logger.error("Unable to find registered plugin for name %s", plugin_name) - continue - - plugin_class = self.plugin_registry.plugins[plugin_name] - - init_payload = { - "system_info": self.system_info, - "logger": self.logger, - "queue_callback": plugin_queue.append, - "log_path": self.log_path, - "session_id": self.session_id, - } - - if plugin_class.CONNECTION_TYPE: - if issubclass(plugin_class, OOBSSHDataPlugin): - mgr_impl = OobSshConnectionManager - connection_args = self.connection_configs.get("RedfishConnectionManager") - if connection_args is None: - self.logger.error( - "%s requires RedfishConnectionManager in the connection config", - plugin_name, - ) - continue - else: - connection_manager_class: Type[ConnectionManager] = ( - plugin_class.CONNECTION_TYPE - ) - if ( - connection_manager_class.__name__ - in self.plugin_registry.connection_managers - ): - mgr_impl = self.plugin_registry.connection_managers[ - connection_manager_class.__name__ - ] - elif ( - inspect.isclass(connection_manager_class) - and issubclass(connection_manager_class, ConnectionManager) - and not inspect.isabstract(connection_manager_class) - ): - # External packages set CONNECTION_TYPE on the plugin; - # use it when not listed under nodescraper.connection_managers entry points. - mgr_impl = connection_manager_class - else: - self.logger.error( - "Unable to find registered connection manager class for %s that is required by", - connection_manager_class.__name__, - ) - continue - connection_args = None - - if mgr_impl not in self.connection_library: - self.logger.info( - "Initializing connection manager for %s", - mgr_impl.__name__, - ) - init_kwargs = { - "system_info": self.system_info, - "logger": self.logger, - "task_result_hooks": self.connection_result_hooks, - "session_id": self.session_id, - } - if connection_args is not None: - init_kwargs["connection_args"] = connection_args - self.connection_library[mgr_impl] = mgr_impl(**init_kwargs) - - init_payload["connection_manager"] = self.connection_library[mgr_impl] - - try: - plugin_inst = plugin_class(**init_payload) - - run_payload = copy.deepcopy(plugin_args) - run_args = TypeUtils.get_func_arg_types(plugin_class.run, plugin_class) - - for arg in run_args.keys(): - if arg == "preserve_connection" and issubclass(plugin_class, DataPlugin): - run_payload[arg] = True - - try: - global_run_args = self.apply_global_args_to_plugin( - plugin_inst, plugin_class, self.plugin_config.global_args - ) - # Merge analysis_args and collection_args - for args_key in ["analysis_args", "collection_args"]: - if args_key in global_run_args and args_key in run_payload: - # Merge: global args override plugin-specific args keys specified in both global and plugin-specific args - run_payload[args_key].update(global_run_args[args_key]) - del global_run_args[args_key] - run_payload.update(global_run_args) - except ValueError as ve: - self.logger.error( - "Invalid global_args for plugin %s: %s. Skipping plugin.", - plugin_name, - str(ve), - ) - continue - - self.logger.info("-" * 50) - plugin_result = plugin_inst.run(**run_payload) - plugin_results.append(plugin_result) - for hook in self.plugin_run_result_hooks: - hook(plugin_result) - except Exception as e: - self.logger.exception( - "Unexpected exception when running plugin %s: %s", plugin_name, e - ) + self.logger.info("-" * 50) + self._run_single_plugin( + plugin_name, + plugin_args, + plugin_results, + queue_callback=plugin_queue.append, + ) except Exception as e: self.logger.exception("Unexpected exception running plugin queue: %s", str(e)) finally: - self.logger.info("Closing connections") - - if self.plugin_config.result_collators: - self.logger.info("Running result collators") - for collator, collator_args in self.plugin_config.result_collators.items(): - collator_class = self.plugin_registry.result_collators.get(collator) - if collator_class is None: - self.logger.warning( - "No result collator found in registry for name: %s", collator - ) - continue - - self.logger.info("Running %s result collator", collator) - collator_inst = collator_class(logger=self.logger, log_path=self.log_path) - collator_inst.collate_results( - plugin_results, - [ - connection_manager.result - for connection_manager in self.connection_library.values() - ], - **collator_args, - ) - for connection_manager in self.connection_library.values(): - connection_manager.disconnect() + try: + # Run post-action plugins before tearing down connections or collating + # results so that post-actions still have access to live connections + # and their results are included in the collator output. + self._run_post_actions(plugin_results) + self._run_result_collators(plugin_results) + finally: + self._close_connections() return plugin_results + def _run_result_collators(self, plugin_results: list[PluginResult]) -> None: + """Run the configured result collators over the plugin and connection results.""" + if not self.plugin_config.result_collators: + return + + self.logger.info("Running result collators") + connection_results = [ + connection_manager.result for connection_manager in self.connection_library.values() + ] + for collator, collator_args in self.plugin_config.result_collators.items(): + collator_class = self.plugin_registry.result_collators.get(collator) + if collator_class is None: + self.logger.warning("No result collator found in registry for name: %s", collator) + continue + + self.logger.info("Running %s result collator", collator) + collator_inst = collator_class(logger=self.logger, log_path=self.log_path) + collator_inst.collate_results(plugin_results, connection_results, **collator_args) + + def _log_connection_result(self, result: TaskResult) -> None: + self.logger.log( + STATUS_LOG_LEVEL_MAP.get(result.status, logging.INFO), + "Connection result for %s: %s%s", + result.task, + result.status.name, + f" - {result.message}" if result.message else "", + ) + + def _close_connections(self) -> None: + """Close every connection and log the connection results. + + Each manager's own result was already saved by connect(). A failing disconnect cannot + prevent the remaining connections from being closed. + """ + self.logger.info("Closing connections") + for connection_manager in self.connection_library.values(): + connection_manager.safe_disconnect() + result = connection_manager.result + if result.status != ExecutionStatus.UNSET or result.events: + self._log_connection_result(result) + + def _run_post_actions(self, plugin_results: list[PluginResult]) -> None: + """Evaluate post-action conditions and run qualifying plugins. + + Post-action plugins are run after all primary plugins have finished but + *before* connections are closed, so they have full access to the same + connection managers. Their :class:`~nodescraper.models.PluginResult` + objects are appended to *plugin_results* in-place, making them visible + to result collators and any ``plugin_run_result_hooks``. + + Each :class:`~nodescraper.models.PostActionPluginConfig` entry in + ``self.plugin_config.post_action_plugins`` is evaluated independently; + those whose conditions are satisfied (OR semantics within each entry's + ``conditions`` list) are run in order. + + .. note:: + The same plugin name may appear in multiple entries with different + ``conditions`` or ``plugin_args``. Each entry is an independent + trigger — if both entries' conditions are met the plugin runs once + per matching entry. To trigger a single run on condition A **or** + condition B, place both conditions inside the same entry's + ``conditions`` list rather than using two entries. + + Args: + plugin_results: Accumulated primary plugin results. Mutated in-place + with results from post-action plugins that fire. + """ + if not self.plugin_config.post_action_plugins: + return + + self.logger.info("Evaluating post-action plugin conditions") + + for post_action in self.plugin_config.post_action_plugins: + plugin_name = post_action.plugin + + if not post_action.should_run(plugin_results): + self.logger.info("Post-action plugin %s: conditions not met, skipping", plugin_name) + continue + + self.logger.info("=" * 50) + self.logger.info("Running post-action plugin: %s", plugin_name) + + self._run_single_plugin( + plugin_name, + post_action.plugin_args, + plugin_results, + queue_callback=None, + ) + + @staticmethod + def merge_plugin_run_args(plugin_args: dict, global_run_args: dict) -> dict: + """Apply per-plugin run() overrides on top of global defaults.""" + run_payload = copy.deepcopy(global_run_args) + plugin_overrides = copy.deepcopy(plugin_args) + for args_key in ("analysis_args", "collection_args"): + plugin_nested = plugin_overrides.pop(args_key, None) + if plugin_nested is None: + continue + global_nested = run_payload.get(args_key) + if isinstance(global_nested, dict) and isinstance(plugin_nested, dict): + run_payload[args_key] = {**global_nested, **plugin_nested} + else: + run_payload[args_key] = plugin_nested + run_payload.update(plugin_overrides) + return run_payload + def apply_global_args_to_plugin( self, plugin_inst: PluginInterface, @@ -342,24 +628,17 @@ def apply_global_args_to_plugin( args_cls = plugin_class._collector_args_class(collector_cls) if args_cls is None: continue - plugin_fields = set(args_cls.model_fields.keys()) - filtered = { - k: v - for k, v in global_args["collection_args"].items() - if k in plugin_fields - } + filtered = TypeUtils.filter_to_model_fields( + args_cls, global_args["collection_args"] + ) if filtered: per_collector_args[collector_cls.__name__] = filtered if per_collector_args: run_args["collection_args"] = per_collector_args elif collector_args is not None and not isinstance(collector_args, dict): - args_cls = ( - collector_args if isinstance(collector_args, type) else type(collector_args) + filtered = TypeUtils.filter_to_model_fields( + collector_args, global_args["collection_args"] ) - plugin_fields = set(args_cls.model_fields.keys()) - filtered = { - k: v for k, v in global_args["collection_args"].items() if k in plugin_fields - } if filtered: run_args["collection_args"] = filtered @@ -368,10 +647,9 @@ def apply_global_args_to_plugin( and hasattr(plugin_class, "ANALYZER_ARGS") and plugin_class.ANALYZER_ARGS is not None ): - analyzer_args = plugin_class.ANALYZER_ARGS - args_cls = analyzer_args if isinstance(analyzer_args, type) else type(analyzer_args) - plugin_fields = set(args_cls.model_fields.keys()) - filtered = {k: v for k, v in global_args["analysis_args"].items() if k in plugin_fields} + filtered = TypeUtils.filter_to_model_fields( + plugin_class.ANALYZER_ARGS, global_args["analysis_args"] + ) if filtered: run_args["analysis_args"] = filtered diff --git a/nodescraper/pluginrecipe/__init__.py b/nodescraper/pluginrecipe/__init__.py index b28ca993..deb0b934 100644 --- a/nodescraper/pluginrecipe/__init__.py +++ b/nodescraper/pluginrecipe/__init__.py @@ -7,7 +7,7 @@ ############################################################################### from nodescraper.models import PluginConfig -from .all_plugins import AllPlugins +from .all_ib_plugins import AllIbPlugins from .node_status import NodeStatus from .pluginrecipe import ( ANALYZE_ONLY, @@ -24,7 +24,7 @@ "ANALYZE_ONLY", "COLLECT_AND_ANALYZE", "COLLECT_ONLY", - "AllPlugins", + "AllIbPlugins", "AnalyzerOnlyPluginRecipe", "CollectorOnlyPluginRecipe", "NodeStatus", diff --git a/nodescraper/pluginrecipe/all_ib_plugins.py b/nodescraper/pluginrecipe/all_ib_plugins.py new file mode 100644 index 00000000..eef7597d --- /dev/null +++ b/nodescraper/pluginrecipe/all_ib_plugins.py @@ -0,0 +1,52 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from nodescraper.connection.inband import InBandConnectionManager + +from .discovery import PluginDiscovery +from .pluginrecipe import PluginRecipe + + +class AllIbPlugins(PluginRecipe): + """Run all registered in-band plugins with default arguments.""" + + @classmethod + def plugin_names(cls) -> tuple[str, ...]: + """Return every in-band plugin registered at runtime. + + Returns: + tuple[str, ...]: Sorted names of all in-band plugins in the plugin registry. + """ + discovery = PluginDiscovery() + return tuple( + sorted( + name + for name in discovery.registered_plugin_names() + if getattr(discovery.load_plugin_class(name), "CONNECTION_TYPE", None) + is InBandConnectionManager + ) + ) diff --git a/nodescraper/pluginrecipe/all_plugins.py b/nodescraper/pluginrecipe/all_plugins.py deleted file mode 100644 index 948c12e7..00000000 --- a/nodescraper/pluginrecipe/all_plugins.py +++ /dev/null @@ -1,24 +0,0 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2025 Advanced Micro Devices, Inc. -# -############################################################################### -from __future__ import annotations - -from .discovery import PluginDiscovery -from .pluginrecipe import PluginRecipe - - -class AllPlugins(PluginRecipe): - """Run all registered plugins with default arguments.""" - - @classmethod - def plugin_names(cls) -> tuple[str, ...]: - """Return every plugin registered at runtime. - - Returns: - tuple[str, ...]: Sorted names of all plugins in the plugin registry. - """ - return PluginDiscovery().registered_plugin_names() diff --git a/nodescraper/pluginrecipe/discovery.py b/nodescraper/pluginrecipe/discovery.py index 65d0186e..3e1ee0e4 100644 --- a/nodescraper/pluginrecipe/discovery.py +++ b/nodescraper/pluginrecipe/discovery.py @@ -27,9 +27,9 @@ from __future__ import annotations import threading -from typing import TYPE_CHECKING, Iterable +from typing import TYPE_CHECKING, Iterable, Mapping -from nodescraper.pluginregistry import PluginRegistry +from nodescraper.pluginregistry import default_registry if TYPE_CHECKING: from nodescraper.interfaces import PluginInterface @@ -43,7 +43,8 @@ class PluginDiscovery: If use_cache is False, it will query the PluginRegistry each time. """ - _plugin_cache: dict[str, type[PluginInterface]] | None = None + _plugin_cache: Mapping[str, type[PluginInterface]] | None = None + _cache_registry: object | None = None _cache_lock = threading.Lock() COLLECTOR_ATTRIBUTE = "COLLECTOR" ANALYZER_ATTRIBUTE = "ANALYZER" @@ -56,14 +57,23 @@ def __init__(self, use_cache: bool = True) -> None: """ self._use_cache = use_cache - def load_plugin_class(self, plugin_name: str) -> type | None: + def _plugins(self) -> Mapping[str, type[PluginInterface]]: + """Plugin mapping from the shared registry (kept as-is, never copied, so lazy loading is preserved).""" if not self._use_cache: - return PluginRegistry().plugins.get(plugin_name) + return default_registry().plugins + registry = default_registry() with PluginDiscovery._cache_lock: - if PluginDiscovery._plugin_cache is None: - PluginDiscovery._plugin_cache = PluginRegistry().plugins.copy() - return PluginDiscovery._plugin_cache.get(plugin_name) + if ( + PluginDiscovery._plugin_cache is None + or PluginDiscovery._cache_registry is not registry + ): + PluginDiscovery._plugin_cache = registry.plugins + PluginDiscovery._cache_registry = registry + return PluginDiscovery._plugin_cache + + def load_plugin_class(self, plugin_name: str) -> type | None: + return self._plugins().get(plugin_name) def plugin_has_collector(self, plugin_name: str) -> bool: """Check if a plugin has a COLLECTOR attribute. @@ -125,6 +135,7 @@ def clear_cache() -> None: """ with PluginDiscovery._cache_lock: PluginDiscovery._plugin_cache = None + PluginDiscovery._cache_registry = None def registered_plugin_names(self) -> tuple[str, ...]: """Return all plugin names known to :class:`~nodescraper.pluginregistry.PluginRegistry`. @@ -132,13 +143,7 @@ def registered_plugin_names(self) -> tuple[str, ...]: Returns: tuple[str, ...]: Sorted registered plugin names. """ - if not self._use_cache: - return tuple(sorted(PluginRegistry().plugins.keys())) - - with PluginDiscovery._cache_lock: - if PluginDiscovery._plugin_cache is None: - PluginDiscovery._plugin_cache = PluginRegistry().plugins - return tuple(sorted(PluginDiscovery._plugin_cache.keys())) + return tuple(sorted(self._plugins().keys())) def plugin_names_matching(self, names: Iterable[str]) -> tuple[str, ...]: """Return plugin names from ``names`` that are registered at runtime. diff --git a/nodescraper/pluginregistry.py b/nodescraper/pluginregistry.py index cca5bbf2..5b190f44 100644 --- a/nodescraper/pluginregistry.py +++ b/nodescraper/pluginregistry.py @@ -26,10 +26,12 @@ import importlib import importlib.metadata import inspect +import logging import pkgutil import threading import types -from typing import Iterable, Optional +from collections.abc import MutableMapping +from typing import Callable, Iterable, Iterator, Optional # Python 3.9 compatibility: EntryPoints type was added in 3.10 try: @@ -50,6 +52,113 @@ ENTRY_POINT_PLUGINS = "nodescraper.plugins" ENTRY_POINT_CONNECTION_MANAGERS = "nodescraper.connection_managers" +_logger = logging.getLogger(__name__) + + +class LazyPluginMapping(MutableMapping[str, type]): + """Mapping of plugin name to plugin class that imports each class on first access. + + - ``keys()``, iteration and ``len()`` list every indexed name without importing anything. + - ``mapping[name]``, ``get(name)`` and ``name in mapping`` import just that plugin, so ``in`` + and ``[]`` always agree. A plugin that fails to import, or is not a valid plugin, is logged + once and then behaves as missing; other plugins are unaffected. + - ``items()``, ``values()`` and ``copy()`` resolve every plugin. + """ + + def __init__(self, base_class: type) -> None: + self._base_class = base_class + self._loaders: dict[str, Callable[[], Optional[type]]] = {} + self._resolved: dict[str, type] = {} + self._failed: set[str] = set() + self._lock = threading.Lock() + + def add_loader(self, name: str, loader: Callable[[], Optional[type]]) -> None: + """Register a callable that returns the class for ``name`` when first needed.""" + with self._lock: + self._loaders[name] = loader + self._resolved.pop(name, None) + self._failed.discard(name) + + def is_resolved(self, name: str) -> bool: + """True if ``name`` has already been imported (does not trigger an import).""" + return name in self._resolved + + def __getitem__(self, name: str) -> type: + cls = self._resolved.get(name) + if cls is not None: + return cls + loader = self._loaders.get(name) + if loader is None or name in self._failed: + raise KeyError(name) + # The import runs outside the lock; two threads may both import, which is harmless. + cls = self._load(name, loader) + with self._lock: + if self._loaders.get(name) is loader: + self._resolved[name] = cls + return cls + + def _load(self, name: str, loader: Callable[[], Optional[type]]) -> type: + """Run ``loader`` and validate the result, marking ``name`` failed if unusable.""" + try: + cls = loader() + except Exception as exc: # a broken plugin must not break the others + _logger.warning("Unable to load plugin %s: %s", name, exc) + self._mark_failed(name) + raise KeyError(name) from exc + if ( + cls is None + or not PluginRegistry._valid_sub_class_check(cls, self._base_class) + or (hasattr(cls, "is_valid") and not cls.is_valid()) + ): + _logger.debug("Plugin %s is not a valid %s", name, self._base_class.__name__) + self._mark_failed(name) + raise KeyError(name) + return cls + + def _mark_failed(self, name: str) -> None: + with self._lock: + self._failed.add(name) + + def __setitem__(self, name: str, value: type) -> None: + with self._lock: + self._resolved[name] = value + self._loaders[name] = lambda: value + self._failed.discard(name) + + def __delitem__(self, name: str) -> None: + with self._lock: + if name not in self._loaders: + raise KeyError(name) + del self._loaders[name] + self._resolved.pop(name, None) + self._failed.discard(name) + + def __contains__(self, name: object) -> bool: + try: + self[name] # type: ignore[index] + except KeyError: + return False + return True + + def __iter__(self) -> Iterator[str]: + return iter([n for n in list(self._loaders) if n not in self._failed]) + + def __len__(self) -> int: + return len(self._loaders) - len(self._failed) + + def items(self) -> list[tuple[str, type]]: # type: ignore[override] + """Resolve every plugin; names that cannot be loaded are skipped.""" + pairs = [(name, self.get(name)) for name in list(self)] + return [(name, cls) for name, cls in pairs if cls is not None] + + def values(self) -> list[type]: # type: ignore[override] + """Resolve every plugin; see :meth:`items`.""" + return [cls for _, cls in self.items()] + + def copy(self) -> dict[str, type]: + """Return a plain dict of all loadable plugins (resolves every plugin).""" + return dict(self.items()) + class PluginRegistry: """This class dynamically loads plugins. Internal plugins are loaded by default using @@ -79,6 +188,7 @@ def __init__( load_internal_plugins: bool = True, load_entry_point_plugins: bool = True, load_entry_point_connection_managers: bool = True, + lazy_load: bool = False, ) -> None: """Initialize the PluginRegistry with optional plugin packages. @@ -88,7 +198,11 @@ def __init__( load_entry_point_plugins (bool, optional): Whether to load plugins from entry points. Defaults to True. load_entry_point_connection_managers (bool, optional): Whether to load connection managers from the ``nodescraper.connection_managers`` entry-point group. Defaults to True. + lazy_load (bool, optional): If True, ``plugins`` is a :class:`LazyPluginMapping` and plugin + modules are imported only when a plugin is first accessed. Defaults to False. """ + self.lazy: bool = lazy_load + if load_internal_plugins: self.plugin_pkg = [ internal_plugins, @@ -101,15 +215,17 @@ def __init__( if plugin_pkg: self.plugin_pkg += plugin_pkg - self.plugins: dict[str, type[PluginInterface]] = PluginRegistry.load_plugins( - PluginInterface, self.plugin_pkg - ) - self.connection_managers: dict[str, type[ConnectionManager]] = PluginRegistry.load_plugins( - ConnectionManager, self.plugin_pkg - ) - self.result_collators: dict[str, type[PluginResultCollator]] = PluginRegistry.load_plugins( - PluginResultCollator, self.plugin_pkg - ) + self.plugins: MutableMapping[str, type[PluginInterface]] + if self.lazy: + self._init_lazy(plugin_pkg or [], load_internal_plugins) + else: + self.plugins = PluginRegistry.load_plugins(PluginInterface, self.plugin_pkg) + self.connection_managers: dict[str, type[ConnectionManager]] = ( + PluginRegistry.load_plugins(ConnectionManager, self.plugin_pkg) + ) + self.result_collators: dict[str, type[PluginResultCollator]] = ( + PluginRegistry.load_plugins(PluginResultCollator, self.plugin_pkg) + ) if load_entry_point_connection_managers: for ( @@ -119,8 +235,47 @@ def __init__( self.connection_managers[name] = mgr_cls if load_entry_point_plugins: - entry_point_plugins = self.load_plugins_from_entry_points() - self.plugins.update(entry_point_plugins) + if self.lazy: + self._add_lazy_entry_point_plugins() + else: + self.plugins.update(self.load_plugins_from_entry_points()) + + def _init_lazy(self, extra_pkgs: list[types.ModuleType], load_internal: bool) -> None: + """Lazy setup: register internal plugins from entry points without importing them. + + Connection managers and result collators are few, so they are still loaded eagerly, + but only from their own packages (not by walking every plugin module). + """ + lazy_plugins = LazyPluginMapping(PluginInterface) + self.plugins = lazy_plugins + if load_internal: + self._add_lazy_entry_point_plugins(package_prefix=internal_plugins.__name__ + ".") + if extra_pkgs: + # Caller-supplied packages have no naming convention to index; load them directly + for name, cls in PluginRegistry.load_plugins(PluginInterface, extra_pkgs).items(): + lazy_plugins[name] = cls + + non_plugin_pkgs = ([internal_connections] if load_internal else []) + extra_pkgs + self.connection_managers = PluginRegistry.load_plugins(ConnectionManager, non_plugin_pkgs) + collator_pkgs = ([internal_collators] if load_internal else []) + extra_pkgs + self.result_collators = PluginRegistry.load_plugins(PluginResultCollator, collator_pkgs) + + def _add_lazy_entry_point_plugins(self, package_prefix: Optional[str] = None) -> None: + """Register entry-point plugins; ``load()`` runs on first access. + + If ``package_prefix`` is given, only entry points whose module starts with it are registered. + """ + assert isinstance(self.plugins, LazyPluginMapping) + for entry_point in PluginRegistry.load_entry_points(ENTRY_POINT_PLUGINS): + # Eager mode keys by class __name__, which is the attribute in ``module:Class``. + value = getattr(entry_point, "value", "") or "" + if package_prefix and not value.startswith(package_prefix): + continue + attr = value.partition(":")[2].strip().split(".")[-1].split("[")[0].strip() + self.plugins.add_loader( + attr or entry_point.name, + entry_point.load, # type: ignore[attr-defined] + ) @staticmethod def load_plugins( @@ -221,16 +376,16 @@ def load_connection_managers_from_entry_points() -> dict[str, type]: PluginRegistry._use_cache and PluginRegistry._entry_point_connection_managers_cache is not None ): - return PluginRegistry._entry_point_connection_managers_cache + return PluginRegistry._entry_point_connection_managers_cache.copy() # If caching disabled, skip lock and always reload if not PluginRegistry._use_cache: - return PluginRegistry._load_connection_managers_uncached() + return PluginRegistry._load_connection_managers_uncached().copy() with PluginRegistry._cache_lock: # Check again inside the lock to prevent duplicate work if PluginRegistry._entry_point_connection_managers_cache is not None: - return PluginRegistry._entry_point_connection_managers_cache + return PluginRegistry._entry_point_connection_managers_cache.copy() managers = PluginRegistry._load_connection_managers_uncached() @@ -274,10 +429,8 @@ def _load_plugins_uncached() -> dict[str, type]: """Internal: Load plugins without caching logic.""" plugins = {} eps: Iterable = PluginRegistry.load_entry_points(ENTRY_POINT_PLUGINS) - for entry_point in eps: plugin_class = entry_point.load() # type: ignore[attr-defined, union-attr] - if not PluginRegistry._valid_sub_class_check( in_cls=plugin_class, base_class=PluginInterface ): @@ -325,3 +478,32 @@ def clear_caches(cls) -> None: cls._entry_point_connection_managers_cache = None cls._module_cache.clear() cls._entry_points_cache.clear() + + +_default_registry: Optional[PluginRegistry] = None +_default_registry_lock = threading.Lock() + + +def default_registry() -> PluginRegistry: + """Return the process-wide shared registry, creating it on first use. + + The CLI installs its own registry with :func:`set_default_registry`; embedders and + library code that are not handed a registry share this one instead of building new ones. + """ + global _default_registry + with _default_registry_lock: + existing = _default_registry + if existing is not None: + return existing + created = PluginRegistry() + with _default_registry_lock: + if _default_registry is None: + _default_registry = created + return _default_registry + + +def set_default_registry(registry: Optional[PluginRegistry]) -> None: + """Install ``registry`` as the shared default (``None`` resets it).""" + global _default_registry + with _default_registry_lock: + _default_registry = registry diff --git a/nodescraper/plugins/inband/amdsmi/__init__.py b/nodescraper/plugins/inband/amdsmi/__init__.py index f117a9fd..89f4d2c1 100644 --- a/nodescraper/plugins/inband/amdsmi/__init__.py +++ b/nodescraper/plugins/inband/amdsmi/__init__.py @@ -23,6 +23,14 @@ # SOFTWARE. # ############################################################################### +from .amdsmi_flavor_collector import AmdSmiFlavorCollector +from .amdsmi_flavor_plugin import AmdSmiFlavorPlugin from .amdsmi_plugin import AmdSmiPlugin +from .amdsmidata_flavor import AmdSmiFlavorDataModel -__all__ = ["AmdSmiPlugin"] +__all__ = [ + "AmdSmiPlugin", + "AmdSmiFlavorPlugin", + "AmdSmiFlavorCollector", + "AmdSmiFlavorDataModel", +] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py index da427753..cf859125 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_analyzer.py @@ -24,6 +24,7 @@ # ############################################################################### import io +import math from collections import defaultdict from typing import Any, Mapping, Optional, Union @@ -36,12 +37,15 @@ AmdSmiMetric, AmdSmiStatic, EccData, + Fabric, Fw, + LinkStatusTable, Partition, Processes, + XgmiLinks, XgmiMetrics, ) -from .analyzer_args import AmdSmiAnalyzerArgs +from .analyzer_args import AmdSmiAnalyzerArgs, PowerConfig from .cper import CperAnalysisTaskMixin @@ -79,6 +83,15 @@ def _gpu_unavailable_description( return description, description +def _is_all_zeros(value: Union[str, list[str]]) -> bool: + """Return True if every comma or space separated entry is zero.""" + values = value if isinstance(value, list) else [value] + tokens = [tok for item in values for tok in str(item).replace(",", " ").split()] + if not tokens: + return False + return all(set(tok) == {"0"} for tok in tokens) + + def _static_mismatch_description(payload: dict[str, Any]) -> tuple[str, str]: """Build description/details from ``check_static_data`` per-GPU payload.""" per_gpu = payload.get("per_gpu") or [] @@ -235,6 +248,70 @@ def check_expected_max_power( console_log=True, ) + def check_power_cap_consistency( + self, + amdsmi_static_data: list[AmdSmiStatic], + power_config: PowerConfig, + ) -> None: + """Check that all GPUs have the same resolved maximum power cap.""" + if power_config.power_cap_mismatch_allowed: + return + + power_caps: dict[int, float] = {} + for gpu in amdsmi_static_data: + limit = gpu.limit + max_power_vu = limit.resolved_max_power() if limit is not None else None + if max_power_vu is None or max_power_vu.value is None: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {gpu.gpu}: power cap is not available", + priority=EventPriority.WARNING, + data={"gpu": gpu.gpu}, + console_log=True, + ) + continue + + try: + power_caps[gpu.gpu] = float(max_power_vu.value) + except (TypeError, ValueError): + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {gpu.gpu}: power cap is invalid", + priority=EventPriority.WARNING, + data={"gpu": gpu.gpu, "power_cap": max_power_vu.value}, + console_log=True, + ) + + if len(power_caps) < 2: + return + + expected_power_cap = next(iter(power_caps.values())) + mismatched_power_caps = { + gpu: power_cap + for gpu, power_cap in power_caps.items() + if power_cap != expected_power_cap + } + if mismatched_power_caps: + desc, details = _gpu_mismatch_description( + "GPU power cap", + expected_power_cap, + mismatched_power_caps, + unit=" W", + ) + self._log_event( + category=EventCategory.PLATFORM, + description=desc, + priority=EventPriority.ERROR, + data={ + "gpus": list(mismatched_power_caps.keys()), + "power_caps": power_caps, + "mismatched_power_caps": mismatched_power_caps, + "expected_power_cap": expected_power_cap, + "details": details, + }, + console_log=True, + ) + def check_expected_driver_version( self, amdsmi_static_data: list[AmdSmiStatic], @@ -314,7 +391,11 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} PCIe link speed is not Gen5 (32 GT/s)", priority=EventPriority.ERROR, - data={"gpu": gpu, "pcie_speed": speed_val, "expected": 32.0}, + data={ + "gpu": gpu, + "pcie_speed": speed_val, + "expected": 32.0, + }, console_log=True, ) except (ValueError, TypeError): @@ -337,7 +418,10 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} has PCIe replay rollover count: {pcie_data.replay_roll_over_count}", priority=EventPriority.WARNING, - data={"gpu": gpu, "replay_roll_over_count": pcie_data.replay_roll_over_count}, + data={ + "gpu": gpu, + "replay_roll_over_count": pcie_data.replay_roll_over_count, + }, console_log=True, ) @@ -381,7 +465,10 @@ def check_amdsmi_metric_pcie( category=EventCategory.IO, description=f"GPU: {gpu} has received {pcie_data.nak_received_count} PCIe NAKs", priority=EventPriority.WARNING, - data={"gpu": gpu, "nak_received_count": pcie_data.nak_received_count}, + data={ + "gpu": gpu, + "nak_received_count": pcie_data.nak_received_count, + }, console_log=True, ) @@ -441,6 +528,93 @@ def check_amdsmi_metric_ecc_totals(self, amdsmi_metric_data: list[AmdSmiMetric]) console_log=True, ) + def check_gpu_memory( + self, + amdsmi_metric_data: Optional[list[AmdSmiMetric]], + minimum_available_percent: float, + ) -> None: + """Check the minimum free VRAM percentage for each GPU.""" + if amdsmi_metric_data is None or len(amdsmi_metric_data) == 0: + self._log_event( + category=EventCategory.PLATFORM, + description="No AMD SMI metric data available", + priority=EventPriority.WARNING, + data={"amdsmi_metric_data": amdsmi_metric_data}, + ) + return + for metric in amdsmi_metric_data: + memory = metric.mem_usage + total_vram = memory.total_vram if memory is not None else None + free_vram = memory.free_vram if memory is not None else None + values = { + "gpu": metric.gpu, + "total_vram": total_vram.value if total_vram is not None else None, + "free_vram": free_vram.value if free_vram is not None else None, + "unit": total_vram.unit if total_vram is not None else None, + "minimum_available_percent": minimum_available_percent, + } + + if total_vram is None or free_vram is None: + self._log_event( + category=EventCategory.PLATFORM, + description=f"GPU {metric.gpu} VRAM information is unavailable", + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + try: + total_value = float(total_vram.value) + free_value = float(free_vram.value) + except (TypeError, ValueError): + self._log_event( + category=EventCategory.PLATFORM, + description=( + f"GPU {metric.gpu} was not able to parse available VRAM from memory usage" + ), + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + if ( + not math.isfinite(total_value) + or not math.isfinite(free_value) + or total_value <= 0 + or free_value < 0 + or total_vram.unit != free_vram.unit + ): + self._log_event( + category=EventCategory.PLATFORM, + description=( + f"GPU {metric.gpu} VRAM values are invalid " + f"(total={total_vram.value} {total_vram.unit}, " + f"free={free_vram.value} {free_vram.unit})" + ), + priority=EventPriority.WARNING, + data=values, + console_log=True, + ) + continue + + available_percent = free_value / total_value * 100 + if available_percent < minimum_available_percent: + self._log_event( + category=EventCategory.PLATFORM, + description=( + f"GPU {metric.gpu} free VRAM is {available_percent:.2f}% " + f"(minimum {minimum_available_percent:.2f}%)" + ), + priority=EventPriority.WARNING, + data={ + **values, + "available_percent": available_percent, + }, + console_log=True, + ) + def check_amdsmi_metric_ecc(self, amdsmi_metric_data: list[AmdSmiMetric]): """Check ECC counts in all blocks for all GPUs @@ -631,7 +805,12 @@ def check_static_data( and device_id[1].upper() not in dev_actual.upper() ): mismatches.append( - (gpu_data.gpu, "device_id", f"{device_id[0]}|{device_id[1]}", dev_actual) + ( + gpu_data.gpu, + "device_id", + f"{device_id[0]}|{device_id[1]}", + dev_actual, + ) ) if subsystem_id[0] is not None and subsystem_id[1] is not None: @@ -933,6 +1112,149 @@ def check_expected_xgmi_link_speed( console_log=True, ) + def check_fabric( + self, + fabric_data: Optional[list[Fabric]], + expected_accel_state: str = "ACTIVE", + expected_fabric_type: str = "UALOE", + ) -> None: + """Check fabric state, type, pod IDs and accelerator maps for all GPUs + + Args: + fabric_data (Optional[list[Fabric]]): fabric data from amd-smi fabric + expected_accel_state (str): expected accel_state value + expected_fabric_type (str): expected fabric_type value + """ + if not fabric_data: + self._log_event( + category=EventCategory.NETWORK, + description="Fabric data is not available and cannot be checked", + priority=EventPriority.WARNING, + data={"fabric": fabric_data}, + ) + return + + expected_state = expected_accel_state.strip().upper() + expected_type = expected_fabric_type.strip().upper() + issues: list[dict[str, Any]] = [] + + def _add(gpu: int, field: str, expected: object, actual: object) -> None: + issues.append( + { + "gpu": gpu, + "field": field, + "expected": str(expected), + "actual": str(actual), + } + ) + + for entry in fabric_data: + info = entry.fabric_info + gpu = entry.gpu + accel_state = info.accel_state if info else None + if accel_state is None or accel_state.strip().upper() != expected_state: + _add(gpu, "accel_state", expected_state, accel_state or "N/A") + if accel_state == "UNCONFIGURED": + # Skip all other fabric checks for this GPU if it is unconfigured + continue + + fabric_type = info.fabric_type if info else None + if fabric_type is None or fabric_type.strip().upper() != expected_type: + _add(gpu, "fabric_type", expected_type, fabric_type or "N/A") + + ppod_id = info.ppod_id if info else None + if ppod_id is None or set(ppod_id) <= {"0", "-"}: + _add(gpu, "ppod_id", "non-zero", ppod_id or "N/A") + + for field in ("local_accelerators", "local_active_accelerators"): + value = getattr(info, field) if info else None + if value is None or _is_all_zeros(value): + _add(gpu, field, "non-zero", value if value is not None else "N/A") + + for field in ("ppod_size", "vpod_size"): + value = getattr(info, field) if info else None + if not value: + _add(gpu, field, "non-zero", value if value is not None else "N/A") + + if issues: + gpus_affected = len({i["gpu"] for i in issues}) + self._log_event( + category=EventCategory.NETWORK, + description=f"Fabric data mismatch on {gpus_affected} GPU(s)", + priority=EventPriority.ERROR, + data={ + "expected_accel_state": expected_state, + "expected_fabric_type": expected_type, + "mismatches": issues, + }, + console_log=True, + ) + + def check_xgmi_or_peer_links_status(self, xgmi_links: Optional[list[XgmiLinks]]) -> None: + """Check XGMI or peer-link status: U passes, SELF is ignored, D/X warn.""" + if not xgmi_links: + self._log_event( + category=EventCategory.IO, + description="XGMI/peer link data is not available and cannot be checked", + priority=EventPriority.WARNING, + data={"xgmi_links": xgmi_links}, + console_log=True, + ) + return + + down_links: list[dict[str, Any]] = [] + degraded_links: list[dict[str, Any]] = [] + healthy_link_count = 0 + for gpu_links in xgmi_links: + for link_index, status in enumerate(gpu_links.link_status): + if status == LinkStatusTable.SELF: + continue + link_data = { + "gpu": gpu_links.gpu, + "link_index": link_index, + "status": status.value, + } + if status == LinkStatusTable.DOWN: + down_links.append(link_data) + elif status == LinkStatusTable.DISABLED: + degraded_links.append(link_data) + elif status == LinkStatusTable.UP: + healthy_link_count += 1 + + if not down_links and not degraded_links and healthy_link_count > 0: + self._log_event( + category=EventCategory.IO, + description="All XGMI/peer GPU links are working fine", + priority=EventPriority.INFO, + data={"healthy_link_count": healthy_link_count}, + console_log=True, + ) + + if down_links: + self._log_event( + category=EventCategory.IO, + description=(f"XGMI/peer links contain {len(down_links)} down/error links"), + priority=EventPriority.WARNING, + data={ + "down_links": down_links, + "link_error_count": len(down_links), + }, + console_log=True, + ) + + if degraded_links: + self._log_event( + category=EventCategory.IO, + description=( + f"XGMI/peer links contain {len(degraded_links)} " "disabled/degraded links" + ), + priority=EventPriority.WARNING, + data={ + "degraded_links": degraded_links, + }, + console_log=True, + ) + def analyze_data( self, data: AmdSmiDataModel, args: Optional[AmdSmiAnalyzerArgs] = None ) -> TaskResult: @@ -962,6 +1284,12 @@ def analyze_data( self.check_amdsmi_metric_ecc_totals(data.metric) self.check_amdsmi_metric_ecc(data.metric) + if args.gpu_memory: + self.check_gpu_memory( + data.metric, + args.gpu_memory.minimum_available_percent, + ) + if args.expected_gpu_processes: self.expected_gpu_processes(data.process, args.expected_gpu_processes) @@ -975,6 +1303,8 @@ def analyze_data( else: if args.expected_max_power: self.check_expected_max_power(data.static, args.expected_max_power) + if args.power: + self.check_power_cap_consistency(data.static, args.power) if args.expected_driver_version: self.check_expected_driver_version(data.static, args.expected_driver_version) @@ -1020,4 +1350,14 @@ def analyze_data( data.xgmi_metric, expected_xgmi_speed=args.expected_xgmi_speed ) + if data.fabric: + self.check_fabric( + data.fabric, + expected_accel_state=args.expected_accel_state, + expected_fabric_type=args.expected_fabric_type, + ) + + if args.check_xgmi_or_peer_links_status: + self.check_xgmi_or_peer_links_status(data.xgmi_link) + return self.result diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py index 92d42252..5dd4b599 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_collector.py @@ -27,7 +27,7 @@ import json import re from tarfile import TarFile -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, TypeVar, Union from pydantic import BaseModel, ValidationError @@ -42,9 +42,20 @@ AmdSmiStatic, AmdSmiVersion, BadPages, + CoreMetric, + CpuDimmPowConsumption, + CpuDimmTempRangeRate, + CpuDimmThermalSensor, + CpuIoBandwidth, + CpuLclkDpmLevel, + CpuMetric, + CpuSvi3VrControllerTemp, + CpuXgmiBandwidth, EccState, + Fabric, Fw, FwListItem, + NodeInfo, Partition, PartitionCompute, PartitionMemory, @@ -78,23 +89,67 @@ from nodescraper.plugins.inband.amdsmi.collector_args import AmdSmiCollectorArgs from nodescraper.utils import get_exception_traceback, shell_quote +T = TypeVar("T", bound=BaseModel) + +_AFID_ABSENT = {"-", "n/a", "na"} +# First integer in the AFID column. The file name can contain digits. +_AFID_VALUE_RE = re.compile(r"\d+") + + +def _parse_afid_output(output: str) -> Optional[int]: + """Return the AFID from amd-smi output, or None when the value is absent. + + Args: + output: stdout from ras --afid --cper-file. + + Returns: + The AFID integer, or None for a dash or N/A. + """ + lines = [line.strip() for line in output.splitlines() if line.strip()] + if not lines: + raise ValueError("empty AFID output") + header_index = next( + (index for index, line in enumerate(lines) if line.lower().startswith("file_name")), + None, + ) + if header_index is None: + data_line = lines[0] + elif header_index + 1 >= len(lines): + raise ValueError("AFID table has no data row") + else: + data_line = lines[header_index + 1] + tokens = data_line.split() + cell = tokens[0] if len(tokens) == 1 else " ".join(tokens[1:]) + if cell.lower() in _AFID_ABSENT: + return None + match = _AFID_VALUE_RE.search(cell) + if match is None: + raise ValueError(f"no AFID in {cell!r}") + return int(match.group(0)) + class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs]): """Class for collection of inband tool amd-smi data.""" AMD_SMI_EXE = "amd-smi" + AMD_SMI_FALLBACK_PATHS: tuple[str, ...] = ( + "/opt/rocm/bin/amd-smi", + "/opt/rocm-*/bin/amd-smi", + "/usr/local/bin/amd-smi", + ) + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.LINUX} DATA_MODEL = AmdSmiDataModel - CMD_VERSION = "version --json" - CMD_LIST = "list --json" - CMD_PROCESS = "process --json" - CMD_PARTITION = "partition --json" - CMD_FIRMWARE = "firmware --json" - CMD_STATIC = "static -g all --json" - CMD_STATIC_GPU = "static -g {gpu_id} --json" + CMD_VERSION = "version" + CMD_LIST = "list" + CMD_PROCESS = "process" + CMD_PARTITION = "partition" + CMD_FIRMWARE = "firmware" + CMD_STATIC = "static -g all" + CMD_STATIC_GPU = "static -g {gpu_id}" CMD_TOPOLOGY = "topology" CMD_METRIC = "metric -g all" CMD_BAD_PAGES = "bad-pages" @@ -102,15 +157,60 @@ class AmdSmiCollector(InBandDataCollector[AmdSmiDataModel, AmdSmiCollectorArgs]) CMD_XGMI_LINK = "xgmi -l" CMD_RAS = "ras --cper --folder={folder}" CMD_RAS_AFID = "ras --afid --cper-file {cper_file}" + CMD_FABRIC = "fabric" + CMD_NODE = "node" + CMD_METRIC_CPU = "metric --cpu all" + CMD_METRIC_CORE = "metric --core all" + CMD_CPU_DIMM_POW = "metric --cpu-dimm-pow-consumption {dimm_addr}" + CMD_CPU_DIMM_THERMAL = "metric --cpu-dimm-thermal-sensor {dimm_addr}" + CMD_CPU_DIMM_TEMP_RANGE_RATE = "metric --cpu-dimm-temp-range-rate {dimm_addr}" + CMD_CPU_SVI3_HOTTEST = "metric --cpu-svi3-vr-controller-temp 0" + CMD_CPU_SVI3_RAIL = "metric --cpu-svi3-vr-controller-temp 1 {rail_index}" + CMD_CPU_XGMI_BANDWIDTH = "metric --cpu-xgmi-bandwidth {bw} {link_id}" + CMD_CPU_IO_BANDWIDTH = "metric --cpu-io-bandwidth {bw} {link_id}" + CMD_CPU_LCLK_DPM_LEVEL = "metric --cpu-lclk-dpm-level {nbio_id}" + CPU_LINK_ID_CANDIDATES: tuple[str, ...] = ( + "P2", + "P3", + "G0", + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + ) + CPU_XGMI_BW_VALUES: tuple[int, ...] = (1, 2, 4) + CPU_IO_BW_VALUES: tuple[int, ...] = (1,) + CPU_SVI3_RAIL_CANDIDATES: tuple[int, ...] = (0, 1, 2, 3, 4) + MAX_DIMM_ADDR = 24 + MAX_NBIO_ID = 8 + CPU_SEQUENTIAL_NA_GRACE = 2 def _check_amdsmi_installed(self) -> bool: """Check if amd-smi is installed + When amd-smi is not on PATH, AMD_SMI_FALLBACK_PATHS is searched and AMD_SMI_EXE is + rebound on this instance to the resolved absolute path. + Returns: bool: True if amd-smi is installed, False otherwise """ cmd_ret = self._run_sut_cmd("which amd-smi") - return bool(cmd_ret.exit_code == 0 and "no amd-smi in" not in cmd_ret.stdout) + if cmd_ret.exit_code == 0 and "no amd-smi in" not in cmd_ret.stdout: + return True + + # Some images omit /opt/rocm/bin from the non-interactive SSH PATH; left unquoted so + # the shell expands the versioned-install glob. + search_paths = " ".join(self.AMD_SMI_FALLBACK_PATHS) + cmd_ret = self._run_sut_cmd(f'for p in {search_paths}; do command -v "$p" && break; done') + resolved = cmd_ret.stdout.strip().splitlines() + if not resolved: + return False + + self.AMD_SMI_EXE = resolved[0].strip() + return True def _run_amd_smi(self, cmd: str) -> Optional[str]: """Run amd-smi command @@ -144,6 +244,7 @@ def _run_amd_smi(self, cmd: str) -> Optional[str]: "command": cmd, "exit_code": cmd_ret.exit_code, "stderr": cmd_ret.stderr, + "stdout": cmd_ret.stdout, }, priority=EventPriority.WARNING, console_log=True, @@ -157,6 +258,7 @@ def _run_amd_smi(self, cmd: str) -> Optional[str]: "command": cmd, "exit_code": cmd_ret.exit_code, "stderr": cmd_ret.stderr, + "stdout": cmd_ret.stdout, }, priority=EventPriority.ERROR, console_log=True, @@ -202,7 +304,8 @@ def _run_amd_smi_dict(self, cmd: str) -> Optional[Union[dict, list[dict]]]: Returns: Optional[Union[dict, list[dict]]]: parsed JSON output or None on error """ - cmd += " --json" + if "--json" not in cmd.split(): + cmd += " --json" cmd_ret = self._run_amd_smi(cmd) if cmd_ret: try: @@ -449,6 +552,286 @@ def get_xgmi_data( return xgmi_metrics or [], xgmi_links or [] + def get_fabric(self) -> List[Fabric]: + """Get fabric data from amd-smi fabric --json. + + ``fabric_telemetry`` is not collected as the payload is very large. + """ + ret = self._run_amd_smi_dict(self.CMD_FABRIC) + if ret is None: + return [] + if isinstance(ret, dict) and "gpu_data" in ret: + ret = ret["gpu_data"] + data = ret if isinstance(ret, list) else [ret] + + fabric_entries: List[dict] = [] + for item in data: + if not isinstance(item, dict): + continue + # each entry is {"gpu": , "fabric": {...}} + entry = item.get("fabric") if isinstance(item.get("fabric"), dict) else item + if not isinstance(entry, dict): + continue + entry = {k: v for k, v in entry.items() if k != "fabric_telemetry"} + entry.setdefault("gpu", item.get("gpu", 0)) + fabric_entries.append(entry) + + built = self._build_amdsmi_sub_data(Fabric, fabric_entries) + return built if isinstance(built, list) else ([built] if built else []) + + def get_node(self) -> List[NodeInfo]: + """Get node-level data from amd-smi node --json.""" + ret = self._run_amd_smi_dict(self.CMD_NODE) + if ret is None: + return [] + if isinstance(ret, dict) and "node_data" in ret: + ret = ret["node_data"] + data = ret if isinstance(ret, list) else [ret] + built = self._build_amdsmi_sub_data(NodeInfo, data) + return built if isinstance(built, list) else ([built] if built else []) + + def get_cpu_metric(self) -> List[CpuMetric]: + """Get per-CPU-socket metrics from amd-smi metric --cpu all --json.""" + ret = self._run_amd_smi_dict(self.CMD_METRIC_CPU) + if ret is None: + return [] + if isinstance(ret, dict) and "cpu_data" in ret: + ret = ret["cpu_data"] + data = ret if isinstance(ret, list) else [ret] + built = self._build_amdsmi_sub_data(CpuMetric, data) + return built if isinstance(built, list) else ([built] if built else []) + + def get_core_metric(self) -> List[CoreMetric]: + """Get per-core metrics from amd-smi metric --core all --json.""" + ret = self._run_amd_smi_dict(self.CMD_METRIC_CORE) + if ret is None: + return [] + if isinstance(ret, dict) and "core_data" in ret: + ret = ret["core_data"] + data = ret if isinstance(ret, list) else [ret] + built = self._build_amdsmi_sub_data(CoreMetric, data) + return built if isinstance(built, list) else ([built] if built else []) + + def _amdsmi_cpu_entries(self, cmd: str) -> List[dict]: + """Run an amd-smi ``metric --cpu-*`` command and return its ``cpu_data`` list. + + Returns an empty list on any hard failure so callers can treat "no + cpu_data at all" as a definitive stop signal. + """ + ret = self._run_amd_smi_dict(cmd) + if not isinstance(ret, dict): + return [] + data = ret.get("cpu_data") + return data if isinstance(data, list) else [] + + def _append_amdsmi_model( + self, + entries: list, + model_cls: type[BaseModel], + data: dict, + context: str, + ) -> None: + """Validate ``data`` against ``model_cls`` and append; log+skip on failure.""" + try: + entries.append(model_cls.model_validate(data)) + except ValidationError as err: + self._log_event( + category=EventCategory.APPLICATION, + description=f"Failed to build {model_cls.__name__} entry; skipping", + data={"errors": err.errors(include_url=False), "context": context}, + priority=EventPriority.WARNING, + ) + + def _collect_cpu_sequential_metric( + self, + cmd_template: str, + path: tuple[str, ...], + model_cls: type[T], + max_value: int, + id_field: str, + ) -> list[T]: + """Probe a sequential CPU metric argument (DIMM_ADDR, NBIOID) per host. + + Calls ``cmd_template`` for ``id_field`` values ``0, 1, 2, ...`` and + descends ``path`` into each returned ``cpu_data`` entry. Stops once + ``CPU_SEQUENTIAL_NA_GRACE`` consecutive values produce no dict at + ``path``, or immediately if the command itself hard-fails. + """ + entries: list[T] = [] + consecutive_na = 0 + for value in range(max_value): + try: + cpu_entries = self._amdsmi_cpu_entries(cmd_template.format(**{id_field: value})) + if not cpu_entries: + break + except Exception: + break + + found_any = False + for cpu_entry in cpu_entries: + node: Any = cpu_entry + for key in path: + node = node.get(key) if isinstance(node, dict) else None + if not isinstance(node, dict): + continue + found_any = True + self._append_amdsmi_model( + entries, + model_cls, + {"cpu": cpu_entry.get("cpu", 0), id_field: value, **node}, + context=f"{id_field}={value}", + ) + + if found_any: + consecutive_na = 0 + else: + consecutive_na += 1 + if consecutive_na >= self.CPU_SEQUENTIAL_NA_GRACE: + break + + return entries + + def get_cpu_dimm_pow_consumption(self) -> List[CpuDimmPowConsumption]: + """Get per-DIMM power consumption from amd-smi metric --cpu-dimm-pow-consumption.""" + return self._collect_cpu_sequential_metric( + self.CMD_CPU_DIMM_POW, + ("dimm_pow_consumption", "response"), + CpuDimmPowConsumption, + self.MAX_DIMM_ADDR, + "dimm_addr", + ) + + def get_cpu_dimm_thermal_sensor(self) -> List[CpuDimmThermalSensor]: + """Get per-DIMM thermal sensor data from amd-smi metric --cpu-dimm-thermal-sensor.""" + return self._collect_cpu_sequential_metric( + self.CMD_CPU_DIMM_THERMAL, + ("dimm_thermal_sensor", "response"), + CpuDimmThermalSensor, + self.MAX_DIMM_ADDR, + "dimm_addr", + ) + + def get_cpu_dimm_temp_range_rate(self) -> List[CpuDimmTempRangeRate]: + """Get per-DIMM temp range/refresh rate from amd-smi metric --cpu-dimm-temp-range-rate.""" + return self._collect_cpu_sequential_metric( + self.CMD_CPU_DIMM_TEMP_RANGE_RATE, + ("dimm_temp_range_rate", "response"), + CpuDimmTempRangeRate, + self.MAX_DIMM_ADDR, + "dimm_addr", + ) + + def get_cpu_lclk_dpm_level(self) -> List[CpuLclkDpmLevel]: + """Get per-NBIO LCLK DPM level range from amd-smi metric --cpu-lclk-dpm-level.""" + return self._collect_cpu_sequential_metric( + self.CMD_CPU_LCLK_DPM_LEVEL, + ("socket_dpm", "dpml_level_range"), + CpuLclkDpmLevel, + self.MAX_NBIO_ID, + "nbio_id", + ) + + def get_cpu_svi3_vr_controller_temp(self) -> List[CpuSvi3VrControllerTemp]: + """Get SVI3 VR controller temp from amd-smi metric --cpu-svi3-vr-controller-temp. + + Always probes the hottest-rail reading (TYPE=0, no rail index) plus + every documented individual rail (TYPE=1, RAIL_INDEX 0-4). + """ + entries: List[CpuSvi3VrControllerTemp] = [] + try: + for cpu_entry in self._amdsmi_cpu_entries(self.CMD_CPU_SVI3_HOTTEST): + group = cpu_entry.get("svi3_vr_controller_temp") + if isinstance(group, dict): + self._append_amdsmi_model( + entries, + CpuSvi3VrControllerTemp, + { + "cpu": cpu_entry.get("cpu", 0), + "rail_type": 0, + "rail_index": None, + **group, + }, + context="rail_type=0", + ) + + for rail_index in self.CPU_SVI3_RAIL_CANDIDATES: + cmd = self.CMD_CPU_SVI3_RAIL.format(rail_index=rail_index) + for cpu_entry in self._amdsmi_cpu_entries(cmd): + group = cpu_entry.get("svi3_vr_controller_temp") + if isinstance(group, dict): + self._append_amdsmi_model( + entries, + CpuSvi3VrControllerTemp, + { + "cpu": cpu_entry.get("cpu", 0), + "rail_type": 1, + "rail_index": rail_index, + **group, + }, + context=f"rail_type=1 rail_index={rail_index}", + ) + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Failed to build CpuSvi3VrControllerTemp", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + ) + return entries + + def get_cpu_xgmi_bandwidth(self) -> List[CpuXgmiBandwidth]: + """Get per-link XGMI bandwidth from amd-smi metric --cpu-xgmi-bandwidth. + + Probes every candidate LINKID_NAME at every candidate XGMI_BW; gaps + are architectural, so every combination is kept. + """ + entries: List[CpuXgmiBandwidth] = [] + for bw in self.CPU_XGMI_BW_VALUES: + for link_id in self.CPU_LINK_ID_CANDIDATES: + cmd = self.CMD_CPU_XGMI_BANDWIDTH.format(bw=bw, link_id=link_id) + for cpu_entry in self._amdsmi_cpu_entries(cmd): + group = cpu_entry.get("xgmi_bandwidth") + if isinstance(group, dict): + self._append_amdsmi_model( + entries, + CpuXgmiBandwidth, + { + "cpu": cpu_entry.get("cpu", 0), + "xgmi_bw": bw, + "link_id": link_id, + **group, + }, + context=f"xgmi_bw={bw} link_id={link_id}", + ) + return entries + + def get_cpu_io_bandwidth(self) -> List[CpuIoBandwidth]: + """Get per-link IO bandwidth from amd-smi metric --cpu-io-bandwidth. + + Same mechanism as ``get_cpu_xgmi_bandwidth``. valid link ids are not + assumed to match those found for xgmi-bandwidth, so this probes the + full candidate set independently. + """ + entries: List[CpuIoBandwidth] = [] + for bw in self.CPU_IO_BW_VALUES: + for link_id in self.CPU_LINK_ID_CANDIDATES: + cmd = self.CMD_CPU_IO_BANDWIDTH.format(bw=bw, link_id=link_id) + for cpu_entry in self._amdsmi_cpu_entries(cmd): + group = cpu_entry.get("io_bandwidth") + if isinstance(group, dict): + self._append_amdsmi_model( + entries, + CpuIoBandwidth, + { + "cpu": cpu_entry.get("cpu", 0), + "io_bw": bw, + "link_id": link_id, + **group, + }, + context=f"io_bw={bw} link_id={link_id}", + ) + return entries + def _get_amdsmi_data( self, args: Optional[AmdSmiCollectorArgs] = None ) -> Optional[AmdSmiDataModel]: @@ -468,6 +851,17 @@ def _get_amdsmi_data( metric = self.get_metric() bad_pages = self.get_bad_pages() xgmi_metric, xgmi_link = self.get_xgmi_data() + fabric = self.get_fabric() + node = self.get_node() + cpu_metric = self.get_cpu_metric() + core_metric = self.get_core_metric() + cpu_dimm_pow_consumption = self.get_cpu_dimm_pow_consumption() + cpu_dimm_thermal_sensor = self.get_cpu_dimm_thermal_sensor() + cpu_dimm_temp_range_rate = self.get_cpu_dimm_temp_range_rate() + cpu_svi3_vr_controller_temp = self.get_cpu_svi3_vr_controller_temp() + cpu_xgmi_bandwidth = self.get_cpu_xgmi_bandwidth() + cpu_io_bandwidth = self.get_cpu_io_bandwidth() + cpu_lclk_dpm_level = self.get_cpu_lclk_dpm_level() cper_data, cper_afids = self.get_cper_data() except Exception as e: self._log_event( @@ -494,6 +888,17 @@ def _get_amdsmi_data( bad_pages=bad_pages or [], xgmi_metric=xgmi_metric or [], xgmi_link=xgmi_link or [], + fabric=fabric or [], + node=node or [], + cpu_metric=cpu_metric or [], + core_metric=core_metric or [], + cpu_dimm_pow_consumption=cpu_dimm_pow_consumption or [], + cpu_dimm_thermal_sensor=cpu_dimm_thermal_sensor or [], + cpu_dimm_temp_range_rate=cpu_dimm_temp_range_rate or [], + cpu_svi3_vr_controller_temp=cpu_svi3_vr_controller_temp or [], + cpu_xgmi_bandwidth=cpu_xgmi_bandwidth or [], + cpu_io_bandwidth=cpu_io_bandwidth or [], + cpu_lclk_dpm_level=cpu_lclk_dpm_level or [], cper_data=cper_data, cper_afids=cper_afids, analysis_firmware_ids=fw_ids, @@ -672,7 +1077,10 @@ def get_process(self) -> Optional[list[Processes]]: self._log_event( category=EventCategory.APPLICATION, description="Failed to build Processes", - data={"errors": err.errors(include_url=False), "gpu_index": gpu_idx}, + data={ + "errors": err.errors(include_url=False), + "gpu_index": gpu_idx, + }, priority=EventPriority.WARNING, ) @@ -803,7 +1211,10 @@ def get_firmware(self) -> Optional[list[Fw]]: self._log_event( category=EventCategory.APPLICATION, description="Failed to build Fw", - data={"errors": err.errors(include_url=False), "gpu_index": gpu_idx}, + data={ + "errors": err.errors(include_url=False), + "gpu_index": gpu_idx, + }, priority=EventPriority.WARNING, ) @@ -940,7 +1351,13 @@ def get_static(self) -> Optional[list[AmdSmiStatic]]: ) # NUMA - numa_node = int(numa.get("node", 0) or 0) + numa_node_raw = numa.get("node") + if numa_node_raw in (None, "", "N/A"): + numa_node: Union[int, str] = "N/A" + elif isinstance(numa_node_raw, str): + numa_node = numa_node_raw + else: + numa_node = int(numa_node_raw) if numa_node_raw is not None else "N/A" affinity_raw = numa.get("affinity") if affinity_raw in (None, "", "N/A"): affinity_val: Union[int, str] = "N/A" @@ -1013,7 +1430,10 @@ def get_static(self) -> Optional[list[AmdSmiStatic]]: self._log_event( category=EventCategory.APPLICATION, description="Failed to build AmdSmiStatic", - data={"errors": err.errors(include_url=False), "gpu_index": gpu_idx}, + data={ + "errors": err.errors(include_url=False), + "gpu_index": gpu_idx, + }, priority=EventPriority.WARNING, ) @@ -1414,7 +1834,10 @@ def get_cper_data(self) -> tuple[List[FileModel], Dict[str, int]]: ) # Load the tar files cper_zip = self._read_sut_file( - f"{AMD_SMI_CPER_FOLDER}.tar.gz", encoding=None, strip=False, log_artifact=True + f"{AMD_SMI_CPER_FOLDER}.tar.gz", + encoding=None, + strip=False, + log_artifact=True, ) # Since encoding=None, this returns BinaryFileArtifact which has contents: bytes if hasattr(cper_zip, "contents"): @@ -1436,7 +1859,10 @@ def get_cper_data(self) -> tuple[List[FileModel], Dict[str, int]]: else: file_content_bytes = b"" cper_data.append( - FileModel(file_contents=file_content_bytes, file_name=member.name) + FileModel( + file_contents=file_content_bytes, + file_name=member.name, + ) ) cper_file_path = f"{AMD_SMI_CPER_FOLDER}/{member.name}" @@ -1502,24 +1928,33 @@ def _get_cper_afid(self, cper_file_path: str) -> Optional[int]: return None try: - afid = int(result.strip()) + afid = _parse_afid_output(result) + except ValueError: self._log_event( category=EventCategory.APPLICATION, - description=f"Successfully retrieved AFID from CPER file: {cper_file_path}", - data={"afid": afid, "cper_file": cper_file_path}, - priority=EventPriority.INFO, + description=f"Failed to parse AFID value from output: {result}", + data={"output": result, "cper_file": cper_file_path}, + priority=EventPriority.ERROR, console_log=True, ) - return afid - except ValueError: + return None + if afid is None: self._log_event( category=EventCategory.APPLICATION, - description=f"Failed to parse AFID value from output: {result}", + description=f"No AFID reported for CPER file: {cper_file_path}", data={"output": result, "cper_file": cper_file_path}, - priority=EventPriority.ERROR, + priority=EventPriority.INFO, console_log=True, ) return None + self._log_event( + category=EventCategory.APPLICATION, + description=f"Successfully retrieved AFID from CPER file: {cper_file_path}", + data={"afid": afid, "cper_file": cper_file_path}, + priority=EventPriority.INFO, + console_log=True, + ) + return afid def collect_data( self, diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py new file mode 100644 index 00000000..364788a2 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_collector.py @@ -0,0 +1,579 @@ +"""amd-smi collector adding mxGPU host-driver (gim/amdgpuv, incl. ESXi) and guest-VF +support on top of the base guest/bare-metal-only AmdSmiCollector. + +The amd-smi CLI syntax and JSON schema depend on the loaded driver "flavor": + - host - mxGPU host driver (gim on Linux, amdgpuv on ESXi) -> HostDriver* models + - guest - amdgpu on a virtual function inside a VM -> Guest* models + - bare-metal - amdgpu on physical hardware -> base AmdSmi* models +This collector detects the flavor, issues the flavor-appropriate commands, and builds +the matching model variants into AmdSmiFlavorDataModel. The base AmdSmiCollector is +left unchanged for callers that do not need host/VF support. +""" + +from __future__ import annotations + +import io +import json +import re +from tarfile import TarFile +from typing import Any, Optional, Union + +from pydantic import BaseModel, ValidationError + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily +from nodescraper.models import TaskResult +from nodescraper.models.datamodel import FileModel +from nodescraper.plugins.inband.amdsmi.amdsmi_collector import AmdSmiCollector +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiListItem, + AmdSmiMetric, + AmdSmiStatic, + AmdSmiVersion, + BadPages, + Fw, + Partition, + Processes, + Topo, + XgmiLinks, + XgmiMetrics, +) +from nodescraper.plugins.inband.amdsmi.collector_args import AmdSmiCollectorArgs +from nodescraper.utils import get_exception_details, get_exception_traceback + +from .amdsmidata_flavor import AmdSmiFlavorDataModel +from .amdsmidata_guest import GuestAmdSmiMetric, GuestAmdSmiStatic +from .amdsmidata_host import ( + HostDriverAmdSmiListItem, + HostDriverAmdSmiMetric, + HostDriverAmdSmiStatic, + HostDriverAmdSmiVersion, + HostDriverBadPages, + HostDriverTopo, + HostDriverXgmiLinks, + HostDriverXgmiMetrics, +) + +AMD_SMI_CPER_FOLDER = "/tmp/amd_smi_cper" + + +class AmdSmiFlavorCollector(AmdSmiCollector): + """amd-smi collector with host/guest driver-flavor support (incl. ESXi mxGPU).""" + + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.LINUX, OSFamily.ESXI} + DATA_MODEL = AmdSmiFlavorDataModel # type: ignore[assignment] + + # amd-smi runs with any of these drivers; the driver sets the CLI/JSON flavor. + _HOST_DRIVER_MODULES = ("gim", "amdgpuv") + _GUEST_DRIVER_MODULES = ("amdgpu",) + + # ---- driver-flavor detection -------------------------------------------------- + + def _get_loaded_gpu_driver(self) -> Optional[str]: + """Loaded AMD GPU driver module name (gim/amdgpuv/amdgpu), or None. Cached.""" + cached = getattr(self, "_loaded_gpu_driver_cache", None) + if cached is not None: + return cached + if self.system_info.os_family == OSFamily.ESXI: + cmd_ret = self._run_sut_cmd("vmkload_mod -l") + else: + cmd_ret = self._run_sut_cmd("lsmod", sudo=True) + if cmd_ret.exit_code != 0: + return None + known = self._HOST_DRIVER_MODULES + self._GUEST_DRIVER_MODULES + driver: Optional[str] = None + for line in cmd_ret.stdout.splitlines(): + module = line.strip().split()[0] if line.strip() else "" + if module in known: + driver = module + break + self._loaded_gpu_driver_cache = driver + return driver + + def _check_gpu_driver_loaded(self) -> bool: + """A driver is required for all amd-smi commands (including help/version).""" + return self._get_loaded_gpu_driver() is not None + + @property + def is_host_driver(self) -> bool: + """True for the mxGPU host driver (gim on Linux, amdgpuv on ESXi).""" + return self._get_loaded_gpu_driver() in self._HOST_DRIVER_MODULES + + def _is_virtualized(self) -> bool: + """True inside a virtualized guest (VM), via the CPU hypervisor flag. Cached.""" + cached = getattr(self, "_is_virtualized_cache", None) + if cached is not None: + return cached + if self.system_info.os_family == OSFamily.ESXI: + self._is_virtualized_cache = False + return False + cmd_ret = self._run_sut_cmd("grep -c hypervisor /proc/cpuinfo") + try: + value = int(cmd_ret.stdout.strip()) > 0 + except (ValueError, AttributeError): + value = False + self._is_virtualized_cache = value + return value + + @property + def is_guest_driver(self) -> bool: + """True for a guest-VF amdgpu (amdgpu inside a VM). Bare-metal amdgpu is not guest.""" + return not self.is_host_driver and self._is_virtualized() + + @property + def _amd_smi_needs_sudo(self) -> bool: + """amd-smi needs sudo on Linux (guest + mxGPU host); not on ESXi (no sudo binary).""" + return self.system_info.os_family != OSFamily.ESXI + + # ---- command execution -------------------------------------------------------- + + def _run_amd_smi(self, cmd: str, sudo: Optional[bool] = None) -> Optional[str]: + """Run amd-smi, elevating on Linux by default (ESXi needs no sudo).""" + if sudo is None: + sudo = self._amd_smi_needs_sudo + cmd_ret = self._run_sut_cmd(f"{self.AMD_SMI_EXE} {cmd}", sudo=sudo) + # Host amd-smi reports a benign "no data" status on stderr with rc=0 (e.g. + # bad-pages when none exist) — a valid empty result, not an error. + if cmd_ret.exit_code == 0 and "No data was found" in cmd_ret.stderr: + self._log_event( + category=EventCategory.APPLICATION, + description="amd-smi returned no data for command", + data={"command": cmd, "stderr": cmd_ret.stderr}, + priority=EventPriority.INFO, + ) + return None + if cmd_ret.stderr != "" or cmd_ret.exit_code != 0: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi command", + data={"command": cmd, "exit_code": cmd_ret.exit_code, "stderr": cmd_ret.stderr}, + priority=EventPriority.ERROR, + console_log=True, + ) + return None + return cmd_ret.stdout + + def _run_amd_smi_dict( + self, + cmd: str, + sudo: Optional[bool] = None, + raise_event: bool = True, + *, + patch_invalid_json: bool = False, + ) -> Union[dict, list, None]: + """Run an amd-smi command with --json and parse the output.""" + cmd += " --json" + cmd_ret = self._run_amd_smi(cmd, sudo=sudo) + if not cmd_ret: + return None + try: + if patch_invalid_json: + # amd-smi has been observed to emit invalid JSON ("]\n[") between records. + cmd_ret = cmd_ret.replace("]\n[", ",") + return json.loads(cmd_ret) + except json.JSONDecodeError as e: + if raise_event: + self._log_event( + category=EventCategory.APPLICATION, + description=f"Error parsing command: `{cmd}` json data", + data={"cmd": cmd, "exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + return None + + def _check_command_supported(self, command: str) -> bool: + """Log an INFO event when amd-smi help does not list the command.""" + if command not in getattr(self, "amd_smi_commands", set()): + self._log_event( + category=EventCategory.APPLICATION, + description=f"amd-smi does not support command: `{command}`", + priority=EventPriority.INFO, + ) + return False + return True + + def detect_amdsmi_commands(self) -> set[str]: + """Parse `amd-smi help` (host) / `amd-smi -h` (guest) for supported commands.""" + command_pattern = re.compile(r"^\s{4}([\w\-]+)\s", re.MULTILINE) + help_flag = "help" if self.is_host_driver else "-h" + help_output = self._run_amd_smi(help_flag) + if help_output is None: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi help command", + priority=EventPriority.ERROR, + console_log=True, + ) + return set() + return set(command_pattern.findall(help_output)) + + # ---- model builder ------------------------------------------------------------ + + def _build_flavor_model( + self, + model_class: type[BaseModel], + json_data: Union[dict, list, None], + *, + return_first: bool = False, + keep_key: Optional[str] = None, + ) -> Any: + """Validate raw amd-smi JSON into ``model_class`` (per-flavor class chosen by caller). + + keep_key drops rows missing that key (host list/topology enumerate non-GPU rows); + return_first collapses to a single record (e.g. version). + """ + if json_data is None: + self._log_event( + category=EventCategory.APPLICATION, + description=f"No data to build model: {model_class.__name__}", + priority=EventPriority.ERROR, + ) + return None + if isinstance(json_data, dict): + try: + return model_class.model_validate(json_data) + except ValidationError as e: + self._log_event( + category=EventCategory.APPLICATION, + description=f"Failed to build amd-smi model: {model_class.__name__}", + data=get_exception_details(e), + priority=EventPriority.WARNING, + ) + return None + + # Build per row so one unparseable entry doesn't discard the whole list. A + # guest amd-smi can enumerate a VF's secondary PCI functions (bdf .1-.7) as + # all-N/A rows alongside the real GPUs; skip those but keep the real ones. + validated: list = [] + skipped: list = [] + for item in json_data: + if not isinstance(item, dict): + self._log_event( + category=EventCategory.APPLICATION, + description=f"Invalid data type for amd-smi model: {model_class.__name__}", + data={"data_type": type(item).__name__}, + priority=EventPriority.WARNING, + ) + return None + if keep_key is not None and keep_key not in item: + continue + try: + validated.append(model_class.model_validate(item)) + except ValidationError as e: + skipped.append(get_exception_details(e)) + if skipped: + self._log_event( + category=EventCategory.APPLICATION, + description=( + f"Skipped {len(skipped)} unparseable {model_class.__name__} " + f"row(s); kept {len(validated)}" + ), + data={"first_error": skipped[0]}, + priority=EventPriority.WARNING, + ) + if return_first: + return validated[0] if validated else None + return validated + + # ---- raw-JSON getters (flavor-aware commands) -------------------------------- + + def _get_amdsmi_version(self) -> Union[AmdSmiVersion, HostDriverAmdSmiVersion, None]: # type: ignore[override] + """Pick the version model by flavor (host nests fields under a "version" key).""" + ret = self._run_amd_smi_dict("version") + model = HostDriverAmdSmiVersion if self.is_host_driver else AmdSmiVersion + return self._build_flavor_model(model, ret, return_first=True) + + def get_gpu_list(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("list"): + return None + return self._run_amd_smi_dict("list", patch_invalid_json=True) + + def get_process(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("process"): + return None + return self._run_amd_smi_dict("process") + + _PARTITION_COLUMNS = ( + "gpu_id", + "memory", + "accelerator_type", + "accelerator_profile_index", + "partition_id", + ) + + def get_partition(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("partition"): + return None + # Host-driver amd-smi has no --json for partition; parse the text table. + if self.is_host_driver: + return self._get_partition_host() + return self._run_amd_smi_dict("partition") + + def _get_partition_host(self) -> Optional[dict]: + """Parse host `partition -c` text (--json unsupported on gim/amdgpuv).""" + raw = self._run_amd_smi("partition -c") + if raw is None: + return None + lines = raw.splitlines() + header_index = None + for index, line in enumerate(lines): + columns = line.split() + if columns and columns[0] == "GPU": + header_index = index + break + if header_index is None: + return None + current_partitions = [] + for line in lines[header_index + 1 :]: + columns = line.split() + is_data_row = len(columns) >= len(self._PARTITION_COLUMNS) and columns[0].isdigit() + if not is_data_row: + break + partition: dict[str, object] = dict(zip(self._PARTITION_COLUMNS, columns)) + partition["gpu_id"] = int(str(partition["gpu_id"])) + current_partitions.append(partition) + if not current_partitions: + return None + return {"current_partition": current_partitions} + + def get_topology(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("topology"): + return None + return self._run_amd_smi_dict("topology") + + def get_static(self) -> Union[list, None]: + if not self._check_command_supported("static"): + return None + # Host-driver amd-smi does not accept "-g all". + static_sub_cmd = "static" if self.is_host_driver else "static -g all" + static_data = self._run_amd_smi_dict(static_sub_cmd) + if static_data is None: + return None + if isinstance(static_data, dict) and "gpu_data" in static_data: + static_data = static_data["gpu_data"] + return [s for s in static_data if isinstance(s, dict) and "gpu" in s] + + def get_metric(self) -> Union[list, None]: + if not self._check_command_supported("metric"): + return None + metric_sub_cmd = "metric" if self.is_host_driver else "metric -g all" + metric_data = self._run_amd_smi_dict(metric_sub_cmd) + if metric_data is None: + return None + if isinstance(metric_data, dict) and "gpu_data" in metric_data: + metric_data = metric_data["gpu_data"] + return [m for m in metric_data if isinstance(m, dict) and "gpu" in m] + + def get_firmware(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("firmware"): + return None + return self._run_amd_smi_dict("firmware") + + def get_bad_pages(self) -> Union[dict, list, None]: # type: ignore[override] + if not self._check_command_supported("bad-pages"): + return None + return self._run_amd_smi_dict("bad-pages") + + def get_xgmi_data_metric(self) -> Optional[dict]: + """Fetch xgmi metric + link data (host uses --metric/--link-status vs guest -m/-l).""" + if not self._check_command_supported("xgmi"): + return None + metric_flag = "--metric" if self.is_host_driver else "-m" + xgmi_metric_data = self._run_amd_smi_dict(f"xgmi {metric_flag}") + if xgmi_metric_data is None: + xgmi_metric_data = [] + elif isinstance(xgmi_metric_data, dict) and "xgmi_metric" in xgmi_metric_data: + xgmi_metric_data = xgmi_metric_data["xgmi_metric"] + if isinstance(xgmi_metric_data, list) and len(xgmi_metric_data) == 1: + xgmi_metric_data = xgmi_metric_data[0] + + link_flag = "--link-status" if self.is_host_driver else "-l" + xgmi_link_data = self._run_amd_smi_dict(f"xgmi {link_flag}", raise_event=False) + if isinstance(xgmi_link_data, dict) and "link_status" in xgmi_link_data: + xgmi_link_data = xgmi_link_data["link_status"] + if xgmi_link_data is None: + xgmi_link_data = [] + return {"metric": xgmi_metric_data, "link": xgmi_link_data} + + def get_cper_data(self) -> tuple[list[FileModel], dict[str, int]]: + """Collect CPER files. Host amd-smi requires --severity; guest/bare-metal do not.""" + if not self._check_command_supported("ras"): + return [], {} + self._run_sut_cmd( + f"mkdir -p {AMD_SMI_CPER_FOLDER} && rm -f {AMD_SMI_CPER_FOLDER}/*.cper " + f"&& rm -f {AMD_SMI_CPER_FOLDER}/*.json", + sudo=False, + ) + if self.is_host_driver: + cper_sub_cmd = f"ras --cper --severity=all --folder={AMD_SMI_CPER_FOLDER}" + else: + cper_sub_cmd = f"ras --cper --folder={AMD_SMI_CPER_FOLDER}" + cper_out = self._run_amd_smi(cper_sub_cmd) + if cper_out is None or not re.findall(r"(\w+\.cper)", cper_out): + return [], {} + self._run_sut_cmd( + f"tar -czf {AMD_SMI_CPER_FOLDER}.tar.gz -C {AMD_SMI_CPER_FOLDER} .", + sudo=self._amd_smi_needs_sudo, + ) + cper_zip = self._read_sut_file( + f"{AMD_SMI_CPER_FOLDER}.tar.gz", encoding=None, strip=False, log_artifact=True + ) + if not hasattr(cper_zip, "contents"): + return [], {} + io_bytes = io.BytesIO(cper_zip.contents) # type: ignore[attr-defined] + cper_data: list[FileModel] = [] + cper_afids: dict[str, int] = {} + try: + with TarFile.open(fileobj=io_bytes, mode="r:gz") as tar_file: + for member in tar_file.getmembers(): + if not (member.isfile() and member.name.endswith(".cper")): + continue + content = tar_file.extractfile(member) + file_bytes = content.read() if content is not None else b"" + cper_data.append(FileModel(file_contents=file_bytes, file_name=member.name)) + afid = self._get_cper_afid(f"{AMD_SMI_CPER_FOLDER}/{member.name}") + if afid is not None: + cper_afids[member.name] = afid + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Error extracting cper data", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + return [], {} + if cper_data: + self._log_event( + category=EventCategory.APPLICATION, + description="CPER data has been extracted from amd-smi", + data={"cper_count": len(cper_data), "afid_count": len(cper_afids)}, + priority=EventPriority.INFO, + ) + return cper_data, cper_afids + + # ---- orchestration ------------------------------------------------------------ + + def _get_amdsmi_data( + self, args: Optional[AmdSmiCollectorArgs] = None + ) -> Optional[AmdSmiFlavorDataModel]: + """Fetch all amd-smi sub-data and build the model with per-flavor variants.""" + try: + version = self._get_amdsmi_version() + gpu_list = self.get_gpu_list() + processes = self.get_process() + partition = self.get_partition() + firmware = self.get_firmware() + topology = self.get_topology() + amdsmi_static = self.get_static() + amdsmi_metric = self.get_metric() + bad_pages = self.get_bad_pages() + xgmi = self.get_xgmi_data_metric() or {"metric": [], "link": []} + cper_data, cper_afids = self.get_cper_data() + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi sub commands", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return None + + host = self.is_host_driver + guest = self.is_guest_driver + + # list / topology / bad-pages / xgmi split host vs non-host. + if host: + list_item_cls: type[BaseModel] = HostDriverAmdSmiListItem + topo_cls: type[BaseModel] = HostDriverTopo + bad_pages_cls: type[BaseModel] = HostDriverBadPages + xgmi_metric_cls: type[BaseModel] = HostDriverXgmiMetrics + xgmi_link_cls: type[BaseModel] = HostDriverXgmiLinks + else: + list_item_cls = AmdSmiListItem + topo_cls = Topo + bad_pages_cls = BadPages + xgmi_metric_cls = XgmiMetrics + xgmi_link_cls = XgmiLinks + + # static / metric are three-way (a guest VF omits physical fields). + if host: + static_cls: type[BaseModel] = HostDriverAmdSmiStatic + metric_cls: type[BaseModel] = HostDriverAmdSmiMetric + elif guest: + static_cls = GuestAmdSmiStatic + metric_cls = GuestAmdSmiMetric + else: + static_cls = AmdSmiStatic + metric_cls = AmdSmiMetric + + gpu_list_model = self._build_flavor_model(list_item_cls, gpu_list, keep_key="gpu") + topo_model = self._build_flavor_model(topo_cls, topology, keep_key="gpu") + bad_pages_model = self._build_flavor_model(bad_pages_cls, bad_pages) if bad_pages else [] + partition_model = self._build_flavor_model(Partition, partition) + # Host-driver amd-smi has no `process` command; avoid a spurious error. + process_model = self._build_flavor_model(Processes, processes) if processes else [] + firmware_model = self._build_flavor_model(Fw, firmware) + static_model = self._build_flavor_model(static_cls, amdsmi_static) + metric_model = self._build_flavor_model(metric_cls, amdsmi_metric) + xgmi_metric_model = self._build_flavor_model(xgmi_metric_cls, xgmi["metric"]) + xgmi_link_model = self._build_flavor_model(xgmi_link_cls, xgmi["link"]) + + try: + fw_ids = args.analysis_firmware_ids if args and args.analysis_firmware_ids else None + base = AmdSmiFlavorDataModel( + version=version, + gpu_list=gpu_list_model or [], + process=process_model or [], + partition=partition_model, + firmware=firmware_model or [], + static=static_model or [], + topology=topo_model or [], + metric=metric_model or [], + bad_pages=bad_pages_model or [], + xgmi_metric=xgmi_metric_model or [], + xgmi_link=xgmi_link_model or [], + cper_data=cper_data, + cper_afids=cper_afids, + analysis_firmware_ids=fw_ids, + analysis_ref=None, + ) + return base.model_copy(update={"analysis_ref": base.build_analysis_ref()}) + except ValidationError as err: + self._log_event( + category=EventCategory.APPLICATION, + description="Failed to build AmdSmiFlavorDataModel", + data={"errors": err.errors(include_url=False)}, + priority=EventPriority.ERROR, + ) + return None + + def collect_data( + self, args: Optional[AmdSmiCollectorArgs] = None + ) -> tuple[TaskResult, Optional[AmdSmiFlavorDataModel]]: + """Collect amd-smi data across driver flavors (guest/bare-metal/host, incl. ESXi).""" + if not self._check_amdsmi_installed() or not self._check_gpu_driver_loaded(): + self._log_event( + category=EventCategory.APPLICATION, + description="amd-smi not installed or no AMD GPU driver loaded", + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.NOT_RAN + return self.result, None + try: + self.amd_smi_commands = self.detect_amdsmi_commands() + amd_smi_data = self._get_amdsmi_data(args) + return self.result, amd_smi_data + except Exception as e: + self._log_event( + category=EventCategory.APPLICATION, + description="Error running amd-smi collector", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result, None diff --git a/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py new file mode 100644 index 00000000..18d46235 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmi_flavor_plugin.py @@ -0,0 +1,18 @@ +from .amdsmi_flavor_collector import AmdSmiFlavorCollector +from .amdsmi_plugin import AmdSmiPlugin +from .amdsmidata_flavor import AmdSmiFlavorDataModel + + +class AmdSmiFlavorPlugin(AmdSmiPlugin): + """amd-smi plugin variant with driver-flavor collection. + + Detects the loaded driver flavor and issues flavor-appropriate amd-smi commands: + - mxGPU host driver (gim on Linux, amdgpuv on ESXi) -> HostDriver* models + - guest amdgpu on a virtual function inside a VM -> Guest* models + - bare-metal amdgpu on physical hardware -> base AmdSmi* models + Data is built into AmdSmiFlavorDataModel. The base AmdSmiPlugin (guest/bare-metal + only) is left unchanged for callers that do not need host/VF support. + """ + + DATA_MODEL = AmdSmiFlavorDataModel # type: ignore[assignment] + COLLECTOR = AmdSmiFlavorCollector diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata.py b/nodescraper/plugins/inband/amdsmi/amdsmidata.py index 3b8aae3c..9b43cf00 100644 --- a/nodescraper/plugins/inband/amdsmi/amdsmidata.py +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata.py @@ -25,7 +25,7 @@ ############################################################################### import re from enum import Enum -from typing import Any, Mapping, Optional, Union +from typing import Any, ClassVar, Literal, Mapping, Optional, Union from pydantic import ( AliasChoices, @@ -40,11 +40,13 @@ from nodescraper.models.datamodel import DataModel, FileModel from nodescraper.utils import find_annotation_in_container +NA_STR = "N/A" +NA_LITERAL = Literal["N/A"] _NUM_UNIT_RE = re.compile(r"^\s*([-+]?\d+(?:\.\d+)?)(?:\s*([A-Za-z%/][A-Za-z0-9%/._-]*))?\s*$") def _value_unit_is_na(x: Any) -> bool: - return x is None or (isinstance(x, str) and x.strip().upper() in {"N/A", "NA", ""}) + return x is None or (isinstance(x, str) and x.strip().upper() in {NA_STR, "NA", ""}) def _coerce_value_unit_raw(v: Any) -> Any: @@ -81,7 +83,7 @@ def _coerce_value_unit_raw(v: Any) -> Any: def na_to_none(values: Union[int, str]): - if values == "N/A": + if values == NA_STR: return None return values @@ -89,7 +91,7 @@ def na_to_none(values: Union[int, str]): def na_to_none_list(values: list[Union[int, str, None]]) -> list[Union[int, str, None]]: ret_list: list[Union[int, str, None]] = values.copy() for i in range(len(ret_list)): - if ret_list[i] == "N/A": + if ret_list[i] == NA_STR: ret_list[i] = None return ret_list @@ -99,14 +101,14 @@ def na_to_none_dict(values: object) -> Optional[dict[str, Any]]: Accepts None; returns None for 'N/A'/'NA'/'' or non-mapping inputs.""" if values is None: return None - if isinstance(values, str) and values.strip().upper() in {"N/A", "NA", ""}: + if isinstance(values, str) and values.strip().upper() in {NA_STR, "NA", ""}: return None if not isinstance(values, Mapping): return None out: dict[str, Any] = {} for k, v in values.items(): - if isinstance(v, str) and v.strip().upper() in {"N/A", "NA", ""}: + if isinstance(v, str) and v.strip().upper() in {NA_STR, "NA", ""}: out[k] = None else: out[k] = v @@ -183,6 +185,29 @@ def coerce_value_unit_input(v: Any) -> Any: return _coerce_value_unit_raw(v) +_SPACED_UNIT_RE = re.compile(r"^\s*([-+]?\d+(?:\.\d+)?)\s+(.+?)\s*$") + + +def coerce_spaced_unit_value_unit_input(v: Any) -> Any: + """Similar to ``coerce_value_unit_input``, but for units containing a space + (e.g. amd-smi's ``"0.0 Degrees C"``). + + ``_NUM_UNIT_RE`` disallows whitespace inside the unit portion + so a multi-word unit falls through to being stored verbatim as + ``value``. + """ + if v is None: + return None + if isinstance(v, ValueUnit): + return v + if isinstance(v, str) and not _value_unit_is_na(v): + m = _SPACED_UNIT_RE.match(v.strip()) + if m: + num, unit = m.groups() + return {"value": float(num) if "." in num else int(num), "unit": unit} + return coerce_value_unit_input(v) + + # Process class ProcessMemoryUsage(BaseModel): gtt_mem: Optional[ValueUnit] @@ -216,7 +241,7 @@ class EccState(Enum): SING_C = "SING_C" MULT_UC = "MULT_UC" POISON = "POISON" - NA = "N/A" + NA = NA_STR class ProcessListItem(BaseModel): @@ -418,13 +443,13 @@ def _normalize_static_ppt_block(val: object) -> object: """Drop N/A-only ``ppt0``/``ppt1`` blocks; pass through already-parsed models.""" if val is None: return None - if isinstance(val, str) and val.strip().upper() in {"N/A", "NA", ""}: + if isinstance(val, str) and val.strip().upper() in {NA_STR, "NA", ""}: return None if not isinstance(val, dict): return val out: dict[str, Any] = {} for key, raw in val.items(): - if isinstance(raw, str) and raw.strip().upper() in {"N/A", "NA", ""}: + if isinstance(raw, str) and raw.strip().upper() in {NA_STR, "NA", ""}: continue if raw is not None: out[key] = raw @@ -498,7 +523,9 @@ class StaticXgmiPlpd(BaseModel): class StaticNuma(BaseModel): - node: int + model_config = ConfigDict(extra="allow") + + node: Union[int, str] # can be N/A affinity: Union[int, str] # can be N/A @@ -974,6 +1001,249 @@ class EccData(BaseModel): )(na_to_none) +def _match_known_rails_case_insensitive(rail_set: frozenset[str], data: Any) -> Any: + """Rename dict keys to their canonical-cased rail name when their upper-cased + form matches an entry in ``rail_set`` (the known, already-uppercase rail names).""" + if not isinstance(data, dict): + return data + out = dict(data) + for key in list(out.keys()): + upper_key = key.upper() + if upper_key in rail_set and upper_key not in out: + out[upper_key] = out.pop(key) + return out + + +class MetricClockRails(BaseModel): + """Enumerated, typed view of the known ``AmdSmiMetric.clock`` rails.""" + + model_config = ConfigDict(extra="allow") + + RAIL_NAMES: ClassVar[tuple[str, ...]] = ( + "GFX_0", + "GFX_1", + "GFX_2", + "GFX_3", + "GFX_4", + "GFX_5", + "GFX_6", + "GFX_7", + "MEM_0", + "VCLK_0", + "VCLK_1", + "VCLK_2", + "VCLK_3", + "DCLK_0", + "DCLK_1", + "DCLK_2", + "DCLK_3", + "SOCCLK_0", + "FCLK_0", + "UCLK_AID", + "SOCCLKS_MID", + ) + _RAIL_SET: ClassVar[frozenset[str]] = frozenset(RAIL_NAMES) + + GFX_0: Optional[MetricClockData] = None + GFX_1: Optional[MetricClockData] = None + GFX_2: Optional[MetricClockData] = None + GFX_3: Optional[MetricClockData] = None + GFX_4: Optional[MetricClockData] = None + GFX_5: Optional[MetricClockData] = None + GFX_6: Optional[MetricClockData] = None + GFX_7: Optional[MetricClockData] = None + MEM_0: Optional[MetricClockData] = None + VCLK_0: Optional[MetricClockData] = None + VCLK_1: Optional[MetricClockData] = None + VCLK_2: Optional[MetricClockData] = None + VCLK_3: Optional[MetricClockData] = None + DCLK_0: Optional[MetricClockData] = None + DCLK_1: Optional[MetricClockData] = None + DCLK_2: Optional[MetricClockData] = None + DCLK_3: Optional[MetricClockData] = None + SOCCLK_0: Optional[MetricClockData] = None + FCLK_0: Optional[MetricClockData] = None + UCLK_AID: Optional[dict[str, Optional[ValueUnit]]] = None + SOCCLKS_MID: Optional[dict[str, Optional[ValueUnit]]] = None + + @model_validator(mode="before") + @classmethod + def _match_rails(cls, data: Any) -> Any: + return _match_known_rails_case_insensitive(cls._RAIL_SET, data) + + @field_validator("UCLK_AID", "SOCCLKS_MID", mode="before") + @classmethod + def _coerce_submap(cls, v: Any) -> Any: + if not isinstance(v, dict): + return v + return {k: coerce_value_unit_input(val) for k, val in v.items()} + + +class MetricGpuBoardTemperature(BaseModel): + """Known ``gpu_board.temperature`` rails (ROCm 7.1+ ``metric`` JSON). + + Each rail is independently optional and N/A-tolerant. + """ + + model_config = ConfigDict(extra="allow") + + RAIL_NAMES: ClassVar[tuple[str, ...]] = ( + "NODE_RETIMER_X", + "NODE_OAM_X_IBC", + "NODE_OAM_X_IBC_2", + "NODE_OAM_X_VDD18_VR", + "NODE_OAM_X_04_HBM_B_VR", + "NODE_OAM_X_04_HBM_D_VR", + "VDDCR_VDD0", + "VDDCR_VDD1", + "VDDCR_VDD2", + "VDDCR_VDD3", + "VDDCR_SOC_A", + "VDDCR_SOC_C", + "VDDCR_SOCIO_A", + "VDDCR_SOCIO_C", + "VDD_085_HBM", + "VDDCR_11_HBM_B", + "VDDCR_11_HBM_D", + "VDD_USR", + "VDDIO_11_E32", + "VDDIO_04_HBM_B", + "VDDIO_04_HBM_D", + "VDDCR_075_HBM_B", + "VDDCR_075_HBM_D", + "VDDIO_11_GTA_A", + "VDDIO_11_GTA_C", + "VDDAN_075_GTA_A", + "VDDAN_075_GTA_C", + "VDDCR_075_UCIE", + "VDDIO_065_UCIEAA", + "VDDIO_065_UCIEAM_A", + "VDDIO_065_UCIEAM_C", + "VDDAN_075", + ) + _RAIL_SET: ClassVar[frozenset[str]] = frozenset(RAIL_NAMES) + + NODE_RETIMER_X: Optional[ValueUnit] = None + NODE_OAM_X_IBC: Optional[ValueUnit] = None + NODE_OAM_X_IBC_2: Optional[ValueUnit] = None + NODE_OAM_X_VDD18_VR: Optional[ValueUnit] = None + NODE_OAM_X_04_HBM_B_VR: Optional[ValueUnit] = None + NODE_OAM_X_04_HBM_D_VR: Optional[ValueUnit] = None + VDDCR_VDD0: Optional[ValueUnit] = None + VDDCR_VDD1: Optional[ValueUnit] = None + VDDCR_VDD2: Optional[ValueUnit] = None + VDDCR_VDD3: Optional[ValueUnit] = None + VDDCR_SOC_A: Optional[ValueUnit] = None + VDDCR_SOC_C: Optional[ValueUnit] = None + VDDCR_SOCIO_A: Optional[ValueUnit] = None + VDDCR_SOCIO_C: Optional[ValueUnit] = None + VDD_085_HBM: Optional[ValueUnit] = None + VDDCR_11_HBM_B: Optional[ValueUnit] = None + VDDCR_11_HBM_D: Optional[ValueUnit] = None + VDD_USR: Optional[ValueUnit] = None + VDDIO_11_E32: Optional[ValueUnit] = None + VDDIO_04_HBM_B: Optional[ValueUnit] = None + VDDIO_04_HBM_D: Optional[ValueUnit] = None + VDDCR_075_HBM_B: Optional[ValueUnit] = None + VDDCR_075_HBM_D: Optional[ValueUnit] = None + VDDIO_11_GTA_A: Optional[ValueUnit] = None + VDDIO_11_GTA_C: Optional[ValueUnit] = None + VDDAN_075_GTA_A: Optional[ValueUnit] = None + VDDAN_075_GTA_C: Optional[ValueUnit] = None + VDDCR_075_UCIE: Optional[ValueUnit] = None + VDDIO_065_UCIEAA: Optional[ValueUnit] = None + VDDIO_065_UCIEAM_A: Optional[ValueUnit] = None + VDDIO_065_UCIEAM_C: Optional[ValueUnit] = None + VDDAN_075: Optional[ValueUnit] = None + + _rail_value_unit = field_validator(*RAIL_NAMES, mode="before")(coerce_value_unit_input) + + @model_validator(mode="before") + @classmethod + def _match_rails(cls, data: Any) -> Any: + return _match_known_rails_case_insensitive(cls._RAIL_SET, data) + + +class MetricGpuBoard(BaseModel): + model_config = ConfigDict(extra="allow") + + temperature: Optional[MetricGpuBoardTemperature] = None + + na_validator = field_validator("temperature", mode="before")(na_to_none) + + +class MetricBaseBoardTemperature(BaseModel): + """Known ``base_board.temperature`` (UBB) rails (ROCm 7.1+ ``metric`` JSON).""" + + model_config = ConfigDict(extra="allow") + + RAIL_NAMES: ClassVar[tuple[str, ...]] = ( + "UBB_FPGA", + "UBB_FRONT", + "UBB_BACK", + "UBB_OAM1", + "UBB_OAM7", + "UBB_IBC", + "UBB_UFPGA", + "UBB_FPGA_0V72_VR", + "UBB_FPGA_3V3_VR", + "OAM_0_1_HSC", + "OAM_2_3_HSC", + "OAM_4_5_HSC", + "OAM_6_7_HSC", + "OAM_0_1_2_3_3V3_VR", + "OAM_4_5_6_7_3V3_VR", + "IBC", + "IBC_HSC", + "RETIMER_0_1_0V9_VR", + "RETIMER_2_3_0V9_VR", + "RETIMER_4_5_0V9_VR", + "RETIMER_6_7_0V9_VR", + "RETIMER_0_1_2_3_1V2_VR", + "RETIMER_4_5_6_7_1V2_VR", + ) + _RAIL_SET: ClassVar[frozenset[str]] = frozenset(RAIL_NAMES) + + UBB_FPGA: Optional[ValueUnit] = None + UBB_FRONT: Optional[ValueUnit] = None + UBB_BACK: Optional[ValueUnit] = None + UBB_OAM1: Optional[ValueUnit] = None + UBB_OAM7: Optional[ValueUnit] = None + UBB_IBC: Optional[ValueUnit] = None + UBB_UFPGA: Optional[ValueUnit] = None + UBB_FPGA_0V72_VR: Optional[ValueUnit] = None + UBB_FPGA_3V3_VR: Optional[ValueUnit] = None + OAM_0_1_HSC: Optional[ValueUnit] = None + OAM_2_3_HSC: Optional[ValueUnit] = None + OAM_4_5_HSC: Optional[ValueUnit] = None + OAM_6_7_HSC: Optional[ValueUnit] = None + OAM_0_1_2_3_3V3_VR: Optional[ValueUnit] = None + OAM_4_5_6_7_3V3_VR: Optional[ValueUnit] = None + IBC: Optional[ValueUnit] = None + IBC_HSC: Optional[ValueUnit] = None + RETIMER_0_1_0V9_VR: Optional[ValueUnit] = None + RETIMER_2_3_0V9_VR: Optional[ValueUnit] = None + RETIMER_4_5_0V9_VR: Optional[ValueUnit] = None + RETIMER_6_7_0V9_VR: Optional[ValueUnit] = None + RETIMER_0_1_2_3_1V2_VR: Optional[ValueUnit] = None + RETIMER_4_5_6_7_1V2_VR: Optional[ValueUnit] = None + + _rail_value_unit = field_validator(*RAIL_NAMES, mode="before")(coerce_value_unit_input) + + @model_validator(mode="before") + @classmethod + def _match_rails(cls, data: Any) -> Any: + return _match_known_rails_case_insensitive(cls._RAIL_SET, data) + + +class MetricBaseBoard(BaseModel): + model_config = ConfigDict(extra="allow") + + temperature: Optional[MetricBaseBoardTemperature] = None + + na_validator = field_validator("temperature", mode="before")(na_to_none) + + class AmdSmiMetric(BaseModel): model_config = ConfigDict(extra="ignore", populate_by_name=True) @@ -992,11 +1262,11 @@ class AmdSmiMetric(BaseModel): energy: Optional[MetricEnergy] mem_usage: MetricMemUsage throttle: MetricThrottle - gpuboard: Optional[Union[dict[str, Any], str]] = Field( + gpuboard: Optional[Union[MetricGpuBoard, str]] = Field( default=None, validation_alias=AliasChoices("gpuboard", "gpu_board"), ) - baseboard: Optional[Union[dict[str, Any], str]] = Field( + baseboard: Optional[Union[MetricBaseBoard, str]] = Field( default=None, validation_alias=AliasChoices("baseboard", "base_board"), ) @@ -1009,12 +1279,32 @@ class AmdSmiMetric(BaseModel): def _normalize_clock(cls, clock: Any) -> Any: return _normalize_metric_clock_map(clock) + @computed_field + def clock_rails(self) -> MetricClockRails: + """Enumerated view of clock catching a dropped/renamed known rail.""" + return MetricClockRails.model_validate(self.clock or {}) + + @model_validator(mode="before") + @classmethod + def _normalize_metric_input(cls, value: object) -> object: + """Normalize one metric object before field validation. + + Args: + value: Raw metric input. + + Returns: + The input, with per-block clocks and the PCIe count alias normalized. + """ + if isinstance(value, dict): + return normalize_amdsmi_metric_dict(value) + return value + @field_validator("ecc_blocks", mode="before") @classmethod def validate_ecc_blocks(cls, value: Union[dict[str, EccData], str]) -> dict[str, EccData]: """Validate the ecc_blocks field.""" if isinstance(value, str): - # If it's a string, we assume it's "N/A" and return an empty dict + # If it's a string, we assume it's NA_STR and return an empty dict return {} return value @@ -1022,7 +1312,7 @@ def validate_ecc_blocks(cls, value: Union[dict[str, EccData], str]) -> dict[str, @classmethod def validate_energy(cls, value: Optional[Any]) -> Optional[MetricEnergy]: """Validate the energy field.""" - if value == "N/A" or value is None: + if value == NA_STR or value is None: return None return value @@ -1035,6 +1325,7 @@ class LinkStatusTable(Enum): DOWN = "D" DISABLED = "X" SELF = "SELF" + NA = NA_STR class BiDirectionalTable(Enum): @@ -1066,6 +1357,8 @@ class AccessTable(Enum): # XGMI + + class XgmiLink(BaseModel): gpu: int bdf: str @@ -1144,6 +1437,473 @@ class Topo(BaseModel): links: list[TopoLink] +# FABRIC + + +class FabricInfo(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + bdf: Optional[str] = None + version: Optional[int] = None + accelerator_id: Optional[int] = None + fabric_type: Optional[str] = None + bandwidth: Optional[ValueUnit] = None + latency: Optional[ValueUnit] = None + ppod_id: Optional[str] = None + ppod_size: Optional[int] = None + vpod_id: Optional[int] = None + vpod_size: Optional[int] = None + local_accelerators: Optional[Union[str, list[str]]] = None + local_active_accelerators: Optional[list[str]] = None + addr_mode: Optional[str] = None + accel_state: Optional[str] = None + + na_validator = field_validator( + "bdf", + "version", + "accelerator_id", + "fabric_type", + "ppod_id", + "ppod_size", + "vpod_id", + "vpod_size", + "local_accelerators", + "local_active_accelerators", + "addr_mode", + "accel_state", + mode="before", + )(na_to_none) + + value_unit_validator = field_validator("bandwidth", "latency", mode="before")( + coerce_value_unit_input + ) + + +class Fabric(BaseModel): + """Per GPU fabric data from ``amd-smi fabric``. + + ``fabric_telemetry`` is intentionally not collected for now as it is very large. + """ + + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + gpu: int + bdf: Optional[str] = None + fabric_info: Optional[FabricInfo] = None + + na_validator = field_validator("bdf", "fabric_info", mode="before")(na_to_none) + + +# NODE + + +class NodePowerManagement(BaseModel): + """``node[].node.power_management`` (NODE_TOTAL_POWER).""" + + model_config = ConfigDict(extra="allow") + + limit: Optional[ValueUnit] = None + status: Optional[str] = None + threshold: Optional[ValueUnit] = None + + na_validator = field_validator("status", mode="before")(na_to_none) + _value_unit = field_validator("limit", "threshold", mode="before")(coerce_value_unit_input) + + +class NodeGtt(BaseModel): + """``node[].node.gtt`` (GTT_SIZE). Plain numbers, not value+unit dicts.""" + + model_config = ConfigDict(extra="allow") + + size_gb: Optional[float] = None + size_pages: Optional[float] = None + + na_validator = field_validator("size_gb", "size_pages", mode="before")(na_to_none) + + +class NodeBaseBoard(BaseModel): + """``node[].node.base_board`` (named chassis temperature sensors).""" + + model_config = ConfigDict(extra="allow") + + temperature: dict[str, ValueUnit] = Field(default_factory=dict) + + _value_unit = field_validator("temperature", mode="before")( + lambda v: ( + {k: coerce_value_unit_input(val) for k, val in v.items()} if isinstance(v, dict) else v + ) + ) + + +class NodeData(BaseModel): + model_config = ConfigDict(extra="allow") + + power_management: Optional[NodePowerManagement] = None + base_board: Optional[NodeBaseBoard] = None + gtt: Optional[NodeGtt] = None + + +class NodeInfo(BaseModel): + """One entry from ``amd-smi node --json``: a top-level array of + ``{"node": {...}}`` objects.""" + + model_config = ConfigDict(extra="allow") + + node: Optional[NodeData] = None + + +# CPU / CORE + + +class CoreMetric(BaseModel): + """Per-core data from ``amd-smi metric --core all --json``.""" + + model_config = ConfigDict(extra="allow") + + core: int + boost_limit: Optional[ValueUnit] = None + curr_active_freq_core_limit: Optional[ValueUnit] = None + core_energy: Optional[ValueUnit] = None + ccd_power: Optional[ValueUnit] = None + floor_limit: Optional[ValueUnit] = None + eff_floor_limit: Optional[ValueUnit] = None + + _value_unit = field_validator( + "boost_limit", + "curr_active_freq_core_limit", + "core_energy", + "ccd_power", + "floor_limit", + "eff_floor_limit", + mode="before", + )(coerce_value_unit_input) + + +class CpuMetricPower(BaseModel): + """``power_metrics`` group; JSON keys have literal spaces, not underscores.""" + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + socket_power: Optional[ValueUnit] = Field(default=None, alias="socket power") + socket_power_limit: Optional[ValueUnit] = Field(default=None, alias="socket power limit") + socket_max_power_limit: Optional[ValueUnit] = Field( + default=None, alias="socket max power limit" + ) + + _value_unit = field_validator( + "socket_power", "socket_power_limit", "socket_max_power_limit", mode="before" + )(coerce_value_unit_input) + + +class CpuMetricProchot(BaseModel): + model_config = ConfigDict(extra="allow") + + prochot_status: Optional[int] = None + + +class CpuMetricFreqFclkMemclk(BaseModel): + model_config = ConfigDict(extra="allow") + + fclk: Optional[ValueUnit] = None + mclk: Optional[ValueUnit] = None + + _value_unit = field_validator("fclk", "mclk", mode="before")(coerce_value_unit_input) + + +class CpuMetricFreqActiveLimit(BaseModel): + model_config = ConfigDict(extra="allow") + + freq: Optional[ValueUnit] = None + freq_src: Optional[str] = None + + _value_unit = field_validator("freq", mode="before")(coerce_value_unit_input) + + +class CpuMetricFreqRange(BaseModel): + model_config = ConfigDict(extra="allow") + + max_socket_freq: Optional[ValueUnit] = None + min_socket_freq: Optional[ValueUnit] = None + + _value_unit = field_validator("max_socket_freq", "min_socket_freq", mode="before")( + coerce_value_unit_input + ) + + +class CpuMetricFreq(BaseModel): + """``freq_metrics`` group""" + + model_config = ConfigDict(extra="allow") + + fclkmemclk: Optional[CpuMetricFreqFclkMemclk] = None + cclkfreqlimit: Optional[ValueUnit] = None + soc_current_active_freq_limit: Optional[CpuMetricFreqActiveLimit] = None + soc_freq_range: Optional[CpuMetricFreqRange] = None + + _value_unit = field_validator("cclkfreqlimit", mode="before")(coerce_value_unit_input) + + +class CpuMetricC0Residency(BaseModel): + model_config = ConfigDict(extra="allow") + + residency: Optional[ValueUnit] = None + + _value_unit = field_validator("residency", mode="before")(coerce_value_unit_input) + + +class CpuMetricSviTelemetry(BaseModel): + model_config = ConfigDict(extra="allow") + + power: Optional[ValueUnit] = None + + _value_unit = field_validator("power", mode="before")(coerce_value_unit_input) + + +class CpuMetricPwrEffMode(BaseModel): + model_config = ConfigDict(extra="allow") + + mode: Optional[str] = None + + +class CpuMetricVersion(BaseModel): + model_config = ConfigDict(extra="allow") + + version: Optional[int] = None + + +class CpuMetricTable(BaseModel): + model_config = ConfigDict(extra="allow") + + cpu_family: Optional[int] = None + cpu_model: Optional[int] = None + response: Optional[ValueUnit] = None + + _value_unit = field_validator("response", mode="before")(coerce_value_unit_input) + + +class CpuMetricSocketEnergy(BaseModel): + model_config = ConfigDict(extra="allow") + + response: Optional[ValueUnit] = None + + _value_unit = field_validator("response", mode="before")(coerce_value_unit_input) + + +class CpuMetricDdrBandwidthResponse(BaseModel): + model_config = ConfigDict(extra="allow") + + ddr_bw_max_bw: Optional[ValueUnit] = None + ddr_bw_utilized_bw: Optional[ValueUnit] = None + ddr_bw_utilized_pct: Optional[ValueUnit] = None + + _value_unit = field_validator( + "ddr_bw_max_bw", "ddr_bw_utilized_bw", "ddr_bw_utilized_pct", mode="before" + )(coerce_value_unit_input) + + +class CpuMetricDdrBandwidth(BaseModel): + """``ddr_bandwidth`` nests its payload one level deeper than the other + ``response``-wrapped groups""" + + model_config = ConfigDict(extra="allow") + + response: Optional[CpuMetricDdrBandwidthResponse] = None + + na_validator = field_validator("response", mode="before")(na_to_none) + + +class CpuMetricCpuTemp(BaseModel): + model_config = ConfigDict(extra="allow") + + response: Optional[ValueUnit] = None + + _value_unit = field_validator("response", mode="before")(coerce_value_unit_input) + + +class CpuMetricXgmiPstateRange(BaseModel): + model_config = ConfigDict(extra="allow") + + min_pstate: Optional[int] = None + max_pstate: Optional[int] = None + + na_validator = field_validator("min_pstate", "max_pstate", mode="before")(na_to_none) + + +class CpuMetricEnabledCommands(BaseModel): + model_config = ConfigDict(extra="allow") + + READ_ENABLED_COMMANDS_BITMASK0: Optional[Union[int, str]] = None + READ_ENABLED_COMMANDS_BITMASK1: Optional[Union[int, str]] = None + READ_ENABLED_COMMANDS_BITMASK2: Optional[Union[int, str]] = None + WRITE_ENABLED_COMMANDS_BITMASK0: Optional[Union[int, str]] = None + WRITE_ENABLED_COMMANDS_BITMASK1: Optional[Union[int, str]] = None + WRITE_ENABLED_COMMANDS_BITMASK2: Optional[Union[int, str]] = None + + na_validator = field_validator( + "READ_ENABLED_COMMANDS_BITMASK0", + "READ_ENABLED_COMMANDS_BITMASK1", + "READ_ENABLED_COMMANDS_BITMASK2", + "WRITE_ENABLED_COMMANDS_BITMASK0", + "WRITE_ENABLED_COMMANDS_BITMASK1", + "WRITE_ENABLED_COMMANDS_BITMASK2", + mode="before", + )(na_to_none) + + +class CpuMetric(BaseModel): + """Per-CPU-socket data from ``amd-smi metric --cpu all --json``.""" + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + cpu: int + power_metrics: Optional[CpuMetricPower] = None + prochot: Optional[CpuMetricProchot] = None + freq_metrics: Optional[CpuMetricFreq] = None + c0_residency: Optional[CpuMetricC0Residency] = None + svi_telemetry_all_rails: Optional[CpuMetricSviTelemetry] = None + pwr_eff_mode: Optional[CpuMetricPwrEffMode] = None + metric_version: Optional[CpuMetricVersion] = None + metrics_table: Optional[CpuMetricTable] = None + socket_energy: Optional[CpuMetricSocketEnergy] = None + ddr_bandwidth: Optional[CpuMetricDdrBandwidth] = None + cpu_temp: Optional[CpuMetricCpuTemp] = None + xgmi_pstate_range: Optional[CpuMetricXgmiPstateRange] = None + railisofreq_policy: Optional[ValueUnit] = None + dfcstate_ctrl: Optional[ValueUnit] = None + pc6_enable: Optional[ValueUnit] = None + cc6_enable: Optional[ValueUnit] = None + tdelta: Optional[ValueUnit] = None + enabled_commands: Optional[CpuMetricEnabledCommands] = None + sdps_limit: Optional[ValueUnit] = None + + _value_unit = field_validator( + "railisofreq_policy", + "dfcstate_ctrl", + "pc6_enable", + "cc6_enable", + "tdelta", + "sdps_limit", + mode="before", + )(coerce_value_unit_input) + + +class CpuDimmPowConsumption(BaseModel): + """Per-DIMM data from ``amd-smi metric --cpu-dimm-pow-consumption DIMM_ADDR --json``. + + ``dimm_addr`` is the address the collector queried with, not the (unreliable) echoed value. + """ + + model_config = ConfigDict(extra="allow") + + cpu: int + dimm_addr: int + dimm_power_consumed: Optional[ValueUnit] = None + dimm_power_update_rate: Optional[ValueUnit] = None + + _value_unit = field_validator("dimm_power_consumed", "dimm_power_update_rate", mode="before")( + coerce_value_unit_input + ) + + +class CpuDimmThermalSensor(BaseModel): + """Per-DIMM data from ``amd-smi metric --cpu-dimm-thermal-sensor DIMM_ADDR --json``.""" + + model_config = ConfigDict(extra="allow") + + cpu: int + dimm_addr: int + dimm_thermal_sensor_value: Optional[int] = None + dimm_thermal_update_rate: Optional[ValueUnit] = None + dimm_thermal_temperature: Optional[ValueUnit] = None + + na_validator = field_validator("dimm_thermal_sensor_value", mode="before")(na_to_none) + _value_unit = field_validator("dimm_thermal_update_rate", mode="before")( + coerce_value_unit_input + ) + _temperature_value_unit = field_validator("dimm_thermal_temperature", mode="before")( + coerce_spaced_unit_value_unit_input + ) + + +class CpuDimmTempRangeRate(BaseModel): + """Per-DIMM data from ``amd-smi metric --cpu-dimm-temp-range-rate DIMM_ADDR --json``.""" + + model_config = ConfigDict(extra="allow") + + cpu: int + dimm_addr: int + dimm_temperature_range: Optional[int] = None + dimm_refresh_rate: Optional[int] = None + + na_validator = field_validator("dimm_temperature_range", "dimm_refresh_rate", mode="before")( + na_to_none + ) + + +class CpuSvi3VrControllerTemp(BaseModel): + """Per-rail data from ``amd-smi metric --cpu-svi3-vr-controller-temp TYPE [RAIL_INDEX] --json``. + + ``rail_type``/``rail_index`` are the arguments the collector queried with + (0=HottestRail/1=IndividualRail; rail 0-4 maps to VDDCR_CPU0, VDDCR_CPU1, + VDDCR_SOC, VDDIO, VDDIO_MEM_S3). + """ + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + cpu: int + rail_type: int + rail_index: Optional[int] = None + rail_selection: Optional[str] = Field(default=None, alias="RAIL_SELECTION") + reported_rail_index: Optional[int] = Field(default=None, alias="RAIL_INDEX") + temperature: Optional[ValueUnit] = Field(default=None, alias="TEMPERATURE") + + na_validator = field_validator("rail_selection", "reported_rail_index", mode="before")( + na_to_none + ) + _value_unit = field_validator("temperature", mode="before")(coerce_value_unit_input) + + +class CpuXgmiBandwidth(BaseModel): + """Per-link data from ``amd-smi metric --cpu-xgmi-bandwidth XGMI_BW LINKID_NAME --json``.""" + + model_config = ConfigDict(extra="allow") + + cpu: int + xgmi_bw: int + link_id: str + band_width: Optional[ValueUnit] = None + + _value_unit = field_validator("band_width", mode="before")(coerce_value_unit_input) + + +class CpuIoBandwidth(BaseModel): + """Per-link data from ``amd-smi metric --cpu-io-bandwidth IO_BW LINKID_NAME --json``.""" + + model_config = ConfigDict(extra="allow") + + cpu: int + io_bw: int + link_id: str + band_width: Optional[ValueUnit] = None + + _value_unit = field_validator("band_width", mode="before")(coerce_value_unit_input) + + +class CpuLclkDpmLevel(BaseModel): + """Per-NBIO data from ``amd-smi metric --cpu-lclk-dpm-level NBIOID --json``.""" + + model_config = ConfigDict(extra="allow") + + cpu: int + nbio_id: int + nbio_min_dpm_level: Optional[int] = None + nbio_max_dpm_level: Optional[int] = None + + na_validator = field_validator("nbio_min_dpm_level", "nbio_max_dpm_level", mode="before")( + na_to_none + ) + + class AmdSmiAnalysisRef(BaseModel): """Collector-filled summary for reference config""" @@ -1189,6 +1949,19 @@ class AmdSmiDataModel(DataModel): metric: Optional[list[AmdSmiMetric]] = Field(default_factory=list) xgmi_metric: Optional[list[XgmiMetrics]] = Field(default_factory=list) xgmi_link: Optional[list[XgmiLinks]] = Field(default_factory=list) + fabric: Optional[list[Fabric]] = Field(default_factory=list) + node: Optional[list[NodeInfo]] = Field(default_factory=list) + cpu_metric: Optional[list[CpuMetric]] = Field(default_factory=list) + core_metric: Optional[list[CoreMetric]] = Field(default_factory=list) + cpu_dimm_pow_consumption: Optional[list[CpuDimmPowConsumption]] = Field(default_factory=list) + cpu_dimm_thermal_sensor: Optional[list[CpuDimmThermalSensor]] = Field(default_factory=list) + cpu_dimm_temp_range_rate: Optional[list[CpuDimmTempRangeRate]] = Field(default_factory=list) + cpu_svi3_vr_controller_temp: Optional[list[CpuSvi3VrControllerTemp]] = Field( + default_factory=list + ) + cpu_xgmi_bandwidth: Optional[list[CpuXgmiBandwidth]] = Field(default_factory=list) + cpu_io_bandwidth: Optional[list[CpuIoBandwidth]] = Field(default_factory=list) + cpu_lclk_dpm_level: Optional[list[CpuLclkDpmLevel]] = Field(default_factory=list) cper_data: Optional[list[FileModel]] = Field(default_factory=list) cper_afids: dict[str, int] = Field(default_factory=dict) @@ -1244,6 +2017,15 @@ def get_bad_pages(self, gpu: int) -> Optional[BadPages]: return item return None + def get_fabric(self, gpu: int) -> Optional[Fabric]: + """Get the fabric data for the given gpu id.""" + if self.fabric is None: + return None + for item in self.fabric: + if item.gpu == gpu: + return item + return None + def _sorted_static_gpus(self) -> list[AmdSmiStatic]: return sorted(self.static or [], key=lambda s: s.gpu) diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py new file mode 100644 index 00000000..82a68f5d --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_flavor.py @@ -0,0 +1,61 @@ +"""amd-smi data model that widens the base (guest/bare-metal only) ``AmdSmiDataModel`` +field types to also accept the mxGPU host-driver and guest-VF model variants. The +collector picks the concrete model per driver flavor; this model only relaxes the +annotations so those instances are accepted and serialized. The base ``AmdSmiDataModel`` +and its consumers are unchanged. +""" + +from __future__ import annotations + +from typing import Optional, Union + +from pydantic import Field + +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiDataModel, + AmdSmiListItem, + AmdSmiMetric, + AmdSmiStatic, + AmdSmiVersion, + BadPages, + Topo, + XgmiLinks, + XgmiMetrics, +) + +from .amdsmidata_guest import GuestAmdSmiMetric, GuestAmdSmiStatic +from .amdsmidata_host import ( + HostDriverAmdSmiListItem, + HostDriverAmdSmiMetric, + HostDriverAmdSmiStatic, + HostDriverAmdSmiVersion, + HostDriverBadPages, + HostDriverTopo, + HostDriverXgmiLinks, + HostDriverXgmiMetrics, +) + + +class AmdSmiFlavorDataModel(AmdSmiDataModel): + """AmdSmiDataModel with host/guest driver-flavor model variants allowed.""" + + # Base (guest/bare-metal) type is listed first; the collector builds the right + # concrete instance per flavor, so widening the annotation is sufficient. + # Widening the base annotations is an intentional LSP override (invariant list), + # so the type: ignore[assignment] markers below are expected. + version: Optional[Union[AmdSmiVersion, HostDriverAmdSmiVersion]] = None # type: ignore[assignment] + gpu_list: Optional[list[Union[AmdSmiListItem, HostDriverAmdSmiListItem]]] = Field( # type: ignore[assignment] + default_factory=list + ) + topology: Optional[list[Union[Topo, HostDriverTopo]]] = Field(default_factory=list) # type: ignore[assignment] + bad_pages: Optional[list[Union[BadPages, HostDriverBadPages]]] = Field(default_factory=list) # type: ignore[assignment] + static: Optional[list[Union[AmdSmiStatic, GuestAmdSmiStatic, HostDriverAmdSmiStatic]]] = Field( # type: ignore[assignment] + default_factory=list + ) + metric: Optional[list[Union[AmdSmiMetric, GuestAmdSmiMetric, HostDriverAmdSmiMetric]]] = Field( # type: ignore[assignment] + default_factory=list + ) + xgmi_metric: Optional[list[Union[XgmiMetrics, HostDriverXgmiMetrics]]] = Field( # type: ignore[assignment] + default_factory=list + ) + xgmi_link: Optional[list[Union[XgmiLinks, HostDriverXgmiLinks]]] = Field(default_factory=list) # type: ignore[assignment] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py new file mode 100644 index 00000000..ef2eb658 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_guest.py @@ -0,0 +1,73 @@ +"""Guest-VF (virtualized amdgpu) Pydantic models for amd-smi data. + +A guest VM sees a single amdgpu virtual function (VF). Its amd-smi build shares +the bare-metal amdgpu JSON schema EXCEPT that a VF does not expose physical / +host-owned fields: fan control, SoC power state, power/thermal limits, NUMA +topology, XGMI power-down policy, and energy/throttle/voltage-curve/perf-level +counters. The bare-metal ``AmdSmi*`` models mark those fields as required, so a +guest payload fails to build against them; these subclasses relax exactly those +fields to optional and inherit all other structure unchanged. + +amd-smi was not exercised on the guest before the host/guest driver-flavor work, +so the base ``AmdSmi*`` models had implicitly assumed bare-metal amdgpu. +""" + +from __future__ import annotations + +from typing import Optional, Union + +from pydantic import ConfigDict + +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + AmdSmiMetric, + AmdSmiStatic, + MetricEnergy, + MetricFan, + MetricMemUsage, + MetricThrottle, + MetricVoltageCurve, + StaticBus, + StaticLimit, + StaticNuma, + StaticSocPstate, + StaticVbios, + StaticXgmiPlpd, +) + + +class GuestStaticBus(StaticBus): + """Guest amd-smi (ROCm 7.14+) adds a per-link ``pcie_levels`` block to the bus + section that the base ``StaticBus`` (``extra="forbid"``) rejects. Ignore unknown + bus fields so a new field doesn't drop the whole static payload, matching the + host models' schema-drift resilience.""" + + model_config = ConfigDict(extra="ignore") + + +class GuestAmdSmiStatic(AmdSmiStatic): + """Bare-metal static schema minus the physical fields a guest VF omits.""" + + # Base bus (StaticBus) forbids extras; the guest build adds pcie_levels. + bus: GuestStaticBus # type: ignore[assignment] + soc_pstate: Optional[StaticSocPstate] = None + xgmi_plpd: Optional[StaticXgmiPlpd] = None + numa: Optional[StaticNuma] = None # type: ignore[assignment] + limit: Optional[StaticLimit] = None + # A guest VF's static payload carries an ``ifwi`` firmware block (name, + # build_date, part_number, version) that the bare-metal schema omits; its + # shape matches StaticVbios. + ifwi: Optional[StaticVbios] = None + + +class GuestAmdSmiMetric(AmdSmiMetric): + """Bare-metal metric schema minus the physical fields a guest VF omits.""" + + # These physical fields are required on the bare-metal base model; a guest VF + # omits them, so relax to Optional (intentional LSP-widening override). + fan: Optional[MetricFan] = None # type: ignore[assignment] + voltage_curve: Optional[MetricVoltageCurve] = None + perf_level: Optional[Union[str, dict]] = None + xgmi_err: Optional[Union[str, dict]] = None + energy: Optional[MetricEnergy] = None + throttle: Optional[MetricThrottle] = None # type: ignore[assignment] + mem_usage: Optional[MetricMemUsage] = None # type: ignore[assignment] diff --git a/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py b/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py new file mode 100644 index 00000000..a1d26b33 --- /dev/null +++ b/nodescraper/plugins/inband/amdsmi/amdsmidata_host.py @@ -0,0 +1,505 @@ +"""Host-driver (mxGPU) Pydantic models for amd-smi data. + +The mxGPU host driver ships an amd-smi build whose JSON schema differs from the +guest amdgpu build. The same host schema is used by both `gim` (Linux host) and +`amdgpuv` (ESXi host), so these models serve both; the base AmdSmi* models serve +the guest amdgpu build. +""" + +from __future__ import annotations + +from typing import Any, Optional, Union + +from pydantic import ( + AliasPath, + BaseModel, + ConfigDict, + Field, + computed_field, + field_validator, +) + +from nodescraper.plugins.inband.amdsmi.amdsmidata import ( + NA_STR, + AmdSmiBaseModel, + EccData, + EccState, + ValueUnit, +) + + +def _host_na_to_none(value: Any) -> Any: + """Convert amd-smi N/A markers to None. + + Newer amd-smi (e.g. 37.0.5) emits N/A both as a scalar ("N/A") and as a + ``{"value": "N/A", "unit": "N/A"}`` dict for ValueUnit fields; the base + ValueUnit coercion does not collapse the dict form inside a ``ValueUnit | None`` + union, so normalize both to None before field validation. + """ + if isinstance(value, str) and value.strip().upper() in (NA_STR, "NA", ""): + return None + if isinstance(value, dict): + inner = value.get("value") + if isinstance(inner, str) and inner.strip().upper() in (NA_STR, "NA", ""): + return None + return value + + +class HostDriverAmdSmiVersion(BaseModel): + """mxGPU host `amd-smi version --json` nests its fields under a top-level + "version" key: ``{"version": {"tool_name": ..., "tool_version": ...}}``. + + AliasPath reads each field from that nested dict, so the collector can build + this model directly from the raw item - no unwrapping needed. (Guest/bare-metal + amdgpu instead put a scalar "version" string in a flat dict; that is the + separate AmdSmiVersion model.) + """ + + tool_name: Optional[str] = Field( + default=None, validation_alias=AliasPath("version", "tool_name") + ) + tool_version: Optional[str] = Field( + default=None, validation_alias=AliasPath("version", "tool_version") + ) + lib_version: Optional[str] = Field( + default=None, validation_alias=AliasPath("version", "lib_version") + ) + driver_version: Optional[str] = Field( + default=None, validation_alias=AliasPath("version", "driver_version") + ) + # mxGPU host amd-smi does not report a ROCm version; kept here (always None) so + # version consumers - e.g. the amdsmitst ROCm-version gate - can read it uniformly. + rocm_version: Optional[str] = None + + +class HostDriverAmdSmiListItem(BaseModel): + """mxGPU host amd-smi list has gpu/bdf/uuid/vfs — no kfd_id/node_id/partition_id.""" + + model_config = ConfigDict(extra="ignore") + + gpu: int + bdf: str + uuid: str + + +class HostDriverStaticAsic(BaseModel): + # Expose num_compute_units (matching the guest model + AmdSmiAnalyzer) while + # accepting the host driver's JSON key "num_of_compute_units" via alias. + model_config = ConfigDict(populate_by_name=True) + + market_name: str + vendor_id: str + vendor_name: str + subvendor_id: str + device_id: str + subsystem_id: str + rev_id: str + asic_serial: str + oam_id: Union[int, str] + num_compute_units: Union[int, str] = Field(alias="num_of_compute_units") + # Host driver amd-smi does not report target_graphics_version; keep optional + # so AmdSmiAnalyzer.static_consistancy_check can read it uniformly. + target_graphics_version: Optional[str] = None + + +class HostDriverStaticIfwi(BaseModel): + name: str + build_date: str + part_number: str + version: str + boot_firmware: Optional[str] = None + + +class HostDriverStaticBus(AmdSmiBaseModel): + bdf: str + max_pcie_width: ValueUnit + max_pcie_speed: ValueUnit + pcie_interface_version: str + slot_type: str + max_pcie_interface_version: Optional[str] = None + + +class HostDriverStaticPpt(AmdSmiBaseModel): + """Host per-PPT power caps (amd-smi 37.0.5+ nests these under limit.ppt0/ppt1). + + The host build keys these ``max_power``/``min_power``/``socket_power`` — unlike + the base ``StaticPowerLimit`` (guest/bare-metal ROCm 7+) which uses the + ``*_power_limit`` key names. + """ + + model_config = ConfigDict(extra="ignore") + + max_power: Optional[ValueUnit] = None + min_power: Optional[ValueUnit] = None + socket_power: Optional[ValueUnit] = None + na_validator = field_validator( + "max_power", + "min_power", + "socket_power", + mode="before", + )(_host_na_to_none) + + +class HostDriverStaticLimit(AmdSmiBaseModel): + # Host amd-smi adds per-partition power fields (ppt0/ppt1, ...) across tool + # versions; ignore unknown fields so a new field doesn't drop all static data. + model_config = ConfigDict(extra="ignore") + + max_power: Optional[ValueUnit] = None + min_power: Optional[ValueUnit] = None + socket_power: Optional[ValueUnit] = None + slowdown_edge_temperature: Optional[ValueUnit] = None + slowdown_hotspot_temperature: Optional[ValueUnit] = None + slowdown_mem_temperature: Optional[ValueUnit] = None + shutdown_edge_temperature: Optional[ValueUnit] = None + shutdown_hotspot_temperature: Optional[ValueUnit] = None + shutdown_mem_temperature: Optional[ValueUnit] = None + # amd-smi 37.0.5 moved the flat max/min/socket power caps under ppt0 (active + # profile) / ppt1 (secondary, often N/A); older builds used the flat fields. + ppt0: Optional[HostDriverStaticPpt] = None + ppt1: Optional[HostDriverStaticPpt] = None + na_validator = field_validator( + "max_power", + "min_power", + "socket_power", + "slowdown_edge_temperature", + "slowdown_hotspot_temperature", + "slowdown_mem_temperature", + "shutdown_edge_temperature", + "shutdown_hotspot_temperature", + "shutdown_mem_temperature", + "ppt0", + "ppt1", + mode="before", + )(_host_na_to_none) + + def resolved_max_power(self) -> Optional[ValueUnit]: + """Match the base StaticLimit contract so the shared analyzer / data-model + max-power helpers work on host data. Older host builds expose a flat + max_power; amd-smi 37.0.5 moved it under limit.ppt0.""" + if self.max_power is not None: + return self.max_power + if self.ppt0 is not None and self.ppt0.max_power is not None: + return self.ppt0.max_power + return None + + +class HostDriverStaticDriver(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + version: str + date: Optional[str] = None + model: Optional[str] = None + + +class HostDriverStaticBoard(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + # "model_" is a pydantic-reserved namespace; expose as amdsmi_model_number + # (matching the base StaticBoard) while reading the JSON "model_number" key. + amdsmi_model_number: str = Field(alias="model_number") + product_serial: str + fru_id: str + product_name: str + manufacturer_name: str + + +class HostDriverStaticRas(BaseModel): + model_config = ConfigDict(extra="ignore") + + eeprom_version: str + parity_schema: EccState + single_bit_schema: EccState + double_bit_schema: EccState + poison_schema: EccState + block_state: Union[dict[str, EccState], str] + + +class HostDriverStaticFbInfo(AmdSmiBaseModel): + total_fb_size: Optional[ValueUnit] = None + pf_fb_reserved: Optional[ValueUnit] = None + pf_fb_offset: Optional[ValueUnit] = None + fb_alignment: Optional[ValueUnit] = None + max_vf_fb_usable: Optional[ValueUnit] = None + min_vf_fb_usable: Optional[ValueUnit] = None + + +class HostDriverStaticNumVf(BaseModel): + supported: int + enabled: int + + +class HostDriverStaticVram(AmdSmiBaseModel): + type: str + vendor: Optional[str] = None + size: Optional[ValueUnit] = None + bit_width: Optional[Union[ValueUnit, int]] = None + max_bandwidth: Optional[ValueUnit] = None + na_validator = field_validator("vendor", "size", "bit_width", "max_bandwidth", mode="before")( + _host_na_to_none + ) + + +class HostDriverStaticNuma(BaseModel): + # gim (Linux mxGPU host) reports cpu_affinity as a dict ({"cpu_list": [...]}); + # amdgpuv (ESXi mxGPU host) reports it as int/str. Accept both. + node: Optional[Union[int, str, dict]] = None + cpu_affinity: Optional[Union[int, str, dict]] = None + socket_affinity: Optional[Union[int, str, dict]] = None + na_validator = field_validator("node", "cpu_affinity", "socket_affinity", mode="before")( + _host_na_to_none + ) + + +class HostDriverStaticPartition(BaseModel): + model_config = ConfigDict(extra="ignore") + + accelerator_partition: str + memory_partition: str + partition_id: Union[list[int], int] + + +class HostDriverStaticXgmiPlpd(BaseModel): + num_supported: int + current_id: int + policies: list[dict] = Field(default_factory=list) + + +class HostDriverAmdSmiStatic(BaseModel): + """mxGPU host static model — uses ifwi instead of vbios, has fb_info/num-vf, etc.""" + + model_config = ConfigDict(extra="ignore") + + gpu: int + asic: HostDriverStaticAsic + bus: HostDriverStaticBus + ifwi: Optional[HostDriverStaticIfwi] = None + limit: Optional[HostDriverStaticLimit] = None + driver: HostDriverStaticDriver + board: HostDriverStaticBoard + ras: HostDriverStaticRas + fb_info: Optional[HostDriverStaticFbInfo] = None + num_vf: Optional[HostDriverStaticNumVf] = Field(default=None, alias="num-vf") + vram: HostDriverStaticVram + cache_info: list[dict] = Field(default_factory=list) + xgmi_plpd: Optional[HostDriverStaticXgmiPlpd] = Field(default=None, alias="xgmi-plpd") + partition: Optional[HostDriverStaticPartition] = None + numa: HostDriverStaticNuma + na_validator = field_validator("limit", "ifwi", "xgmi_plpd", mode="before")(_host_na_to_none) + + +# --- Metric models --- + + +class HostDriverMetricUsage(BaseModel): + gfx_activity: Optional[ValueUnit] = None + umc_activity: Optional[ValueUnit] = None + mm_activity: Optional[ValueUnit] = None + vcn_activity: list[Optional[Union[ValueUnit, str]]] = Field(default_factory=list) + jpeg_activity: list[Optional[Union[ValueUnit, str]]] = Field(default_factory=list) + na_validator = field_validator("gfx_activity", "umc_activity", "mm_activity", mode="before")( + _host_na_to_none + ) + + +class HostDriverMetricPower(BaseModel): + socket_power: Optional[ValueUnit] = None + gfx_voltage: Optional[ValueUnit] = None + soc_voltage: Optional[ValueUnit] = None + mem_voltage: Optional[ValueUnit] = None + power_management: Optional[str] = None + na_validator = field_validator( + "socket_power", + "gfx_voltage", + "soc_voltage", + "mem_voltage", + "power_management", + mode="before", + )(_host_na_to_none) + + +class HostDriverMetricClockData(BaseModel): + clk: Optional[ValueUnit] = None + min_clk: Optional[ValueUnit] = None + max_clk: Optional[ValueUnit] = None + clk_locked: Optional[Union[int, str, dict]] = None + deep_sleep: Optional[Union[int, str, dict]] = None + na_validator = field_validator( + "clk", "min_clk", "max_clk", "clk_locked", "deep_sleep", mode="before" + )(_host_na_to_none) + + +class HostDriverMetricTemperature(BaseModel): + edge: Optional[ValueUnit] = None + hotspot: Optional[ValueUnit] = None + mem: Optional[ValueUnit] = None + na_validator = field_validator("edge", "hotspot", "mem", mode="before")(_host_na_to_none) + + +class HostDriverMetricPcie(BaseModel): + width: Optional[int] = None + speed: Optional[ValueUnit] = None + bandwidth: Optional[ValueUnit] = None + replay_count: Optional[int] = None + l0_to_recovery_count: Optional[int] = None + replay_roll_over_count: Optional[int] = None + nak_sent_count: Optional[int] = None + nak_received_count: Optional[int] = None + na_validator = field_validator( + "width", + "speed", + "bandwidth", + "replay_count", + "l0_to_recovery_count", + "replay_roll_over_count", + "nak_sent_count", + "nak_received_count", + mode="before", + )(_host_na_to_none) + + +class HostDriverMetricEccTotals(BaseModel): + total_correctable_count: Optional[int] = None + total_uncorrectable_count: Optional[int] = None + total_deferred_count: Optional[int] = None + cache_correctable_count: Optional[int] = None + cache_uncorrectable_count: Optional[int] = None + na_validator = field_validator( + "total_correctable_count", + "total_uncorrectable_count", + "total_deferred_count", + "cache_correctable_count", + "cache_uncorrectable_count", + mode="before", + )(_host_na_to_none) + + +class HostDriverMetricEnergy(BaseModel): + total_energy_consumption: Optional[ValueUnit] = None + na_validator = field_validator("total_energy_consumption", mode="before")(_host_na_to_none) + + +class HostDriverMetricGpuBoard(BaseModel): + model_config = ConfigDict(extra="allow") + + +class HostDriverAmdSmiMetric(BaseModel): + """mxGPU host metric model — no fan/voltage_curve/perf_level/xgmi_err/mem_usage/throttle, + has gpuboard instead.""" + + model_config = ConfigDict(extra="ignore") + + gpu: int + usage: Union[HostDriverMetricUsage, str] + power: HostDriverMetricPower + clock: dict[str, Union[HostDriverMetricClockData, dict]] + temperature: HostDriverMetricTemperature + pcie: HostDriverMetricPcie + ecc: HostDriverMetricEccTotals + ecc_blocks: Union[dict[str, EccData], str] = Field(default_factory=dict) + energy: Optional[HostDriverMetricEnergy] = None + gpuboard: Optional[HostDriverMetricGpuBoard] = None + + @field_validator("ecc_blocks", mode="before") + @classmethod + def validate_ecc_blocks(cls, value): + if isinstance(value, str): + return {} + return value + + @field_validator("energy", mode="before") + @classmethod + def validate_energy(cls, value): + if value == NA_STR or value is None: + return None + return value + + +# --- Topology --- + + +class HostDriverTopoLink(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + gpu: int + bdf: str + weight: int + link_type: str + num_hops: int + bandwidth: str = "" + fb_sharing: Optional[str] = None + coherent: Optional[str] = None + atomics: Optional[str] = None + dma: Optional[str] = None + bi_dir: Optional[str] = Field(default=None, alias="bi-dir") + + @computed_field + def bandwidth_from(self) -> Optional[int]: + bw_split = self.bandwidth.split("-") + return int(bw_split[0]) if len(bw_split) == 2 else None + + @computed_field + def bandwidth_to(self) -> Optional[int]: + bw_split = self.bandwidth.split("-") + return int(bw_split[1]) if len(bw_split) == 2 else None + + +class HostDriverTopo(BaseModel): + gpu: int + bdf: str + links: list[HostDriverTopoLink] + + +# --- Bad pages --- + + +class HostDriverBadPageEntry(BaseModel): + model_config = ConfigDict(extra="ignore") + + bad_page: int + retired_bad_page: str + timestamp: Optional[str] = None + mem_channel: Optional[int] = None + mcumc_id: Optional[int] = None + + +class HostDriverBadPages(BaseModel): + gpu: int + bad_pages: list[HostDriverBadPageEntry] = Field(default_factory=list) + + +# --- XGMI --- + + +class HostDriverXgmiLink(BaseModel): + gpu: int + bdf: str + read: Optional[ValueUnit] = None + write: Optional[ValueUnit] = None + na_validator = field_validator("read", "write", mode="before")(_host_na_to_none) + + +class HostDriverXgmiLinkMetrics(BaseModel): + bit_rate: Optional[ValueUnit] = None + max_bandwidth: Optional[ValueUnit] = None + links: list[HostDriverXgmiLink] = Field(default_factory=list) + na_validator = field_validator("max_bandwidth", "bit_rate", mode="before")(_host_na_to_none) + + +class HostDriverXgmiMetrics(BaseModel): + gpu: int + bdf: str + link_metrics: HostDriverXgmiLinkMetrics + + +class HostDriverXgmiLinkStatus(BaseModel): + gpu: int + bdf: str + status: str + + +class HostDriverXgmiLinks(BaseModel): + gpu: int + bdf: str + link_status: list[HostDriverXgmiLinkStatus] diff --git a/nodescraper/plugins/inband/amdsmi/analyzer_args.py b/nodescraper/plugins/inband/amdsmi/analyzer_args.py index cd08bd49..d53bd2fd 100644 --- a/nodescraper/plugins/inband/amdsmi/analyzer_args.py +++ b/nodescraper/plugins/inband/amdsmi/analyzer_args.py @@ -26,12 +26,31 @@ from datetime import datetime from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from nodescraper.models import AnalyzerArgs from nodescraper.plugins.inband.amdsmi.amdsmidata import AmdSmiDataModel +class PowerConfig(BaseModel): + """GPU power-cap consistency policy.""" + + power_cap_mismatch_allowed: bool = Field( + default=False, + description="Whether different GPU power caps are allowed.", + ) + + +class GpuMemoryConfig(BaseModel): + """GPU VRAM availability threshold.""" + + minimum_available_percent: float = Field( + ge=0, + le=100, + description="Minimum free VRAM percentage required for each GPU.", + ) + + class AmdSmiAnalyzerArgs(AnalyzerArgs): check_static_data: bool = Field( default=False, @@ -43,6 +62,10 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): expected_max_power: Optional[int] = Field( default=None, description="Expected maximum power value (e.g. watts)." ) + power: Optional[PowerConfig] = Field( + default=None, + description="GPU power-cap consistency policy.", + ) expected_power_management: Optional[str] = Field( default=None, description=( @@ -63,6 +86,14 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): default=None, description="Expected firmware versions keyed by amd-smi fw_id (e.g. PLDM_BUNDLE).", ) + gpu_memory: Optional[GpuMemoryConfig] = Field( + default=None, + description="Minimum free VRAM threshold to validate for each GPU.", + ) + check_xgmi_or_peer_links_status: bool = Field( + default=False, + description="Check XGMI or peer-link status for each GPU.", + ) l0_to_recovery_count_error_threshold: Optional[int] = Field( default=3, description="L0-to-recovery count above which an error is raised.", @@ -83,6 +114,12 @@ class AmdSmiAnalyzerArgs(AnalyzerArgs): expected_xgmi_speed: Optional[list[float]] = Field( default=None, description="Expected xGMI speed value(s) (e.g. link rate)." ) + expected_accel_state: str = Field( + default="ACTIVE", description="Expected amd-smi fabric accel_state value." + ) + expected_fabric_type: str = Field( + default="UALOE", description="Expected amd-smi fabric fabric_type value." + ) analysis_range_start: Optional[datetime] = Field( default=None, description="Start of time range for time-windowed analysis." ) diff --git a/nodescraper/plugins/inband/bios/bios_collector.py b/nodescraper/plugins/inband/bios/bios_collector.py index e0ab1011..e94242ef 100644 --- a/nodescraper/plugins/inband/bios/bios_collector.py +++ b/nodescraper/plugins/inband/bios/bios_collector.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import re from typing import Optional from nodescraper.base import InBandDataCollector @@ -35,9 +36,11 @@ class BiosCollector(InBandDataCollector[BiosDataModel, None]): """Collect BIOS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = BiosDataModel CMD_WINDOWS = "wmic bios get SMBIOSBIOSVersion /Value" CMD = "sh -c 'cat /sys/devices/virtual/dmi/id/bios_version'" + CMD_ESXI = "smbiosDump | grep -A5 'BIOS Info (Type 0)' | grep 'Version:' | head -1" def collect_data( self, @@ -57,6 +60,11 @@ def collect_data( bios = [line for line in res.stdout.splitlines() if "SMBIOSBIOSVersion=" in line][ 0 ].split("=")[1] + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + match = re.search(r'Version:\s*"?([^"]+)"?', res.stdout) + bios = match.group(1).strip() if match else res.stdout.strip() else: res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: diff --git a/nodescraper/plugins/inband/device_enumeration/collector_args.py b/nodescraper/plugins/inband/device_enumeration/collector_args.py new file mode 100644 index 00000000..c2d7b35c --- /dev/null +++ b/nodescraper/plugins/inband/device_enumeration/collector_args.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class DeviceEnumerationCollectorArgs(CollectorArgs): + """Collector args for device enumeration. + + On ESXi, GPUs and their SR-IOV VFs are counted by PCI device ID (esxcli has no + device filter). Provide the expected PF/VF device IDs here; when unset the ESXi + GPU/VF counts are skipped. The caller populates these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi device counting.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF counting.", + ) diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py index 9b0dc295..78c54ad0 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_collector.py @@ -30,12 +30,16 @@ from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult +from .collector_args import DeviceEnumerationCollectorArgs from .deviceenumdata import DeviceEnumerationDataModel -class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, None]): +class DeviceEnumerationCollector( + InBandDataCollector[DeviceEnumerationDataModel, DeviceEnumerationCollectorArgs] +): """Collect CPU and GPU count""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DeviceEnumerationDataModel CMD_GPU_COUNT_LINUX = ( @@ -55,6 +59,17 @@ class DeviceEnumerationCollector(InBandDataCollector[DeviceEnumerationDataModel, 'powershell -Command "(Get-VMHostPartitionableGpu | Measure-Object).Count"' ) + # ESXi busybox `lspci -d` dumps hex instead of filtering, so use esxcli. GPUs are + # counted by device ID (PF vs VF), anchored on "Device ID:" to avoid also matching + # "SubDevice ID:". The match is case-insensitive and tolerates zero-padding + # ("0x744C" / "0x0000744c"); the trailing [^0-9a-f]/$ guard stops a shorter ID from + # matching a longer one (e.g. 744c vs 744cd). + CMD_CPU_COUNT_ESXI = "esxcli hardware cpu global get | awk '/CPU Packages:/ {print $NF}'" + CMD_PCI_COUNT_ESXI = ( + "esxcli hardware pci list | " + "grep -iE '^ *Device ID: 0x0*{device_id}([^0-9a-f]|$)' | wc -l" + ) + def _warning( self, description: str, @@ -72,12 +87,47 @@ def _warning( priority=EventPriority.WARNING, ) - def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: + def _parse_count( + self, + res: CommandArtifact, + description: str, + category: EventCategory = EventCategory.PLATFORM, + ) -> Optional[int]: + """Parse a numeric count from command stdout, warning (not raising) on a + non-zero exit or non-numeric output (e.g. an unexpected esxcli/awk result).""" + if res.exit_code != 0: + self._warning(description=description, command=res, category=category) + return None + text = (res.stdout or "").strip() + if not text.isdigit(): + self._warning( + description=f"{description} (non-numeric output: {text!r})", + command=res, + category=category, + ) + return None + return int(text) + + def _esxi_device_count(self, device_id: Optional[int]) -> CommandArtifact: + """Count PCI devices on ESXi whose Device ID matches ``device_id`` (as hex). + + A None id produces an unmatched pattern (count 0) so the caller still gets a + valid CommandArtifact to parse. + """ + hex_id = format(device_id, "x") if device_id is not None else "__unset__" + return self._run_sut_cmd(self.CMD_PCI_COUNT_ESXI.format(device_id=hex_id)) + + def collect_data( + self, args: Optional[DeviceEnumerationCollectorArgs] = None + ) -> tuple[TaskResult, Optional[DeviceEnumerationDataModel]]: """ Read CPU and GPU count On Linux, use lscpu and lspci + On ESXi, use esxcli (GPU/VF counts need args.devid_ep / devid_ep_vf) On Windows, use WMI and hyper-v cmdlets """ + if args is None: + args = DeviceEnumerationCollectorArgs() if self.system_info.os_family == OSFamily.LINUX: lscpu_res = self._run_sut_cmd(self.CMD_LSCPU_LINUX, log_artifact=False) @@ -92,6 +142,17 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio # Collect lshw output lshw_res = self._run_sut_cmd(self.CMD_LSHW_LINUX, sudo=True, log_artifact=False) + elif self.system_info.os_family == OSFamily.ESXI: + cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_ESXI) + if args.devid_ep is None: + self._log_event( + category=EventCategory.PLATFORM, + description="devid_ep not set; cannot count GPUs/VFs on ESXi by device ID", + priority=EventPriority.WARNING, + ) + # PFs and (SR-IOV) VFs are distinguished by device ID on ESXi. + gpu_count_res = self._esxi_device_count(args.devid_ep) + vf_count_res = self._esxi_device_count(args.devid_ep_vf) else: cpu_count_res = self._run_sut_cmd(self.CMD_CPU_COUNT_WINDOWS) gpu_count_res = self._run_sut_cmd(self.CMD_GPU_COUNT_WINDOWS) @@ -121,24 +182,19 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[DeviceEnumeratio else: self._warning(description="Cannot collect lscpu output", command=lscpu_res) else: - if cpu_count_res.exit_code == 0: - device_enum.cpu_count = int(cpu_count_res.stdout) - else: - self._warning(description="Cannot determine CPU count", command=cpu_count_res) + cpu_count = self._parse_count(cpu_count_res, "Cannot determine CPU count") + if cpu_count is not None: + device_enum.cpu_count = cpu_count - if gpu_count_res.exit_code == 0: - device_enum.gpu_count = int(gpu_count_res.stdout) - else: - self._warning(description="Cannot determine GPU count", command=gpu_count_res) + gpu_count = self._parse_count(gpu_count_res, "Cannot determine GPU count") + if gpu_count is not None: + device_enum.gpu_count = gpu_count - if vf_count_res.exit_code == 0: - device_enum.vf_count = int(vf_count_res.stdout) - else: - self._warning( - description="Cannot determine VF count", - command=vf_count_res, - category=EventCategory.SW_DRIVER, - ) + vf_count = self._parse_count( + vf_count_res, "Cannot determine VF count", category=EventCategory.SW_DRIVER + ) + if vf_count is not None: + device_enum.vf_count = vf_count # Collect lshw output on Linux if self.system_info.os_family == OSFamily.LINUX: diff --git a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py index baf2aa2d..cff51210 100644 --- a/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py +++ b/nodescraper/plugins/inband/device_enumeration/device_enumeration_plugin.py @@ -26,13 +26,18 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import DeviceEnumerationAnalyzerArgs +from .collector_args import DeviceEnumerationCollectorArgs from .device_enumeration_analyzer import DeviceEnumerationAnalyzer from .device_enumeration_collector import DeviceEnumerationCollector from .deviceenumdata import DeviceEnumerationDataModel class DeviceEnumerationPlugin( - InBandDataPlugin[DeviceEnumerationDataModel, None, DeviceEnumerationAnalyzerArgs] + InBandDataPlugin[ + DeviceEnumerationDataModel, + DeviceEnumerationCollectorArgs, + DeviceEnumerationAnalyzerArgs, + ] ): """Plugin for collection and analysis of BIOS data""" @@ -40,6 +45,8 @@ class DeviceEnumerationPlugin( COLLECTOR = DeviceEnumerationCollector + COLLECTOR_ARGS = DeviceEnumerationCollectorArgs + ANALYZER = DeviceEnumerationAnalyzer ANALYZER_ARGS = DeviceEnumerationAnalyzerArgs diff --git a/nodescraper/plugins/inband/dimm/dimm_collector.py b/nodescraper/plugins/inband/dimm/dimm_collector.py index b6b91987..00fc807f 100644 --- a/nodescraper/plugins/inband/dimm/dimm_collector.py +++ b/nodescraper/plugins/inband/dimm/dimm_collector.py @@ -23,27 +23,37 @@ # SOFTWARE. # ############################################################################### -import re -from typing import Optional +import csv +from typing import ClassVar, Optional + +from typing_extensions import override from nodescraper.base import InBandDataCollector -from nodescraper.connection.inband import TextFileArtifact +from nodescraper.connection.inband import CommandArtifact, TextFileArtifact from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily from nodescraper.models import TaskResult from .collector_args import DimmCollectorArgs -from .dimmdata import DimmDataModel +from .dimmdata import DimmDataModel, DimmInfo class DimmCollector(InBandDataCollector[DimmDataModel, DimmCollectorArgs]): """Collect data on installed DIMMs""" - DATA_MODEL = DimmDataModel + DATA_MODEL: type[DimmDataModel] = DimmDataModel + + # Both platforms dump every field the firmware exposes and decode it here, + # rather than filtering on the SUT, so that a shell quirk on any given host + # cannot silently drop modules from the inventory. + CMD_WINDOWS: ClassVar[str] = "wmic memorychip get /format:csv" + CMD_DMIDECODE: ClassVar[str] = "dmidecode -q --type 17" + CMD_DMIDECODE_FULL: ClassVar[str] = "dmidecode" - CMD_WINDOWS = "wmic memorychip get Capacity" - CMD = """sh -c 'dmidecode -t 17 | tr -s " " | grep -v "Volatile\\|None\\|Module" | grep Size' 2>/dev/null""" - CMD_DMIDECODE_FULL = "dmidecode" + # Section title of a DMI type 17 record, which is all that identifies one + # under -q since that hides the handle lines. + MEMORY_DEVICE: ClassVar[str] = "Memory Device" + @override def collect_data( self, args: Optional[DimmCollectorArgs] = None, @@ -52,97 +62,21 @@ def collect_data( if args is None: args = DimmCollectorArgs() - dimm_str = None if self.system_info.os_family == OSFamily.WINDOWS: - res = self._run_sut_cmd(self.CMD_WINDOWS) - if res.exit_code == 0: - capacities = {} - total = 0 - for line in res.stdout.splitlines(): - value = line.strip() - if value.isdigit(): - value = int(value) - total += value - if value not in capacities: - capacities[value] = 1 - else: - capacities[value] += 1 - dimm_str = f"{total / 1024 / 1024:.2f}GB @ " - for capacity, count in capacities.items(): - dimm_str += f"{count} x {capacity / 1024 / 1024:.2f}GB " + records = self._collect_windows_records() else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" self.result.status = ExecutionStatus.NOT_RAN return self.result, None - # Collect full dmidecode output as artifact - dmidecode_full_res = self._run_sut_cmd(self.CMD_DMIDECODE_FULL, sudo=True) - if dmidecode_full_res.exit_code == 0 and dmidecode_full_res.stdout: - self.result.artifacts.append( - TextFileArtifact(filename="dmidecode.txt", contents=dmidecode_full_res.stdout) - ) - else: - self._log_event( - category=EventCategory.OS, - description="Could not collect full dmidecode output", - data={ - "command": dmidecode_full_res.command, - "exit_code": dmidecode_full_res.exit_code, - "stderr": dmidecode_full_res.stderr, - }, - priority=EventPriority.WARNING, - ) - - res = self._run_sut_cmd(self.CMD, sudo=True) - if res.exit_code == 0: - total = 0 - topology = {} - size = "" - dimm_size_pattern = re.compile(r"Size:\s+(\d+)\s+([A-Za-z]+)") - matches = dimm_size_pattern.findall(res.stdout) - if matches: - for match in matches: - size = match[1] - total += int(match[0]) - key = match[0] + match[1] - if not topology.get(key, None): - topology[key] = 1 - else: - topology[key] += 1 - topology["total"] = total - topology["size"] = size - total_gb = topology.pop("total") - size = topology.pop("size") - if total_gb == 0: - dimm_str = "0 GB" - else: - dimm_entries = [f"{v} x {k}" for k, v in topology.items()] - dimm_str = f"{total_gb}{size} @ {' '.join(dimm_entries)}" - if res.exit_code != 0: - self._log_event( - category=EventCategory.OS, - description="Error checking dimms", - data={ - "command": res.command, - "exit_code": res.exit_code, - "stderr": res.stderr, - }, - priority=EventPriority.ERROR, - console_log=True, - ) + records = self._collect_dmidecode_records() - if dimm_str: - dimm_data = DimmDataModel(dimms=dimm_str) - self._log_event( - category=EventCategory.IO, - description="Installed DIMM check", - data=dimm_data.model_dump(), - priority=EventPriority.INFO, - ) - self.result.message = f"DIMM: {dimm_str}" - else: - dimm_data = None + # Records for empty slots, and any the firmware reports too incompletely + # to decode, come back as None and are dropped from the inventory. + dimms = [dimm for dimm in map(DimmInfo.from_record, records) if dimm] + + if not dimms: self._log_event( category=EventCategory.IO, description="DIMM info not found", @@ -150,5 +84,154 @@ def collect_data( ) self.result.message = "DIMM info not found" self.result.status = ExecutionStatus.ERROR + return self.result, None + + dimm_data = DimmDataModel(dimms=dimms) + self.result.message = f"DIMM: {dimm_data}" return self.result, dimm_data + + def _collect_dmidecode_records(self) -> list[dict[str, str]]: + """Dump the DMI tables and pull the memory device records out of them. + + A full dump is collected first and kept as an artifact, then the type 17 + records are read from a quiet, targeted dump. The full dump is used as a + fallback when the targeted one fails, since it carries the same records. + + Returns: + list[dict[str, str]]: one raw record per memory device slot. + """ + full_dump = self._run_dmidecode(self.CMD_DMIDECODE_FULL) + if full_dump: + self.result.artifacts.append( + TextFileArtifact(filename="dmidecode.txt", contents=full_dump) + ) + else: + self._log_event( + category=EventCategory.OS, + description="Could not collect full dmidecode output", + priority=EventPriority.WARNING, + ) + + dump = self._run_dmidecode(self.CMD_DMIDECODE) or full_dump + if not dump: + return [] + + return self._parse_dmidecode(dump) + + def _run_dmidecode(self, command: str) -> Optional[str]: + """Run a dmidecode command and return its output. + + Args: + command (str): dmidecode command to run. + + Returns: + Optional[str]: raw stdout, or None if the command produced none. + """ + res = self._run_sut_cmd(command, sudo=True) + + # Hosts that already run as root often ship no sudo binary at all, so a + # sudo specific failure is worth one retry without it. + if res.exit_code != 0 and "sudo" in str(res.stderr).lower(): + res = self._run_sut_cmd(command, sudo=False) + + if res.exit_code != 0 or not res.stdout: + self._log_cmd_error(res) + return None + + return res.stdout + + def _collect_windows_records(self) -> list[dict[str, str]]: + """Dump every Win32_PhysicalMemory property and split it into records. + + Returns: + list[dict[str, str]]: one raw record per memory device slot. + """ + res = self._run_sut_cmd(self.CMD_WINDOWS) + if res.exit_code != 0 or not res.stdout: + self._log_cmd_error(res) + return [] + + self.result.artifacts.append( + TextFileArtifact(filename="memorychip.csv", contents=res.stdout) + ) + + return self._parse_wmic_csv(res.stdout) + + def _log_cmd_error(self, res: CommandArtifact) -> None: + """Log a failed memory query without aborting the rest of the collection. + + Args: + res (CommandArtifact): result of the failed command. + """ + self._log_event( + category=EventCategory.OS, + description="Error checking dimms", + priority=EventPriority.ERROR, + data=res.model_dump(include={"command", "stderr", "exit_code"}), + console_log=True, + ) + + @classmethod + def _parse_dmidecode(cls, dump: str) -> list[dict[str, str]]: + """Split a raw dmidecode dump into its memory device records. + + Only DMI type 17 records are kept, and only their top level fields, so + that the "Volatile Size", "Cache Size" and "Logical Size" fields nested + under NVDIMM records are never mistaken for a module capacity. + + Args: + dump (str): raw dmidecode output. + + Returns: + list[dict[str, str]]: one raw record per memory device slot. + """ + records: list[dict[str, str]] = [] + fields: Optional[dict[str, str]] = None + + for line in dump.splitlines(): + field = line.strip() + if not field: + # A blank line closes the current record. + fields = None + elif field.startswith("Handle "): + # Without -q every record opens with a handle line naming its + # DMI type, which is the most precise way to spot type 17. + fields = None + if "DMI type 17," in field: + fields = {} + records.append(fields) + elif field == cls.MEMORY_DEVICE: + # Under -q the handle lines are hidden, so the section title is + # what opens the record. + if fields is None: + fields = {} + records.append(fields) + elif fields is not None: + key, separator, value = field.partition(":") + key = key.strip() + # Nested fields repeat names such as "Size", so the first value + # seen for a key wins. + if separator and key not in fields: + fields[key] = value.strip() + + return records + + @staticmethod + def _parse_wmic_csv(stdout: str) -> list[dict[str, str]]: + """Split `wmic memorychip get /format:csv` output into records. + + wmic emits a header row naming each property followed by one row per + module, so the header is used to key the fields and any property that a + given Windows build does not report is simply absent from the record. + + Args: + stdout (str): raw wmic output. + + Returns: + list[dict[str, str]]: one raw record per memory device slot. + """ + # wmic pads its output with blank lines that confuse the csv reader. + rows = [line for line in stdout.splitlines() if line.strip()] + + return list(csv.DictReader(rows)) diff --git a/nodescraper/plugins/inband/dimm/dimm_plugin.py b/nodescraper/plugins/inband/dimm/dimm_plugin.py index 30dcc86d..5a0f5672 100644 --- a/nodescraper/plugins/inband/dimm/dimm_plugin.py +++ b/nodescraper/plugins/inband/dimm/dimm_plugin.py @@ -23,7 +23,10 @@ # SOFTWARE. # ############################################################################### +from typing import Optional + from nodescraper.base import InBandDataPlugin +from nodescraper.interfaces.dataplugin import CollectorArgsClasses, CollectorClasses from .collector_args import DimmCollectorArgs from .dimm_collector import DimmCollector @@ -33,8 +36,8 @@ class DimmPlugin(InBandDataPlugin[DimmDataModel, DimmCollectorArgs, None]): """Plugin for collection and analysis of DIMM data""" - DATA_MODEL = DimmDataModel + DATA_MODEL: type[DimmDataModel] = DimmDataModel - COLLECTOR = DimmCollector + COLLECTOR: Optional[CollectorClasses] = DimmCollector - COLLECTOR_ARGS = DimmCollectorArgs + COLLECTOR_ARGS: Optional[CollectorArgsClasses] = DimmCollectorArgs diff --git a/nodescraper/plugins/inband/dimm/dimmdata.py b/nodescraper/plugins/inband/dimm/dimmdata.py index 3bc80a8c..9bd25f92 100644 --- a/nodescraper/plugins/inband/dimm/dimmdata.py +++ b/nodescraper/plugins/inband/dimm/dimmdata.py @@ -23,8 +23,366 @@ # SOFTWARE. # ############################################################################### +from typing import ClassVar, Optional, Union + +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + ValidationError, + computed_field, + field_validator, +) +from typing_extensions import override + from nodescraper.models import DataModel +from nodescraper.utils import SIZE_UNITS, bytes_to_human_readable + +# Field values that mean "nothing here" rather than a real value. +PLACEHOLDERS = frozenset( + { + "", + "unknown", + "not specified", + "not provided", + "none", + "no module installed", + "[empty]", + "to be filled by o.e.m.", + } +) + +# SMBIOS memory type codes (SMBIOS spec 7.18.2). dmidecode resolves these +# itself; on Windows they arrive as raw codes in SMBIOSMemoryType. +SMBIOS_MEMORY_TYPES = { + 1: "Other", + 2: "Unknown", + 3: "DRAM", + 15: "SDRAM", + 16: "SGRAM", + 17: "RDRAM", + 18: "DDR", + 19: "DDR2", + 20: "DDR2 FB-DIMM", + 24: "DDR3", + 25: "FBD2", + 26: "DDR4", + 27: "LPDDR", + 28: "LPDDR2", + 29: "LPDDR3", + 30: "LPDDR4", + 31: "Logical non-volatile device", + 32: "HBM", + 33: "HBM2", + 34: "DDR5", + 35: "LPDDR5", + 36: "HBM3", +} + +# Win32_PhysicalMemory FormFactor codes. WMI uses its own enumeration here, +# which does not match the SMBIOS form factor codes. +WMI_FORM_FACTORS = { + 0: "Unknown", + 1: "Other", + 2: "SIP", + 3: "DIP", + 4: "ZIP", + 5: "SOJ", + 6: "Proprietary", + 7: "SIMM", + 8: "DIMM", + 9: "TSOP", + 10: "PGA", + 11: "RIMM", + 12: "SODIMM", + 13: "SRIMM", + 14: "SMD", + 15: "SSMP", + 16: "QFP", + 17: "TQFP", + 18: "SOIC", + 19: "LCC", + 20: "PLCC", + 21: "BGA", + 22: "FPBGA", + 23: "LGA", +} + + +def clean(value: str) -> Optional[str]: + """Normalise a text field, mapping placeholder values to None.""" + value = value.strip() + return value if value.lower() not in PLACEHOLDERS else None + + +def parse_value(value: str) -> Optional[tuple[int, str]]: + """Split a numeric field into its number and its unit. + + Every numeric field either source reports is a count followed by an + optional unit, e.g. "64 GB", "4800 MT/s", "80 bits" or a bare "2". + + Args: + value (str): raw field. + + Returns: + Optional[tuple[int, str]]: the number and its unit, the unit being + empty when the field carries none. None when the field does not start + with a number at all, e.g. "No Module Installed" or "DDR5". + """ + number, _, unit = value.strip().partition(" ") + if not number.isdigit(): + return None + + return int(number), unit.strip() + + +def parse_size(value: str, si: bool = True) -> Optional[int]: + """Convert a size field such as "64 GB" into bytes. + + Args: + value (str): raw size field. + + Returns: + Optional[int]: size in bytes, or None when the slot holds no usable + module, e.g. "No Module Installed", "Unknown" or a zero size. + """ + parsed = parse_value(value) + if not parsed: + return None + + size, unit = parsed + + factor = SIZE_UNITS.get(unit.upper() or "B") + if not factor: + return None + + return size * factor or None + + +def decode(codes: dict[int, str], value: str) -> Optional[str]: + """Resolve an SMBIOS or WMI enum code to its name. + + Values that are already named, as dmidecode reports them, pass through + untouched. + + Args: + codes (dict[int, str]): enum table to resolve against. + value (str): raw code or name. + + Returns: + Optional[str]: the decoded name, the raw code if the table does not + cover it, or None if the value carries no meaning. + """ + parsed = parse_value(value) + # dmidecode names the value outright, Windows reports a bare enum code. + if not parsed or parsed[1]: + return clean(value) + + code = parsed[0] + return clean(codes.get(code, f"Type {code}")) + + +class DimmInfo(BaseModel): + """Details of a single populated memory slot. + + Every field is aliased to the names dmidecode and Win32_PhysicalMemory use + for it, so a raw record from either source validates straight into the + model and the units, enum codes and placeholder values are decoded here + rather than by the caller. + """ + + model_config: ClassVar[ConfigDict] = ConfigDict(populate_by_name=True) + + size: str = Field( + validation_alias=AliasChoices("size_bytes", "Size", "Capacity"), + description="Module capacity as reported by the system", + ) + locator: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("locator", "Locator", "DeviceLocator"), + description="Slot the module is installed in", + ) + bank_locator: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("bank_locator", "Bank Locator", "BankLabel"), + description="Memory bank the slot belongs to", + ) + manufacturer: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("manufacturer", "Manufacturer"), + ) + part_number: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("part_number", "Part Number", "PartNumber"), + ) + serial_number: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("serial_number", "Serial Number", "SerialNumber"), + ) + memory_type: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("memory_type", "Type", "SMBIOSMemoryType"), + description="Memory technology, e.g. DDR5", + ) + form_factor: Optional[str] = Field( + default=None, + validation_alias=AliasChoices("form_factor", "Form Factor", "FormFactor"), + description="Physical form factor, e.g. DIMM", + ) + speed_mts: Optional[int] = Field( + default=None, + validation_alias=AliasChoices("speed_mts", "Speed"), + description="Rated speed in MT/s", + ) + configured_speed_mts: Optional[int] = Field( + default=None, + validation_alias=AliasChoices( + "configured_speed_mts", "Configured Memory Speed", "ConfiguredClockSpeed" + ), + description="Speed the module is running at in MT/s", + ) + rank: Optional[int] = Field( + default=None, + validation_alias=AliasChoices("rank", "Rank"), + ) + data_width_bits: Optional[int] = Field( + default=None, + validation_alias=AliasChoices("data_width_bits", "Data Width", "DataWidth"), + ) + total_width_bits: Optional[int] = Field( + default=None, + validation_alias=AliasChoices("total_width_bits", "Total Width", "TotalWidth"), + description="Data width plus any error correction width", + ) + + @field_validator( + "locator", + "bank_locator", + "manufacturer", + "part_number", + "serial_number", + mode="before", + ) + @classmethod + def text_conformer(cls, value: object) -> object: + """Drop placeholder text such as "Unknown" or "Not Specified".""" + return clean(value) if isinstance(value, str) else value + + @field_validator( + "speed_mts", + "configured_speed_mts", + "rank", + "data_width_bits", + "total_width_bits", + mode="before", + ) + @classmethod + def int_conformer(cls, value: object) -> object: + """Take the number, dropping units such as the "MT/s" of a speed.""" + if not isinstance(value, str): + return value + + parsed = parse_value(value) + return parsed[0] if parsed else None + + @field_validator("memory_type", mode="before") + @classmethod + def memory_type_conformer(cls, value: object) -> object: + """Resolve the SMBIOSMemoryType codes that Windows reports.""" + return decode(SMBIOS_MEMORY_TYPES, value) if isinstance(value, str) else value + + @field_validator("form_factor", mode="before") + @classmethod + def form_factor_conformer(cls, value: object) -> object: + """Resolve the FormFactor codes that Windows reports.""" + return decode(WMI_FORM_FACTORS, value) if isinstance(value, str) else value + + @classmethod + def from_record(cls, record: dict[str, str]) -> Optional["DimmInfo"]: + """Build a module from a raw dmidecode or wmic record. + + Args: + record (dict[str, str]): raw field names mapped to their raw values. + + Returns: + Optional[DimmInfo]: the module, or None if the record describes an + empty slot or is too malformed to decode. + """ + try: + return cls.model_validate(record) + except ValidationError: + return None + + @computed_field # type: ignore[misc] + @property + def human_size(self) -> str: + """Module capacity in bytes.""" + if not self.size_bytes: + return "Unknown Size" + return bytes_to_human_readable(self.size_bytes) + + @computed_field # type: ignore[misc] + @property + def size_bytes(self) -> Union[int, None]: + """Module capacity in bytes.""" + return parse_size(self.size) + + @override + def __str__(self) -> str: + """Describe the module, e.g. "DIMM 0: 64GB DDR5 4800MT/s Micron".""" + details = [ + detail + for detail in ( + self.human_size or "Unknown Size", + self.memory_type, + f"{self.speed_mts}MT/s" if self.speed_mts else None, + self.manufacturer, + ) + if detail + ] + summary = " ".join(details) + return f"{self.locator}: {summary}" if self.locator else summary class DimmDataModel(DataModel): - dimms: str + """Inventory of the memory modules installed in the system""" + + dimms: list[DimmInfo] = Field(default_factory=list) + + @computed_field # type: ignore[misc] + @property + def dimm_count(self) -> int: + """Number of populated memory slots.""" + return len(self.dimms) + + @computed_field # type: ignore[misc] + @property + def total_size_bytes(self) -> int: + """Combined capacity of every populated module, in bytes.""" + return sum(dimm.size_bytes or 0 for dimm in self.dimms) + + @computed_field # type: ignore[misc] + @property + def human_size(self) -> str: + """Module capacity in bytes.""" + return bytes_to_human_readable(self.total_size_bytes) + + @computed_field # type: ignore[misc] + @property + def population(self) -> dict[Union[int, str], int]: + """Module count keyed by capacity, smallest capacity first.""" + counts: dict[Union[int, str], int] = {} + for dimm in self.dimms: + counts[dimm.human_size or "Unknown Size Bytes"] = ( + counts.get(dimm.human_size or "Unknown Size Bytes", 0) + 1 + ) + return {size: count for size, count in sorted(counts.items())} + + @override + def __str__(self) -> str: + """Summarise the inventory, e.g. "256GB @ 2 x 64GB 1 x 128GB".""" + if not self.dimms: + return "0GB" + breakdown = ", ".join(f"{count} x {size}" for size, count in self.population.items()) + return f"{self.human_size} @ {breakdown}" diff --git a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py index 5ae53f77..5bf93c36 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_analyzer.py @@ -30,7 +30,7 @@ from nodescraper.base.match_ignore import parse_ignore_match_rules from nodescraper.base.regexanalyzer import ErrorRegex, RegexAnalyzer from nodescraper.connection.inband import TextFileArtifact -from nodescraper.enums import EventCategory, EventPriority +from nodescraper.enums import EventCategory, EventPriority, OSFamily from nodescraper.models import Event, TaskResult from .analyzer_args import DmesgAnalyzerArgs @@ -47,10 +47,24 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): - """Check dmesg for errors""" + """Check dmesg (Linux) or vmkernel.log (ESXi) for errors""" DATA_MODEL = DmesgData + # ESXi vmkernel.log timestamp, e.g. "2026-08-05T19:53:35.178Z" (ISO8601 dot-ms + Z). + # Linux uses the base RegexAnalyzer.TIMESTAMP_PATTERN (comma-form). + ESXI_TIMESTAMP_PATTERN: re.Pattern = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)") + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # On ESXi, extract vmkernel.log timestamps so event grouping and date-range + # filtering both work; Linux keeps the base comma-form pattern. + if self._is_esxi(): + self.TIMESTAMP_PATTERN = self.ESXI_TIMESTAMP_PATTERN + + def _is_esxi(self) -> bool: + return self.system_info.os_family == OSFamily.ESXI + ERROR_REGEX: list[ErrorRegex] = [ ErrorRegex( regex=re.compile(r"(?:oom_kill_process.*)|(?:Out of memory.*)"), @@ -273,6 +287,30 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): message="RAS Deferred Error", event_category=EventCategory.RAS, ), + # ESXi mxGPU (gim/amdgpuv) RAS phrasing differs from Linux: the block name is + # capitalized ("... detected in MMHUB Block."), there is no "in total", and no + # "kern :err:" prefix. These match the ESXi host-driver forms and are inert on + # Linux logs (which use the lowercase "in total in block" phrasing above). + ErrorRegex( + regex=re.compile(r"(\d+ new uncorrectable hardware errors detected in \w+ Block.*)"), + message="RAS Uncorrectable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(\d+ new correctable hardware errors detected in \w+ Block.*)"), + message="RAS Correctable Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(GPU detected ECC Fatal Error\.)"), + message="RAS ECC Fatal Error", + event_category=EventCategory.RAS, + ), + ErrorRegex( + regex=re.compile(r"(Issuing Whole GPU reset\.)"), + message="GPU Reset", + event_category=EventCategory.RAS, + ), ErrorRegex( regex=re.compile( r"((?:\[Hardware Error\]:\s+)?event severity: corrected.*)" @@ -463,6 +501,15 @@ class DmesgAnalyzer(RegexAnalyzer[DmesgData, DmesgAnalyzerArgs]): ), ] + # Date-range filtering must recognize both Linux dmesg comma-form timestamps + # (2024-10-01T05:00:00,000000-05:00) and ESXi vmkernel.log dot-ms/Z timestamps + # (2026-08-20T09:35:58.380Z). filter_dmesg stays a classmethod (public API), so it + # carries its own combined pattern rather than the instance TIMESTAMP_PATTERN. + _FILTER_TIMESTAMP_PATTERN: re.Pattern = re.compile( + r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)" + r"|(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)" + ) + @classmethod def filter_dmesg( cls, @@ -470,7 +517,7 @@ def filter_dmesg( analysis_range_start: Optional[datetime.datetime] = None, analysis_range_end: Optional[datetime.datetime] = None, ) -> str: - """Filter a dmesg log by date + """Filter a dmesg (Linux) or vmkernel.log (ESXi) log by date Args: dmesg_content (str): unfiltered dmesg log @@ -482,9 +529,16 @@ def filter_dmesg( filtered_dmesg = "" found_start = False if analysis_range_start else True for line in dmesg_content.splitlines(): - date = re.search(r"(\d{4}-\d+-\d+T\d+:\d+:\d+),(\d+[+-]\d+:\d+)", line) - if date is not None: - date = datetime.datetime.fromisoformat(f"{date.group(1)}.{date.group(2)}") + match = cls._FILTER_TIMESTAMP_PATTERN.search(line) + if match is not None: + if match.group(1) is not None: + # Linux comma-form: swap the comma for a dot so fromisoformat accepts it + iso = f"{match.group(1)}.{match.group(2)}" + else: + # ESXi dot-Z form: normalize the trailing Z so fromisoformat accepts it + # on Python < 3.11 as well + iso = match.group(3).replace("Z", "+00:00") + date = datetime.datetime.fromisoformat(iso) # show date in UTC now date = date.astimezone(datetime.timezone.utc) if analysis_range_start and not found_start and date >= analysis_range_start: @@ -743,11 +797,22 @@ def analyze_data( self.result.events += known_err_events if args.check_unknown_dmesg_errors: + if self._is_esxi(): + # ESXi vmkernel severity tokens are unreliable (-ALERT is used for benign + # boot notices; -ERROR/-CRIT are never emitted). The reliable error signal + # is the driver-internal severity in the message body: "gim error/warning", + # "amdgpuv error/warning", or the bracket form "[amdgpuv warn]". + unknown_error_regex = re.compile( + r"(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning) [^:]*:\s*(.*)" + r"|\[(?:gim|amdgpuv|amdgpu) (?:err|error|warn|warning)\]:?\s*(.*)" + ) + else: + unknown_error_regex = re.compile( + r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" + ) unknown_dmesg_error_regexes = [ ErrorRegex( - regex=re.compile( - r"kern :(?:err|crit|alert|emerg)\s+: \d{4}-\d+-\d+T\d+:\d+:\d+,\d+[+-]\d+:\d+ (.*)" - ), + regex=unknown_error_regex, message="Unknown dmesg error", event_category=EventCategory.UNKNOWN, event_priority=EventPriority.WARNING, diff --git a/nodescraper/plugins/inband/dmesg/dmesg_collector.py b/nodescraper/plugins/inband/dmesg/dmesg_collector.py index c280d7d2..4fcbc4b7 100644 --- a/nodescraper/plugins/inband/dmesg/dmesg_collector.py +++ b/nodescraper/plugins/inband/dmesg/dmesg_collector.py @@ -38,24 +38,39 @@ class DmesgCollector(InBandDataCollector[DmesgData, DmesgCollectorArgs]): """Read dmesg log""" - SUPPORTED_OS_FAMILY = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY = {OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = DmesgData CMD = "dmesg --time-format iso -x" + # ESXi has no dmesg ring buffer; the kernel log is the vmkernel.log file. + CMD_ESXI = "cat /var/log/vmkernel.log" CMD_LOGS = ( r"ls -1 /var/log/dmesg* 2>/dev/null | grep -E '^/var/log/dmesg(\.[0-9]+(\.gz)?)?$' || true" ) + # ESXi rotates vmkernel.log to vmkernel. / vmkernel..gz. + CMD_LOGS_ESXI = r"ls -1 /var/log/vmkernel.* 2>/dev/null | grep -E '^/var/log/vmkernel\.[0-9]+(\.gz)?$' || true" def _collect_dmesg_rotations(self): - """Collect dmesg logs""" - list_res = self._run_sut_cmd(self.CMD_LOGS, sudo=True) + """Collect dmesg (Linux) / vmkernel.log (ESXi) rotated logs""" + is_esxi = self.system_info.os_family == OSFamily.ESXI + if is_esxi: + log_label = "vmkernel" + cmd_logs = self.CMD_LOGS_ESXI + else: + log_label = "dmesg" + cmd_logs = self.CMD_LOGS + list_res = self._run_sut_cmd(cmd_logs, sudo=True) paths = [p.strip() for p in (list_res.stdout or "").splitlines() if p.strip()] if not paths: + if is_esxi: + description = "No /var/log/vmkernel.log files found (including rotations)." + else: + description = "No /var/log/dmesg files found (including rotations)." self._log_event( category=EventCategory.OS, - description="No /var/log/dmesg files found (including rotations).", + description=description, data={"list_exit_code": list_res.exit_code}, priority=EventPriority.WARNING, ) @@ -68,7 +83,7 @@ def _collect_dmesg_rotations(self): cmd = f"gzip -dc {qp} 2>/dev/null || zcat {qp} 2>/dev/null" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -84,7 +99,7 @@ def _collect_dmesg_rotations(self): cmd = f"cat {qp}" res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code == 0 and res.stdout is not None: - fname = nice_rotated_name(p, "dmesg") + fname = nice_rotated_name(p, log_label) self.logger.info("Collected dmesg log: %s", fname) self.result.artifacts.append( TextFileArtifact(filename=fname, contents=res.stdout) @@ -121,8 +136,13 @@ def _get_dmesg_content(self) -> str: str: dmesg output """ - self.logger.info("Running dmesg command on system") - res = self._run_sut_cmd(self.CMD, sudo=True, log_artifact=False) + is_esxi = self.system_info.os_family == OSFamily.ESXI + if is_esxi: + cmd = self.CMD_ESXI + else: + cmd = self.CMD + self.logger.info("Reading kernel log from system") + res = self._run_sut_cmd(cmd, sudo=True, log_artifact=False) if res.exit_code != 0: self._log_event( category=EventCategory.OS, diff --git a/nodescraper/plugins/inband/dmesg/run_compare.py b/nodescraper/plugins/inband/dmesg/run_compare.py index 11c536b8..f054252d 100644 --- a/nodescraper/plugins/inband/dmesg/run_compare.py +++ b/nodescraper/plugins/inband/dmesg/run_compare.py @@ -1,82 +1,82 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import logging -import os -from typing import Optional, Tuple - -from .dmesg_plugin import DmesgPlugin -from .dmesgdata import DmesgData - - -def find_dmesg_datamodel_path(run_path: str) -> Optional[str]: - """Find the DmesgPlugin collector datamodel under a scraper run directory.""" - return DmesgPlugin.find_datamodel_path_in_run(run_path) - - -def load_dmesg_data(path: str) -> Tuple[Optional[DmesgData], Optional[str]]: - """Load DmesgData from a scraper run directory or a datamodel file path.""" - path = os.path.abspath(path) - if not os.path.exists(path): - return None, None - dm_path = path if os.path.isfile(path) else DmesgPlugin.find_datamodel_path_in_run(path) - if not dm_path: - return None, None - dm = DmesgPlugin.load_datamodel_from_path(dm_path) - return (dm, dm_path) if dm is not None else (None, None) - - -def compute_extracted_errors(dm: DmesgData) -> list[str]: - """Apply DmesgPlugin analyzer regexes to dmesg content (in memory only).""" - out = DmesgPlugin.get_extracted_errors(dm) - return out if out is not None else [] - - -def compare_dmesg_runs( - path1: str, - path2: str, - logger: Optional[logging.Logger] = None, -) -> Tuple[list[str], list[str], str, str]: - """Load two DmesgPlugin runs, compute extracted errors in memory, and compare.""" - log = logger or logging.getLogger(__name__) - label1 = os.path.basename(path1.rstrip(os.sep)) - label2 = os.path.basename(path2.rstrip(os.sep)) - - d1 = DmesgPlugin.load_run_data(path1) - d2 = DmesgPlugin.load_run_data(path2) - - if d1 is None: - log.warning("No DmesgPlugin datamodel found at: %s", path1) - return [], [], label1, label2 - if d2 is None: - log.warning("No DmesgPlugin datamodel found at: %s", path2) - return [], [], label1, label2 - - err1 = set(d1.get("extracted_errors") or []) - err2 = set(d2.get("extracted_errors") or []) - only_in_1 = sorted(err1 - err2) - only_in_2 = sorted(err2 - err1) - - return only_in_1, only_in_2, label1, label2 +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging +import os +from typing import Optional, Tuple + +from .dmesg_plugin import DmesgPlugin +from .dmesgdata import DmesgData + + +def find_dmesg_datamodel_path(run_path: str) -> Optional[str]: + """Find the DmesgPlugin collector datamodel under a scraper run directory.""" + return DmesgPlugin.find_datamodel_path_in_run(run_path) + + +def load_dmesg_data(path: str) -> Tuple[Optional[DmesgData], Optional[str]]: + """Load DmesgData from a scraper run directory or a datamodel file path.""" + path = os.path.abspath(path) + if not os.path.exists(path): + return None, None + dm_path = path if os.path.isfile(path) else DmesgPlugin.find_datamodel_path_in_run(path) + if not dm_path: + return None, None + dm = DmesgPlugin.load_datamodel_from_path(dm_path) + return (dm, dm_path) if dm is not None else (None, None) + + +def compute_extracted_errors(dm: DmesgData) -> list[str]: + """Apply DmesgPlugin analyzer regexes to dmesg content (in memory only).""" + out = DmesgPlugin.get_extracted_errors(dm) + return out if out is not None else [] + + +def compare_dmesg_runs( + path1: str, + path2: str, + logger: Optional[logging.Logger] = None, +) -> Tuple[list[str], list[str], str, str]: + """Load two DmesgPlugin runs, compute extracted errors in memory, and compare.""" + log = logger or logging.getLogger(__name__) + label1 = os.path.basename(path1.rstrip(os.sep)) + label2 = os.path.basename(path2.rstrip(os.sep)) + + d1 = DmesgPlugin.load_run_data(path1) + d2 = DmesgPlugin.load_run_data(path2) + + if d1 is None: + log.warning("No DmesgPlugin datamodel found at: %s", path1) + return [], [], label1, label2 + if d2 is None: + log.warning("No DmesgPlugin datamodel found at: %s", path2) + return [], [], label1, label2 + + err1 = set(d1.get("extracted_errors") or []) + err2 = set(d2.get("extracted_errors") or []) + only_in_1 = sorted(err1 - err2) + only_in_2 = sorted(err2 - err1) + + return only_in_1, only_in_2, label1, label2 diff --git a/nodescraper/plugins/inband/fabrics/fabrics_collector.py b/nodescraper/plugins/inband/fabrics/fabrics_collector.py index 59c884de..5cac68f6 100644 --- a/nodescraper/plugins/inband/fabrics/fabrics_collector.py +++ b/nodescraper/plugins/inband/fabrics/fabrics_collector.py @@ -24,7 +24,7 @@ # ############################################################################### import re -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union from nodescraper.base import InBandDataCollector from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus @@ -79,6 +79,8 @@ def _parse_ibstat(self, output: str) -> List[IbstatDevice]: if line.startswith("CA "): # Save previous device if exists if current_device: + if current_port is not None: + current_device.ports[current_port] = current_port_attrs devices.append(current_device) # Extract CA name @@ -90,7 +92,7 @@ def _parse_ibstat(self, output: str) -> List[IbstatDevice]: current_port_attrs = {} # Port line (e.g., "Port 1:") - elif line.startswith("Port ") and ":" in line: + elif re.match(r"Port\s+\d+:\s*$", line_stripped): # Save previous port if exists if current_device and current_port is not None: current_device.ports[current_port] = current_port_attrs @@ -158,6 +160,8 @@ def _parse_ibv_devinfo(self, output: str) -> List[IbvDeviceInfo]: if line.startswith("hca_id:"): # Save previous device if exists if current_device: + if current_port is not None: + current_device.ports[current_port] = current_port_attrs devices.append(current_device) parts = line.split(":", 1) @@ -418,9 +422,26 @@ def collect_data( res_ibstat = self._run_sut_cmd(self.CMD_IBSTAT) if res_ibstat.exit_code == 0: ibstat_devices = self._parse_ibstat(res_ibstat.stdout) + ibstat_ports = [] + for device in ibstat_devices: + for port_num, attrs in device.ports.items(): + port: Dict[str, Union[int, str, None]] = { + "device": device.ca_name, + "port": port_num, + } + if "Link layer" in attrs: + port["link_layer"] = attrs["Link layer"] + if "Rate" in attrs: + port["rate"] = attrs["Rate"] + if "State" in attrs: + port["state"] = attrs["State"] + if "Active MTU" in attrs: + port["active_mtu"] = attrs["Active MTU"] + ibstat_ports.append(port) self._log_event( category=EventCategory.NETWORK, description=f"Collected {len(ibstat_devices)} IB devices from ibstat", + data={"ports": ibstat_ports}, priority=EventPriority.INFO, ) else: @@ -435,9 +456,29 @@ def collect_data( res_ibv = self._run_sut_cmd(self.CMD_IBV_DEVINFO) if res_ibv.exit_code == 0: ibv_devices = self._parse_ibv_devinfo(res_ibv.stdout) + ibv_device_info = [] + for ibv_device in ibv_devices: + ports = [] + for port_num, attrs in ibv_device.ports.items(): + port_entry: Dict[str, Union[int, str]] = {"port": port_num} + if "state" in attrs: + port_entry["state"] = attrs["state"] + if "link_layer" in attrs: + port_entry["link_layer"] = attrs["link_layer"] + if "active_mtu" in attrs: + port_entry["active_mtu"] = attrs["active_mtu"] + ports.append(port_entry) + ibv_device_info.append( + { + "device": ibv_device.device, + "transport": ibv_device.transport_type, + "ports": ports, + } + ) self._log_event( category=EventCategory.NETWORK, description=f"Collected {len(ibv_devices)} IB devices from ibv_devinfo", + data={"devices": ibv_device_info}, priority=EventPriority.INFO, ) else: diff --git a/nodescraper/plugins/inband/journal/analyzer_args.py b/nodescraper/plugins/inband/journal/analyzer_args.py index ee5390c1..6a30b3ae 100644 --- a/nodescraper/plugins/inband/journal/analyzer_args.py +++ b/nodescraper/plugins/inband/journal/analyzer_args.py @@ -1,43 +1,43 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Optional - -from pydantic import Field - -from nodescraper.models import TimeRangeAnalysisArgs - - -class JournalAnalyzerArgs(TimeRangeAnalysisArgs): - """Arguments for journal analyzer""" - - check_priority: Optional[int] = Field( - default=None, - description="Check against journal log priority (0=emergency..7=debug). If an entry has priority <= check_priority, an ERROR event is raised.", - ) - group: bool = Field( - default=True, - description="If True, group entries that have the same priority and message.", - ) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import TimeRangeAnalysisArgs + + +class JournalAnalyzerArgs(TimeRangeAnalysisArgs): + """Arguments for journal analyzer""" + + check_priority: Optional[int] = Field( + default=None, + description="Check against journal log priority (0=emergency..7=debug). If an entry has priority <= check_priority, an ERROR event is raised.", + ) + group: bool = Field( + default=True, + description="If True, group entries that have the same priority and message.", + ) diff --git a/nodescraper/plugins/inband/journal/collector_args.py b/nodescraper/plugins/inband/journal/collector_args.py index 6832ffc7..3ee9953f 100644 --- a/nodescraper/plugins/inband/journal/collector_args.py +++ b/nodescraper/plugins/inband/journal/collector_args.py @@ -1,38 +1,38 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2025 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -from typing import Optional - -from pydantic import Field - -from nodescraper.models import CollectorArgs - - -class JournalCollectorArgs(CollectorArgs): - boot: Optional[int] = Field( - default=None, - description="Optional boot ID to limit journal collection to a specific boot.", - ) +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class JournalCollectorArgs(CollectorArgs): + boot: Optional[int] = Field( + default=None, + description="Optional boot ID to limit journal collection to a specific boot.", + ) diff --git a/nodescraper/plugins/inband/journal/journal_analyzer.py b/nodescraper/plugins/inband/journal/journal_analyzer.py index fbf14eaf..3fd73389 100644 --- a/nodescraper/plugins/inband/journal/journal_analyzer.py +++ b/nodescraper/plugins/inband/journal/journal_analyzer.py @@ -1,238 +1,241 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from datetime import datetime -from typing import Optional, TypedDict - -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.interfaces import DataAnalyzer -from nodescraper.models import TaskResult - -from .analyzer_args import JournalAnalyzerArgs -from .journaldata import JournalData, JournalJsonEntry - - -class JournalEvent(TypedDict): - count: int - first_occurrence: datetime - last_occurrence: datetime - - -class JournalPriority: - EMERGENCY = 0 - ALERT = 1 - CRITICAL = 2 - ERROR = 3 - WARNING = 4 - NOTICE = 5 - INFO = 6 - DEBUG = 7 - - -class JournalAnalyzer(DataAnalyzer[JournalData, JournalAnalyzerArgs]): - """Check journalctl for errors""" - - DATA_MODEL = JournalData - - @classmethod - def filter_journal( - cls, - journal_content_json: list[JournalJsonEntry], - analysis_range_start: Optional[datetime] = None, - analysis_range_end: Optional[datetime] = None, - ) -> list[JournalJsonEntry]: - """Filter a journal log by date - - Args: - journal_content_json (list[JournalJsonEntry]): unfiltered journal log - analysis_range_start (Optional[datetime], optional): start of analysis range. Defaults to None. - analysis_range_end (Optional[datetime], optional): end of analysis range. Defaults to None. - - Returns: - list[JournalJsonEntry]: filtered journal log - """ - - filtered_journal = [] - - found_start = False if analysis_range_start else True - - # Parse through the journal log and filter by date - for entry in journal_content_json: - date = entry.REALTIME_TIMESTAMP - - # Skip entries without valid timestamp - if date is None: - continue - - if analysis_range_start and not found_start and date >= analysis_range_start: - found_start = True - elif analysis_range_end and date >= analysis_range_end: - break - - # only read entries after starting timestamp is found, ignore entries that do not have valid date - if found_start: - filtered_journal.append(entry) - - return filtered_journal - - def _priority_to_entry_priority(self, priority: int) -> EventPriority: - if priority <= JournalPriority.ERROR: - entry_priority = EventPriority.ERROR - elif priority == JournalPriority.WARNING: - entry_priority = EventPriority.WARNING - elif priority >= JournalPriority.NOTICE: - entry_priority = EventPriority.INFO - else: - # Unknown? - entry_priority = EventPriority.ERROR - return entry_priority - - def _analyze_journal_entries_by_priority( - self, journal_content_json: list[JournalJsonEntry], priority: int, group: bool - ) -> None: - """Analyze a list of Journal Entries for a priority. - if WARNING, CRITICAL or it is unknown then log an error/warning Journal Entry. - Parameters - ---------- - journal_content_json : list[JournalJsonEntry] - List of JournalJsonEntry to analyze - priority : int - Priority threshold to check against - group : bool - Whether to group similar entries - """ - # Use a tuple of (message, priority) as the key instead of the JournalJsonEntry object - journal_event_map: dict[tuple[str, int], JournalEvent] = {} - - # Check against journal log priority levels. emergency(0), alert(1), critical(2), error(3), warning(4), notice(5), info(6), debug(7) - for entry in journal_content_json: - if entry.PRIORITY <= priority: - self.result.status = ExecutionStatus.ERROR - if not group: - entry_dict = entry.model_dump() # Convert JournalJsonEntry to dictionary - entry_dict["task_name"] = self.__class__.__name__ - self._log_event( - category=EventCategory.OS, - description="Journal log entry with priority level %s" % entry.PRIORITY, - data=entry_dict, - priority=self._priority_to_entry_priority(entry.PRIORITY), - console_log=False, - ) - else: - # Handle MESSAGE as either string or list - message = entry.MESSAGE - if isinstance(message, list): - message = " ".join(message) - - # Create a tuple key from message and priority - entry_key = (message, entry.PRIORITY) - if journal_event_map.get(entry_key) is None: - journal_event_map[entry_key] = { - "count": 1, - "first_occurrence": ( - entry.REALTIME_TIMESTAMP - if entry.REALTIME_TIMESTAMP - else datetime.fromtimestamp(0) - ), - "last_occurrence": ( - entry.REALTIME_TIMESTAMP - if entry.REALTIME_TIMESTAMP - else datetime.fromtimestamp(0) - ), - } - else: - journal_event_map[entry_key]["count"] += 1 - if entry.REALTIME_TIMESTAMP: - journal_event_map[entry_key][ - "last_occurrence" - ] = entry.REALTIME_TIMESTAMP - - # log all events that were grouped - if group: - for (message, entry_priority), event_data in journal_event_map.items(): - self._log_event( - category=EventCategory.OS, - description="Journal entries found in OS journal log", - priority=self._priority_to_entry_priority(entry_priority), - data={ - "message": message, - "priority": entry_priority, - "count": event_data["count"], - "first_occurrence": event_data["first_occurrence"], - "last_occurrence": event_data["last_occurrence"], - }, - console_log=False, - ) - - def analyze_data( - self, data: JournalData, args: Optional[JournalAnalyzerArgs] = None - ) -> TaskResult: - """Analyze the OS journal log for errors - - Parameters - ---------- - data : JournalData - Journal data to analyze - args : Optional[JournalAnalyzerArgs], optional - Analysis arguments, by default None - - Returns - ------- - TaskResult - A TaskResult object containing the result of the analysis - If journal log entries are found ExecutionStatus.OK - If journal log entries are found with priority level less than or equal to check_priority ExecutionStatus.ERROR - """ - if args is None: - args = JournalAnalyzerArgs() - - journal_content_json = data.journal_content_json - - # Filter by time range if specified - if args.analysis_range_start or args.analysis_range_end: - self.logger.info( - "Filtering journal log using range %s - %s", - args.analysis_range_start, - args.analysis_range_end, - ) - journal_content_json = self.filter_journal( - journal_content_json=journal_content_json, - analysis_range_start=args.analysis_range_start, - analysis_range_end=args.analysis_range_end, - ) - - self.result.status = ExecutionStatus.OK - - if args.check_priority is not None: - self._analyze_journal_entries_by_priority( - journal_content_json, args.check_priority, args.group - ) - - if self.result.status == ExecutionStatus.OK: - self.result.message = "No journal errors found" - else: - self.result.message = f"Found journal entries with priority <= {args.check_priority}" - - return self.result +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from datetime import datetime, timezone +from typing import Optional, TypedDict + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import JournalAnalyzerArgs +from .journaldata import JournalData, JournalJsonEntry + + +class JournalEvent(TypedDict): + count: int + first_occurrence: datetime + last_occurrence: datetime + + +class JournalPriority: + EMERGENCY = 0 + ALERT = 1 + CRITICAL = 2 + ERROR = 3 + WARNING = 4 + NOTICE = 5 + INFO = 6 + DEBUG = 7 + + +class JournalAnalyzer(DataAnalyzer[JournalData, JournalAnalyzerArgs]): + """Check journalctl for errors""" + + DATA_MODEL = JournalData + + @classmethod + def filter_journal( + cls, + journal_content_json: list[JournalJsonEntry], + analysis_range_start: Optional[datetime] = None, + analysis_range_end: Optional[datetime] = None, + ) -> list[JournalJsonEntry]: + """Filter a journal log by date + + Args: + journal_content_json (list[JournalJsonEntry]): unfiltered journal log + analysis_range_start (Optional[datetime], optional): start of analysis range. Defaults to None. + analysis_range_end (Optional[datetime], optional): end of analysis range. Defaults to None. + + Returns: + list[JournalJsonEntry]: filtered journal log + """ + + def as_utc(value: Optional[datetime]) -> Optional[datetime]: + if value is not None and value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + + analysis_range_start = as_utc(analysis_range_start) + analysis_range_end = as_utc(analysis_range_end) + + filtered_journal = [] + + found_start = False if analysis_range_start else True + + # Parse through the journal log and filter by date + for entry in journal_content_json: + date = entry.REALTIME_TIMESTAMP + + # Skip entries without valid timestamp + if date is None: + continue + + if analysis_range_start and not found_start and date >= analysis_range_start: + found_start = True + elif analysis_range_end and date >= analysis_range_end: + break + + # only read entries after starting timestamp is found, ignore entries that do not have valid date + if found_start: + filtered_journal.append(entry) + + return filtered_journal + + def _priority_to_entry_priority(self, priority: int) -> EventPriority: + if priority <= JournalPriority.ERROR: + entry_priority = EventPriority.ERROR + elif priority == JournalPriority.WARNING: + entry_priority = EventPriority.WARNING + elif priority >= JournalPriority.NOTICE: + entry_priority = EventPriority.INFO + else: + # Unknown? + entry_priority = EventPriority.ERROR + return entry_priority + + def _analyze_journal_entries_by_priority( + self, journal_content_json: list[JournalJsonEntry], priority: int, group: bool + ) -> None: + """Analyze a list of Journal Entries for a priority. + if WARNING, CRITICAL or it is unknown then log an error/warning Journal Entry. + Parameters + ---------- + journal_content_json : list[JournalJsonEntry] + List of JournalJsonEntry to analyze + priority : int + Priority threshold to check against + group : bool + Whether to group similar entries + """ + # Use a tuple of (message, priority) as the key instead of the JournalJsonEntry object + journal_event_map: dict[tuple[str, int], JournalEvent] = {} + + # Check against journal log priority levels. emergency(0), alert(1), critical(2), error(3), warning(4), notice(5), info(6), debug(7) + for entry in journal_content_json: + if entry.PRIORITY <= priority: + self.result.status = ExecutionStatus.ERROR + if not group: + entry_dict = entry.model_dump() # Convert JournalJsonEntry to dictionary + entry_dict["task_name"] = self.__class__.__name__ + self._log_event( + category=EventCategory.OS, + description="Journal log entry with priority level %s" % entry.PRIORITY, + data=entry_dict, + priority=self._priority_to_entry_priority(entry.PRIORITY), + console_log=False, + ) + else: + entry_key = (entry.MESSAGE, entry.PRIORITY) + missing_timestamp = datetime.fromtimestamp(0, tz=timezone.utc) + if journal_event_map.get(entry_key) is None: + journal_event_map[entry_key] = { + "count": 1, + "first_occurrence": ( + entry.REALTIME_TIMESTAMP + if entry.REALTIME_TIMESTAMP + else missing_timestamp + ), + "last_occurrence": ( + entry.REALTIME_TIMESTAMP + if entry.REALTIME_TIMESTAMP + else missing_timestamp + ), + } + else: + journal_event_map[entry_key]["count"] += 1 + if entry.REALTIME_TIMESTAMP: + journal_event_map[entry_key][ + "last_occurrence" + ] = entry.REALTIME_TIMESTAMP + + # log all events that were grouped + if group: + for (message, entry_priority), event_data in journal_event_map.items(): + self._log_event( + category=EventCategory.OS, + description="Journal entries found in OS journal log", + priority=self._priority_to_entry_priority(entry_priority), + data={ + "message": message, + "priority": entry_priority, + "count": event_data["count"], + "first_occurrence": event_data["first_occurrence"], + "last_occurrence": event_data["last_occurrence"], + }, + console_log=False, + ) + + def analyze_data( + self, data: JournalData, args: Optional[JournalAnalyzerArgs] = None + ) -> TaskResult: + """Analyze the OS journal log for errors + + Parameters + ---------- + data : JournalData + Journal data to analyze + args : Optional[JournalAnalyzerArgs], optional + Analysis arguments, by default None + + Returns + ------- + TaskResult + A TaskResult object containing the result of the analysis + If journal log entries are found ExecutionStatus.OK + If journal log entries are found with priority level less than or equal to check_priority ExecutionStatus.ERROR + """ + if args is None: + args = JournalAnalyzerArgs() + + journal_content_json = data.journal_content_json + + # Filter by time range if specified + if args.analysis_range_start or args.analysis_range_end: + self.logger.info( + "Filtering journal log using range %s - %s", + args.analysis_range_start, + args.analysis_range_end, + ) + journal_content_json = self.filter_journal( + journal_content_json=journal_content_json, + analysis_range_start=args.analysis_range_start, + analysis_range_end=args.analysis_range_end, + ) + + self.result.status = ExecutionStatus.OK + + if args.check_priority is not None: + self._analyze_journal_entries_by_priority( + journal_content_json, args.check_priority, args.group + ) + + if self.result.status == ExecutionStatus.OK: + self.result.message = "No journal errors found" + else: + self.result.message = f"Found journal entries with priority <= {args.check_priority}" + + return self.result diff --git a/nodescraper/plugins/inband/journal/journaldata.py b/nodescraper/plugins/inband/journal/journaldata.py index a6639422..fdc25e1f 100644 --- a/nodescraper/plugins/inband/journal/journaldata.py +++ b/nodescraper/plugins/inband/journal/journaldata.py @@ -24,10 +24,10 @@ # ############################################################################### import os -from datetime import datetime +from datetime import datetime, timezone from typing import Optional, Union -from pydantic import ConfigDict, Field, field_validator +from pydantic import AwareDatetime, ConfigDict, Field, field_validator from nodescraper.models import DataModel @@ -42,35 +42,34 @@ class JournalJsonEntry(DataModel): SYSLOG_IDENTIFIER: Optional[str] = Field(None, alias="SYSLOG_IDENTIFIER") CURSOR: Optional[str] = Field(None, alias="__CURSOR") SYSLOG_FACILITY: Optional[int] = Field(None, alias="SYSLOG_FACILITY") - SOURCE_REALTIME_TIMESTAMP: Optional[datetime] = Field(None, alias="_SOURCE_REALTIME_TIMESTAMP") - REALTIME_TIMESTAMP: Optional[datetime] = Field(None, alias="__REALTIME_TIMESTAMP") + SOURCE_REALTIME_TIMESTAMP: Optional[AwareDatetime] = Field( + None, alias="_SOURCE_REALTIME_TIMESTAMP" + ) + REALTIME_TIMESTAMP: Optional[AwareDatetime] = Field(None, alias="__REALTIME_TIMESTAMP") PRIORITY: int = Field(default=7, alias="PRIORITY") # Default to DEBUG (7) if not present BOOT_ID: Optional[str] = Field(None, alias="_BOOT_ID") SOURCE_MONOTONIC_TIMESTAMP: Optional[float] = Field(None, alias="_SOURCE_MONOTONIC_TIMESTAMP") MONOTONIC_TIMESTAMP: Optional[float] = Field(None, alias="__MONOTONIC_TIMESTAMP") - MESSAGE: Union[str, list[str]] = Field(default="", alias="MESSAGE") + MESSAGE: str = Field(default="", alias="MESSAGE") - # assume datetime has microseconds - @field_validator("SOURCE_REALTIME_TIMESTAMP", mode="before") - @classmethod - def validate_source_realtime_timestamp(cls, v): - if v is None: - return None - try: - return datetime.fromisoformat(v) - except ValueError: - return datetime.fromtimestamp(float(v) / 1e6) - - # assume datetime has microseconds - @field_validator("REALTIME_TIMESTAMP", mode="before") + @field_validator("SOURCE_REALTIME_TIMESTAMP", "REALTIME_TIMESTAMP", mode="before") @classmethod def validate_realtime_timestamp(cls, v): + """Return a UTC datetime.""" if v is None: return None - try: - return datetime.fromisoformat(v) - except ValueError: - return datetime.fromtimestamp(float(v) / 1e6) + + if isinstance(v, datetime): + parsed = v + else: + try: + parsed = datetime.fromisoformat(str(v)) + except ValueError: + return datetime.fromtimestamp(float(v) / 1e6, tz=timezone.utc) + + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) @field_validator("SOURCE_MONOTONIC_TIMESTAMP", mode="before") @classmethod @@ -113,16 +112,18 @@ def validate_syslog_facility(cls, v): @field_validator("MESSAGE", mode="before") @classmethod def validate_message(cls, v): - """Convert MESSAGE field to string or list[str] based on input type""" - if v is None: - return "" - - # If it's a list but not all items are strings, convert each item to string - if isinstance(v, list): - return [str(item) for item in v] - - # Return string representation for any other type - return str(v) + """Convert MESSAGE to text, including journald's JSON byte-array form.""" + + def to_text(item) -> str: + if item is None: + return "" + if isinstance(item, list): + return bytes(item).decode("utf-8", errors="replace") + return str(item) + + if isinstance(v, list) and not all(isinstance(b, int) for b in v): + return " ".join(to_text(item) for item in v) + return to_text(v) class JournalData(DataModel): diff --git a/nodescraper/plugins/inband/kernel/kernel_collector.py b/nodescraper/plugins/inband/kernel/kernel_collector.py index 6b188940..d93fc215 100644 --- a/nodescraper/plugins/inband/kernel/kernel_collector.py +++ b/nodescraper/plugins/inband/kernel/kernel_collector.py @@ -36,6 +36,7 @@ class KernelCollector(InBandDataCollector[KernelDataModel, None]): """Read kernel version""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = KernelDataModel CMD_WINDOWS = "wmic os get Version /Value" CMD = "sh -c 'uname -a'" @@ -88,6 +89,9 @@ def collect_data( "=" )[1] else: + # Non-Windows (Linux and ESXi). ESXi `uname -a` yields the release in the + # same field the Linux parser reads (verified: "9.1.0"); numa_balancing has + # no ESXi equivalent and its command fails gracefully, leaving None. res = self._run_sut_cmd(self.CMD) if res.exit_code == 0: kernel_info = res.stdout diff --git a/nodescraper/plugins/inband/memory/analyzer_args.py b/nodescraper/plugins/inband/memory/analyzer_args.py index 88dcd5d2..9a4000a2 100644 --- a/nodescraper/plugins/inband/memory/analyzer_args.py +++ b/nodescraper/plugins/inband/memory/analyzer_args.py @@ -23,6 +23,8 @@ # SOFTWARE. # ############################################################################### +from typing import Optional + from pydantic import Field from nodescraper.models.analyzerargs import AnalyzerArgs @@ -31,6 +33,12 @@ class MemoryAnalyzerArgs(AnalyzerArgs): + minimum_free_memory_percent: Optional[float] = Field( + default=None, + ge=0, + le=100, + description="Minimum available system memory percentage required.", + ) ratio: float = Field( default=0.66, description="Required free-memory ratio (0-1). Analysis fails if free/total < ratio.", diff --git a/nodescraper/plugins/inband/memory/memory_analyzer.py b/nodescraper/plugins/inband/memory/memory_analyzer.py index f0de6013..98b124f6 100644 --- a/nodescraper/plugins/inband/memory/memory_analyzer.py +++ b/nodescraper/plugins/inband/memory/memory_analyzer.py @@ -61,6 +61,41 @@ def _bytes_to_gb(n: float) -> float: total_memory = convert_to_bytes(data.mem_total) used_memory = total_memory - available_memory + available_percent = None + if args.minimum_free_memory_percent is not None: + if total_memory <= 0: + self.result.status = ExecutionStatus.WARNING + self.result.message = "Total memory is unavailable" + self._log_event( + category=EventCategory.OS, + description="Cannot validate minimum free memory percentage", + priority=EventPriority.WARNING, + data={"total_memory": total_memory, "available_memory": available_memory}, + console_log=True, + ) + return self.result + + available_percent = available_memory / total_memory * 100 + if available_percent < args.minimum_free_memory_percent: + self.result.status = ExecutionStatus.ERROR + self.result.message = "Minimum free memory percentage not met" + self._log_event( + category=EventCategory.OS, + description=( + f"Available memory is {available_percent:.2f}% " + f"(minimum {args.minimum_free_memory_percent:.2f}%)" + ), + priority=EventPriority.CRITICAL, + data={ + "available_memory": available_memory, + "total_memory": total_memory, + "available_percent": available_percent, + "minimum_free_memory_percent": args.minimum_free_memory_percent, + }, + console_log=True, + ) + return self.result + threshold_bytes = convert_to_bytes(args.memory_threshold) if total_memory > threshold_bytes: @@ -81,6 +116,11 @@ def _bytes_to_gb(n: float) -> float: f"Memory usage is within limit: Used {used_gb:.2f} GB " f"(allowed {allowed_gb:.2f} GB; base={base_source} {base_gb:.2f} GB × ratio={args.ratio:.2f})" ) + if available_percent is not None: + self.result.message += ( + f"; available memory is {available_percent:.2f}% " + f"(minimum {args.minimum_free_memory_percent:.2f}%)" + ) self.result.status = ExecutionStatus.OK else: self.result.message = "Memory usage exceeded max allowed!" diff --git a/nodescraper/plugins/inband/memory/memory_collector.py b/nodescraper/plugins/inband/memory/memory_collector.py index 35c279d9..e228562d 100644 --- a/nodescraper/plugins/inband/memory/memory_collector.py +++ b/nodescraper/plugins/inband/memory/memory_collector.py @@ -173,7 +173,11 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[MemoryDataModel] self._log_event( category=EventCategory.OS, description="Free and total memory read", - data=mem_data.model_dump(), + data={ + "mem_total": mem_total, + "mem_available": mem_available, + "mem_free": mem_free, + }, priority=EventPriority.INFO, ) if mem_available: diff --git a/nodescraper/plugins/inband/memory/memorydata.py b/nodescraper/plugins/inband/memory/memorydata.py index 9e9209bb..38f33605 100644 --- a/nodescraper/plugins/inband/memory/memorydata.py +++ b/nodescraper/plugins/inband/memory/memorydata.py @@ -89,3 +89,10 @@ class MemoryDataModel(DataModel): mem_available: Optional[str] = None lsmem_data: Optional[LsmemData] = None numa_topology: Optional[NumaTopology] = None + + def __str__(self) -> str: + return ( + f"MemoryDataModel(mem_free={self.mem_free}, mem_total={self.mem_total}, " + f"mem_available={self.mem_available}, lsmem_data={self.lsmem_data}, " + f"numa_topology={self.numa_topology})" + ) diff --git a/nodescraper/plugins/inband/network/analyzer_args.py b/nodescraper/plugins/inband/network/analyzer_args.py new file mode 100644 index 00000000..6651f2a5 --- /dev/null +++ b/nodescraper/plugins/inband/network/analyzer_args.py @@ -0,0 +1,39 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class NetworkAnalyzerArgs(AnalyzerArgs): + """Arguments for network and ethtool analysis.""" + + expected_nic_firmware: Optional[str] = Field( + default=None, + description="Exact firmware version expected from each collected NIC.", + ) diff --git a/nodescraper/plugins/inband/network/network_analyzer.py b/nodescraper/plugins/inband/network/network_analyzer.py index 90819e09..496d28f6 100644 --- a/nodescraper/plugins/inband/network/network_analyzer.py +++ b/nodescraper/plugins/inband/network/network_analyzer.py @@ -24,20 +24,25 @@ # ############################################################################### import re +from typing import Optional from nodescraper.base.regexanalyzer import RegexAnalyzer from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.models import TaskResult +from nodescraper.utils import _validate_firmware_policy +from .analyzer_args import NetworkAnalyzerArgs from .networkdata import NetworkDataModel -class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, None]): +class NetworkAnalyzer(RegexAnalyzer[NetworkDataModel, NetworkAnalyzerArgs]): """Check network statistics for errors.""" DATA_MODEL = NetworkDataModel - def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: + def analyze_data( + self, data: NetworkDataModel, args: Optional[NetworkAnalyzerArgs] = None + ) -> TaskResult: """Analyze ethtool -S statistics via RDMA-scoped vendor models. Args: @@ -47,6 +52,40 @@ def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: Returns: TaskResult with OK, WARNING (no devices, or only warning-tier counters), or ERROR. """ + args = args or NetworkAnalyzerArgs() + firmware_records = [] + for interface, info in data.ethtool_info.items(): + vendor = _vendor_for_driver(info.driver) + if vendor is None: + continue + firmware_records.append( + { + "identity": interface, + "version": info.firmware_version, + "source": "ethtool", + "vendor": vendor, + "driver": info.driver, + "interface": interface, + "pci_bdf": info.bus_info, + } + ) + firmware_policy_issues = _validate_firmware_policy( + firmware_records, + ( + {"expected_nic_firmware": args.expected_nic_firmware} + if args.expected_nic_firmware + else None + ), + ) + for issue in firmware_policy_issues: + self._log_event( + category=EventCategory.NETWORK, + description="Network adapter firmware policy mismatch", + data=issue, + priority=EventPriority.WARNING, + console_log=True, + ) + if not data.ethtool_info and not data.ethtool_statistics: self.result.message = "No network devices found" self.result.status = ExecutionStatus.WARNING @@ -165,8 +204,25 @@ def analyze_data(self, data: NetworkDataModel, args=None) -> TaskResult: elif vendor_warning: self.result.message = "Network warning counters non-zero in statistics" self.result.status = ExecutionStatus.WARNING + elif firmware_policy_issues: + self.result.message = "Network adapter firmware policy mismatch" + self.result.status = ExecutionStatus.WARNING else: self.result.message = "No network errors detected in statistics" self.result.status = ExecutionStatus.OK return self.result + + +def _vendor_for_driver(driver: Optional[str]) -> Optional[str]: + """Return a stable vendor label for common ethtool driver families.""" + if not driver: + return None + normalized = driver.lower() + if normalized.startswith(("bnxt", "bnx2")): + return "Broadcom" + if normalized.startswith(("mlx", "ib_")): + return "Mellanox" + if normalized.startswith(("ionic",)): + return "Pensando" + return None diff --git a/nodescraper/plugins/inband/network/network_collector.py b/nodescraper/plugins/inband/network/network_collector.py index 9f109f89..0e030e2f 100644 --- a/nodescraper/plugins/inband/network/network_collector.py +++ b/nodescraper/plugins/inband/network/network_collector.py @@ -450,6 +450,24 @@ def _parse_ethtool(self, interface: str, output: str) -> EthtoolInfo: return ethtool_info + def _parse_ethtool_driver_info(self, info: EthtoolInfo, output: str) -> None: + """Merge ``ethtool -i`` identity and firmware fields into an interface model.""" + for line in output.splitlines(): + if ":" not in line: + continue + key, value = (part.strip() for part in line.split(":", 1)) + if not key or not value: + continue + info.settings[key] = value + normalized_key = key.lower().replace("-", "_").replace(" ", "_") + if normalized_key == "driver": + info.driver = value + elif normalized_key in {"bus_info", "bus"}: + info.bus_info = value + elif normalized_key in {"firmware_version", "firmware"}: + firmware_match = re.search(r"\d+(?:\.\d+){2,}", value) + info.firmware_version = firmware_match.group(0) if firmware_match else value + def _parse_ethtool_statistics(self, output: str, interface: str) -> Dict[str, str]: """Parse 'ethtool -S ' output into a key-value dictionary. @@ -512,6 +530,7 @@ def _collect_ethtool_info( self._log_event( category=EventCategory.NETWORK, description=f"Collected ethtool info for interface: {iface.name}", + data={"interface": iface.name, "speed": ethtool_info.speed}, priority=EventPriority.INFO, ) else: @@ -522,6 +541,41 @@ def _collect_ethtool_info( priority=EventPriority.WARNING, ) + if iface.name == "lo" or any(flag.upper() == "LOOPBACK" for flag in iface.flags): + continue + + # ``ethtool `` does not include firmware identity on all + # drivers, so always retain the separate ``ethtool -i`` response. + res_driver = self._run_sut_cmd( + self.CMD_ETHTOOL_I_TEMPLATE.format(interface=iface.name), sudo=True + ) + if res_driver.exit_code == 0: + ethtool_info = ethtool_data.get( + iface.name, EthtoolInfo(interface=iface.name, raw_output="") + ) + self._parse_ethtool_driver_info(ethtool_info, res_driver.stdout) + ethtool_data[iface.name] = ethtool_info + self._log_event( + category=EventCategory.NETWORK, + description=( + f"Collected ethtool -i driver info for interface: " f"{iface.name}" + ), + priority=EventPriority.INFO, + ) + else: + self._log_event( + category=EventCategory.NETWORK, + description=( + f"Error collecting ethtool -i driver info for interface: " f"{iface.name}" + ), + data={ + "command": res_driver.command, + "exit_code": res_driver.exit_code, + "stderr": res_driver.stderr, + }, + priority=EventPriority.WARNING, + ) + return ethtool_data, skipped def _collect_ethtool_statistic(self, netdev: str, driver: str) -> Optional[EthtoolStatistics]: @@ -788,6 +842,12 @@ def collect_data( self._log_event( category=EventCategory.NETWORK, description=f"Collected {len(interfaces)} network interfaces", + data={ + "interfaces": [ + {"name": iface.name, "state": iface.state, "mtu": iface.mtu} + for iface in interfaces + ] + }, priority=EventPriority.INFO, ) else: diff --git a/nodescraper/plugins/inband/network/network_plugin.py b/nodescraper/plugins/inband/network/network_plugin.py index 69e7b671..5d6d9822 100644 --- a/nodescraper/plugins/inband/network/network_plugin.py +++ b/nodescraper/plugins/inband/network/network_plugin.py @@ -25,13 +25,14 @@ ############################################################################### from nodescraper.base import InBandDataPlugin +from .analyzer_args import NetworkAnalyzerArgs from .collector_args import NetworkCollectorArgs from .network_analyzer import NetworkAnalyzer from .network_collector import NetworkCollector from .networkdata import NetworkDataModel -class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, None]): +class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, NetworkAnalyzerArgs]): """Plugin for collection of network configuration data""" DATA_MODEL = NetworkDataModel @@ -39,5 +40,6 @@ class NetworkPlugin(InBandDataPlugin[NetworkDataModel, NetworkCollectorArgs, Non COLLECTOR = NetworkCollector COLLECTOR_ARGS = NetworkCollectorArgs + ANALYZER_ARGS = NetworkAnalyzerArgs ANALYZER = NetworkAnalyzer diff --git a/nodescraper/plugins/inband/network/networkdata.py b/nodescraper/plugins/inband/network/networkdata.py index 86e8cb6f..bcd7fc95 100644 --- a/nodescraper/plugins/inband/network/networkdata.py +++ b/nodescraper/plugins/inband/network/networkdata.py @@ -97,6 +97,9 @@ class EthtoolInfo(BaseModel): interface: str # Interface name this info belongs to raw_output: str # Raw ethtool command output + driver: Optional[str] = None # Kernel driver from ethtool -i + bus_info: Optional[str] = None # PCI bus identity from ethtool -i + firmware_version: Optional[str] = None # Firmware version from ethtool -i settings: Dict[str, str] = Field(default_factory=dict) # Parsed key-value settings supported_link_modes: List[str] = Field(default_factory=list) # Supported link modes advertised_link_modes: List[str] = Field(default_factory=list) # Advertised link modes diff --git a/nodescraper/plugins/inband/nic/analyzer_args.py b/nodescraper/plugins/inband/nic/analyzer_args.py index 65214b76..cec45ac2 100644 --- a/nodescraper/plugins/inband/nic/analyzer_args.py +++ b/nodescraper/plugins/inband/nic/analyzer_args.py @@ -33,6 +33,10 @@ class NicAnalyzerArgs(AnalyzerArgs): """Analyzer args for niccli/nicctl data, with expected_values keyed by canonical command key.""" + expected_nic_firmware: Optional[str] = Field( + default=None, + description="Exact firmware version expected from each collected NIC.", + ) expected_values: Optional[Dict[str, Dict[str, Any]]] = Field( default=None, description="Per-command expected checks keyed by canonical key (see command_to_canonical_key).", diff --git a/nodescraper/plugins/inband/nic/nic_analyzer.py b/nodescraper/plugins/inband/nic/nic_analyzer.py index 30543867..5cbcc869 100644 --- a/nodescraper/plugins/inband/nic/nic_analyzer.py +++ b/nodescraper/plugins/inband/nic/nic_analyzer.py @@ -1,322 +1,391 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -import re -from typing import Any, Dict, List, Optional - -from nodescraper.base.regexanalyzer import ErrorRegex -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.interfaces import DataAnalyzer -from nodescraper.models import TaskResult - -from .analyzer_args import NicAnalyzerArgs -from .nic_data import NicDataModel - -# Default regexes for nicctl show card logs (boot-fault, persistent, non-persistent) -DEFAULT_NICCTL_LOG_ERROR_REGEX: List[ErrorRegex] = [ - ErrorRegex( - regex=re.compile(r"\berror\b", re.IGNORECASE), - message="nicctl card log: error", - event_category=EventCategory.NETWORK, - event_priority=EventPriority.WARNING, - ), - ErrorRegex( - regex=re.compile(r"\bfail(?:ed|ure)?\b", re.IGNORECASE), - message="nicctl card log: fail/failed/failure", - event_category=EventCategory.NETWORK, - event_priority=EventPriority.WARNING, - ), - ErrorRegex( - regex=re.compile(r"\bfault\b", re.IGNORECASE), - message="nicctl card log: fault", - event_category=EventCategory.NETWORK, - event_priority=EventPriority.WARNING, - ), - ErrorRegex( - regex=re.compile(r"\bcritical\b", re.IGNORECASE), - message="nicctl card log: critical", - event_category=EventCategory.NETWORK, - event_priority=EventPriority.WARNING, - ), -] - - -def _nicctl_log_error_regex_list( - args: NicAnalyzerArgs, -) -> List[ErrorRegex]: - """Return list of ErrorRegex for nicctl card logs (from args or default).""" - if not args.nicctl_log_error_regex: - return list(DEFAULT_NICCTL_LOG_ERROR_REGEX) - out: List[ErrorRegex] = [] - for item in args.nicctl_log_error_regex: - if isinstance(item, ErrorRegex): - out.append(item) - elif isinstance(item, dict): - d = dict(item) - d["regex"] = re.compile(d["regex"]) if isinstance(d.get("regex"), str) else d["regex"] - if "event_category" in d and isinstance(d["event_category"], str): - d["event_category"] = EventCategory(d["event_category"]) - if "event_priority" in d: - p = d["event_priority"] - if isinstance(p, str): - d["event_priority"] = getattr(EventPriority, p.upper(), EventPriority.WARNING) - elif isinstance(p, int): - d["event_priority"] = EventPriority(p) - out.append(ErrorRegex(**d)) - return out - - -def _normalize_prio_map(d: Optional[Dict[Any, Any]]) -> Optional[Dict[int, int]]: - """Convert expected_qos_prio_map (config may have str keys) to Dict[int, int].""" - if d is None: - return None - return {int(k): int(v) for k, v in d.items()} - - -def _normalize_tsa_map(d: Optional[Dict[Any, Any]]) -> Optional[Dict[int, str]]: - """Convert expected_qos_tsa_map (config may have str keys) to Dict[int, str].""" - if d is None: - return None - return {int(k): str(v) for k, v in d.items()} - - -class NicAnalyzer(DataAnalyzer[NicDataModel, NicAnalyzerArgs]): - """Analyze niccli/nicctl data; checks Broadcom support_rdma, performance_profile (RoCE), pcie_relaxed_ordering (enabled), and getqos (expected QoS across adapters).""" - - DATA_MODEL = NicDataModel - - def analyze_data( - self, data: NicDataModel, args: Optional[NicAnalyzerArgs] = None - ) -> TaskResult: - """Run checks on the collected data (Broadcom support_rdma, performance_profile, pcie_relaxed_ordering, getqos per device).""" - if args is None: - args = NicAnalyzerArgs() - - has_broadcom = bool(data.broadcom_nic_support_rdma) - has_nicctl_logs = bool( - data.nicctl_card_logs and any((c or "").strip() for c in data.nicctl_card_logs.values()) - ) - if not has_broadcom and not has_nicctl_logs: - self.result.message = "No Broadcom support_rdma or nicctl card log data to check" - self.result.status = ExecutionStatus.OK - return self.result - - disabled_values = set(args.support_rdma_disabled_values) - any_disabled = False - for device_num, value in sorted(data.broadcom_nic_support_rdma.items()): - value_lower = (value or "").strip().lower() - if value_lower in disabled_values: - any_disabled = True - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: support_rdma is disabled or off", - data={"device_num": device_num, "support_rdma_output": value}, - priority=EventPriority.WARNING, - console_log=True, - ) - else: - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: support_rdma = {value!r}", - data={"device_num": device_num, "support_rdma_output": value}, - priority=EventPriority.INFO, - ) - - if any_disabled: - self.result.message = "One or more Broadcom devices have support_rdma disabled" - self.result.status = ExecutionStatus.WARNING - - # performance_profile expected value check (default RoCE) - expected_profile = args.performance_profile_expected.strip() - expected_profile_lower = expected_profile.lower() - any_non_roce = False - if data.broadcom_nic_performance_profile: - for device_num, value in sorted(data.broadcom_nic_performance_profile.items()): - value_normalized = (value or "").strip().lower() - if expected_profile_lower not in value_normalized: - any_non_roce = True - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: performance_profile is {value!r} (expected {expected_profile})", - data={"device_num": device_num, "performance_profile_output": value}, - priority=EventPriority.WARNING, - console_log=True, - ) - else: - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: performance_profile = {expected_profile}", - data={"device_num": device_num, "performance_profile_output": value}, - priority=EventPriority.INFO, - ) - - # pcie_relaxed_ordering check (default: output should indicate "enabled") - expected_ro = args.pcie_relaxed_ordering_expected.strip().lower() - any_relaxed_ordering_bad = False - if data.broadcom_nic_pcie_relaxed_ordering and expected_ro: - for device_num, value in sorted(data.broadcom_nic_pcie_relaxed_ordering.items()): - value_lower = (value or "").strip().lower() - if expected_ro not in value_lower: - any_relaxed_ordering_bad = True - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: pcie_relaxed_ordering does not show {args.pcie_relaxed_ordering_expected!r} (got {value!r})", - data={"device_num": device_num, "pcie_relaxed_ordering_output": value}, - priority=EventPriority.WARNING, - console_log=True, - ) - else: - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: pcie_relaxed_ordering = {args.pcie_relaxed_ordering_expected}", - data={"device_num": device_num, "pcie_relaxed_ordering_output": value}, - priority=EventPriority.INFO, - ) - - # getqos: expected QoS (priorities, PFC, ETS) across all adapters - any_qos_mismatch = False - expected_prio = _normalize_prio_map(args.expected_qos_prio_map) - expected_tsa = _normalize_tsa_map(args.expected_qos_tsa_map) - if ( - expected_prio is not None - or args.expected_qos_pfc_enabled is not None - or expected_tsa is not None - or args.expected_qos_tc_bandwidth is not None - ): - for device_num, qos in sorted(data.broadcom_nic_qos.items()): - mismatches = [] - if expected_prio is not None and qos.prio_map != expected_prio: - mismatches.append(f"prio_map {qos.prio_map!r} != expected {expected_prio!r}") - if ( - args.expected_qos_pfc_enabled is not None - and qos.pfc_enabled != args.expected_qos_pfc_enabled - ): - mismatches.append( - f"pfc_enabled {qos.pfc_enabled!r} != expected {args.expected_qos_pfc_enabled!r}" - ) - if expected_tsa is not None and qos.tsa_map != expected_tsa: - mismatches.append(f"tsa_map {qos.tsa_map!r} != expected {expected_tsa!r}") - if ( - args.expected_qos_tc_bandwidth is not None - and qos.tc_bandwidth != args.expected_qos_tc_bandwidth - ): - mismatches.append( - f"tc_bandwidth {qos.tc_bandwidth!r} != expected {args.expected_qos_tc_bandwidth!r}" - ) - if mismatches: - any_qos_mismatch = True - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: getqos does not match expected QoS: {'; '.join(mismatches)}", - data={ - "device_num": device_num, - "qos": qos.model_dump(), - "mismatches": mismatches, - }, - priority=EventPriority.WARNING, - console_log=True, - ) - else: - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: getqos matches expected (priorities, PFC, ETS)", - data={"device_num": device_num}, - priority=EventPriority.INFO, - ) - elif args.require_qos_consistent_across_adapters and len(data.broadcom_nic_qos) >= 2: - qos_list = list(data.broadcom_nic_qos.values()) - first = qos_list[0] - for device_num, qos in sorted(data.broadcom_nic_qos.items()): - if ( - qos.prio_map != first.prio_map - or qos.pfc_enabled != first.pfc_enabled - or qos.tsa_map != first.tsa_map - ): - any_qos_mismatch = True - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: getqos differs from other adapters (priorities, PFC, or ETS not consistent)", - data={"device_num": device_num, "qos": qos.model_dump()}, - priority=EventPriority.WARNING, - console_log=True, - ) - else: - self._log_event( - category=EventCategory.NETWORK, - description=f"Broadcom device {device_num}: getqos consistent with other adapters", - data={"device_num": device_num}, - priority=EventPriority.INFO, - ) - - # nicctl card logs (boot-fault, persistent, non-persistent): run error regexes and log matches to user. - any_nicctl_log_errors = False - if data.nicctl_card_logs: - regex_list = _nicctl_log_error_regex_list(args) - for log_type, content in data.nicctl_card_logs.items(): - if not (content or "").strip(): - continue - for err_regex in regex_list: - for match in err_regex.regex.finditer(content): - matched_text = match.group(0).strip() or match.group(0) - if len(matched_text) > 500: - matched_text = matched_text[:497] + "..." - any_nicctl_log_errors = True - self._log_event( - category=err_regex.event_category, - description=f"nicctl card log ({log_type}): {err_regex.message} — {matched_text!r}", - data={ - "log_type": log_type, - "message": err_regex.message, - "match_content": matched_text, - }, - priority=err_regex.event_priority, - console_log=True, - ) - - if ( - any_disabled - or any_non_roce - or any_relaxed_ordering_bad - or any_qos_mismatch - or any_nicctl_log_errors - ): - self.result.status = ExecutionStatus.WARNING - parts = [] - if any_disabled: - parts.append("support_rdma") - if any_non_roce: - parts.append("performance_profile") - if any_relaxed_ordering_bad: - parts.append("pcie_relaxed_ordering") - if any_qos_mismatch: - parts.append("getqos") - if any_nicctl_log_errors: - parts.append("nicctl_card_logs") - self.result.message = f"Broadcom/nic check(s) failed: {' and/or '.join(parts)}" - else: - self.result.status = ExecutionStatus.OK - self.result.message = "Broadcom support_rdma, performance_profile, pcie_relaxed_ordering, getqos, and nicctl card logs checks OK" - return self.result +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import re +from typing import Any, Dict, List, Optional + +from nodescraper.base.regexanalyzer import ErrorRegex +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult +from nodescraper.utils import _validate_firmware_policy + +from .analyzer_args import NicAnalyzerArgs +from .nic_data import NicDataModel + +# Default regexes for nicctl show card logs (boot-fault, persistent, non-persistent) +DEFAULT_NICCTL_LOG_ERROR_REGEX: List[ErrorRegex] = [ + ErrorRegex( + regex=re.compile(r"\berror\b", re.IGNORECASE), + message="nicctl card log: error", + event_category=EventCategory.NETWORK, + event_priority=EventPriority.WARNING, + ), + ErrorRegex( + regex=re.compile(r"\bfail(?:ed|ure)?\b", re.IGNORECASE), + message="nicctl card log: fail/failed/failure", + event_category=EventCategory.NETWORK, + event_priority=EventPriority.WARNING, + ), + ErrorRegex( + regex=re.compile(r"\bfault\b", re.IGNORECASE), + message="nicctl card log: fault", + event_category=EventCategory.NETWORK, + event_priority=EventPriority.WARNING, + ), + ErrorRegex( + regex=re.compile(r"\bcritical\b", re.IGNORECASE), + message="nicctl card log: critical", + event_category=EventCategory.NETWORK, + event_priority=EventPriority.WARNING, + ), +] + + +def _nicctl_log_error_regex_list( + args: NicAnalyzerArgs, +) -> List[ErrorRegex]: + """Return list of ErrorRegex for nicctl card logs (from args or default).""" + if not args.nicctl_log_error_regex: + return list(DEFAULT_NICCTL_LOG_ERROR_REGEX) + out: List[ErrorRegex] = [] + for item in args.nicctl_log_error_regex: + if isinstance(item, ErrorRegex): + out.append(item) + elif isinstance(item, dict): + d = dict(item) + d["regex"] = re.compile(d["regex"]) if isinstance(d.get("regex"), str) else d["regex"] + if "event_category" in d and isinstance(d["event_category"], str): + d["event_category"] = EventCategory(d["event_category"]) + if "event_priority" in d: + p = d["event_priority"] + if isinstance(p, str): + d["event_priority"] = getattr(EventPriority, p.upper(), EventPriority.WARNING) + elif isinstance(p, int): + d["event_priority"] = EventPriority(p) + out.append(ErrorRegex(**d)) + return out + + +def _normalize_prio_map(d: Optional[Dict[Any, Any]]) -> Optional[Dict[int, int]]: + """Convert expected_qos_prio_map (config may have str keys) to Dict[int, int].""" + if d is None: + return None + return {int(k): int(v) for k, v in d.items()} + + +def _normalize_tsa_map(d: Optional[Dict[Any, Any]]) -> Optional[Dict[int, str]]: + """Convert expected_qos_tsa_map (config may have str keys) to Dict[int, str].""" + if d is None: + return None + return {int(k): str(v) for k, v in d.items()} + + +def _firmware_records(data: NicDataModel) -> list[Dict[str, Any]]: + """Build normalized records for Broadcom and Pensando NIC output.""" + records: list[Dict[str, Any]] = [] + devices = {device.device_num: device for device in data.broadcom_nic_devices} + for device_num, version in data.broadcom_nic_firmware.items(): + device = devices.get(device_num) + records.append( + { + "identity": str(device_num), + "version": version, + "source": data.broadcom_cli_type or "broadcom", + "vendor": "Broadcom", + "model": device.model if device else None, + "interface": device.interface_name if device else None, + "pci_bdf": device.pci_address if device else None, + } + ) + for device in data.broadcom_nic_devices: + if device.device_num not in data.broadcom_nic_firmware: + records.append( + { + "identity": str(device.device_num), + "version": None, + "source": data.broadcom_cli_type or "broadcom", + "vendor": "Broadcom", + "model": device.model, + "interface": device.interface_name, + "pci_bdf": device.pci_address, + } + ) + for firmware in data.pensando_nic_version_firmware: + records.append( + { + "identity": firmware.nic_id, + "version": firmware.firmware_a, + "source": "nicctl", + "vendor": "Pensando", + "pci_bdf": firmware.pcie_bdf, + } + ) + return records + + +class NicAnalyzer(DataAnalyzer[NicDataModel, NicAnalyzerArgs]): + """Analyze niccli/nicctl data; checks Broadcom support_rdma, performance_profile (RoCE), pcie_relaxed_ordering (enabled), and getqos (expected QoS across adapters).""" + + DATA_MODEL = NicDataModel + + def analyze_data( + self, data: NicDataModel, args: Optional[NicAnalyzerArgs] = None + ) -> TaskResult: + """Run checks on the collected data (Broadcom support_rdma, performance_profile, pcie_relaxed_ordering, getqos per device).""" + if args is None: + args = NicAnalyzerArgs() + + firmware_policy_issues = _validate_firmware_policy( + _firmware_records(data), + ( + {"expected_nic_firmware": args.expected_nic_firmware} + if args.expected_nic_firmware + else None + ), + ) + for issue in firmware_policy_issues: + self._log_event( + category=EventCategory.NETWORK, + description="Network adapter firmware policy mismatch", + data=issue, + priority=EventPriority.WARNING, + console_log=True, + ) + has_firmware_data = bool( + data.broadcom_nic_devices + or data.broadcom_nic_firmware + or data.pensando_nic_version_firmware + ) + + has_broadcom = bool(data.broadcom_nic_support_rdma) + has_nicctl_logs = bool( + data.nicctl_card_logs and any((c or "").strip() for c in data.nicctl_card_logs.values()) + ) + if not has_broadcom and not has_nicctl_logs and not has_firmware_data: + self.result.message = "No Broadcom support_rdma or nicctl card log data to check" + self.result.status = ExecutionStatus.OK + return self.result + + disabled_values = set(args.support_rdma_disabled_values) + any_disabled = False + for device_num, value in sorted(data.broadcom_nic_support_rdma.items()): + value_lower = (value or "").strip().lower() + if value_lower in disabled_values: + any_disabled = True + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: support_rdma is disabled or off", + data={"device_num": device_num, "support_rdma_output": value}, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: support_rdma = {value!r}", + data={"device_num": device_num, "support_rdma_output": value}, + priority=EventPriority.INFO, + ) + + if any_disabled: + self.result.message = "One or more Broadcom devices have support_rdma disabled" + self.result.status = ExecutionStatus.WARNING + + # performance_profile expected value check (default RoCE) + expected_profile = args.performance_profile_expected.strip() + expected_profile_lower = expected_profile.lower() + any_non_roce = False + if data.broadcom_nic_performance_profile: + for device_num, value in sorted(data.broadcom_nic_performance_profile.items()): + value_normalized = (value or "").strip().lower() + if expected_profile_lower not in value_normalized: + any_non_roce = True + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: performance_profile is {value!r} (expected {expected_profile})", + data={"device_num": device_num, "performance_profile_output": value}, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: performance_profile = {expected_profile}", + data={"device_num": device_num, "performance_profile_output": value}, + priority=EventPriority.INFO, + ) + + # pcie_relaxed_ordering check (default: output should indicate "enabled") + expected_ro = args.pcie_relaxed_ordering_expected.strip().lower() + any_relaxed_ordering_bad = False + if data.broadcom_nic_pcie_relaxed_ordering and expected_ro: + for device_num, value in sorted(data.broadcom_nic_pcie_relaxed_ordering.items()): + value_lower = (value or "").strip().lower() + if expected_ro not in value_lower: + any_relaxed_ordering_bad = True + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: pcie_relaxed_ordering does not show {args.pcie_relaxed_ordering_expected!r} (got {value!r})", + data={"device_num": device_num, "pcie_relaxed_ordering_output": value}, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: pcie_relaxed_ordering = {args.pcie_relaxed_ordering_expected}", + data={"device_num": device_num, "pcie_relaxed_ordering_output": value}, + priority=EventPriority.INFO, + ) + + # getqos: expected QoS (priorities, PFC, ETS) across all adapters + any_qos_mismatch = False + expected_prio = _normalize_prio_map(args.expected_qos_prio_map) + expected_tsa = _normalize_tsa_map(args.expected_qos_tsa_map) + if ( + expected_prio is not None + or args.expected_qos_pfc_enabled is not None + or expected_tsa is not None + or args.expected_qos_tc_bandwidth is not None + ): + for device_num, qos in sorted(data.broadcom_nic_qos.items()): + mismatches = [] + if expected_prio is not None and qos.prio_map != expected_prio: + mismatches.append(f"prio_map {qos.prio_map!r} != expected {expected_prio!r}") + if ( + args.expected_qos_pfc_enabled is not None + and qos.pfc_enabled != args.expected_qos_pfc_enabled + ): + mismatches.append( + f"pfc_enabled {qos.pfc_enabled!r} != expected {args.expected_qos_pfc_enabled!r}" + ) + if expected_tsa is not None and qos.tsa_map != expected_tsa: + mismatches.append(f"tsa_map {qos.tsa_map!r} != expected {expected_tsa!r}") + if ( + args.expected_qos_tc_bandwidth is not None + and qos.tc_bandwidth != args.expected_qos_tc_bandwidth + ): + mismatches.append( + f"tc_bandwidth {qos.tc_bandwidth!r} != expected {args.expected_qos_tc_bandwidth!r}" + ) + if mismatches: + any_qos_mismatch = True + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: getqos does not match expected QoS: {'; '.join(mismatches)}", + data={ + "device_num": device_num, + "mismatches": mismatches, + }, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: getqos matches expected (priorities, PFC, ETS)", + data={"device_num": device_num}, + priority=EventPriority.INFO, + ) + elif args.require_qos_consistent_across_adapters and len(data.broadcom_nic_qos) >= 2: + qos_list = list(data.broadcom_nic_qos.values()) + first = qos_list[0] + for device_num, qos in sorted(data.broadcom_nic_qos.items()): + if ( + qos.prio_map != first.prio_map + or qos.pfc_enabled != first.pfc_enabled + or qos.tsa_map != first.tsa_map + ): + any_qos_mismatch = True + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: getqos differs from other adapters (priorities, PFC, or ETS not consistent)", + data={"device_num": device_num, "qos": qos.model_dump()}, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + self._log_event( + category=EventCategory.NETWORK, + description=f"Broadcom device {device_num}: getqos consistent with other adapters", + data={"device_num": device_num}, + priority=EventPriority.INFO, + ) + + # nicctl card logs (boot-fault, persistent, non-persistent): run error regexes and log matches to user. + any_nicctl_log_errors = False + if data.nicctl_card_logs: + regex_list = _nicctl_log_error_regex_list(args) + for log_type, content in data.nicctl_card_logs.items(): + if not (content or "").strip(): + continue + for err_regex in regex_list: + for match in err_regex.regex.finditer(content): + matched_text = match.group(0).strip() or match.group(0) + if len(matched_text) > 500: + matched_text = matched_text[:497] + "..." + any_nicctl_log_errors = True + self._log_event( + category=err_regex.event_category, + description=f"nicctl card log ({log_type}): {err_regex.message} — {matched_text!r}", + data={ + "log_type": log_type, + "message": err_regex.message, + "match_content": matched_text, + }, + priority=err_regex.event_priority, + console_log=True, + ) + + any_firmware_mismatch = bool(firmware_policy_issues) + if ( + any_disabled + or any_non_roce + or any_relaxed_ordering_bad + or any_qos_mismatch + or any_nicctl_log_errors + or any_firmware_mismatch + ): + self.result.status = ExecutionStatus.WARNING + parts = [] + if any_disabled: + parts.append("support_rdma") + if any_non_roce: + parts.append("performance_profile") + if any_relaxed_ordering_bad: + parts.append("pcie_relaxed_ordering") + if any_qos_mismatch: + parts.append("getqos") + if any_nicctl_log_errors: + parts.append("nicctl_card_logs") + if any_firmware_mismatch: + parts.append("firmware") + self.result.message = f"Broadcom/nic check(s) failed: {' and/or '.join(parts)}" + else: + self.result.status = ExecutionStatus.OK + self.result.message = "Broadcom support_rdma, performance_profile, pcie_relaxed_ordering, getqos, and nicctl card logs checks OK" + return self.result diff --git a/nodescraper/plugins/inband/nic/nic_collector.py b/nodescraper/plugins/inband/nic/nic_collector.py index 27827bfe..605e249a 100644 --- a/nodescraper/plugins/inband/nic/nic_collector.py +++ b/nodescraper/plugins/inband/nic/nic_collector.py @@ -67,6 +67,21 @@ command_to_canonical_key, ) +_FIRMWARE_VERSION_RE = re.compile(r"\b\d+(?:\.\d+){2,}\b") + + +def _extract_firmware_version(output: str) -> Optional[str]: + """Extract the version from a firmware/package-version command response.""" + for line in output.splitlines(): + lowered = line.lower() + if not any(token in lowered for token in ("firmware", "fwpackage", "pkg_ver", "package")): + continue + matches = _FIRMWARE_VERSION_RE.findall(line) + if matches: + return matches[-1] + return None + + # niccli version threshold: legacy (<=233) vs new (>233) command syntax. NICCLI_VERSION_LEGACY_MAX = 233 # Commands use -dev/-getoption/getqos; for version > this use --dev/--getoption/qos --ets --show @@ -641,6 +656,7 @@ def collect_data( custom_commands = args.commands if args and args.commands else None results: dict[str, NicCommandResult] = {} + broadcom_firmware: Dict[int, str] = {} # Detect which Broadcom CLI is present (bcmcli takes priority over niccli). broadcom_cli = self._detect_broadcom_cli(args, results) @@ -779,6 +795,7 @@ def _bcmcli_cmd(cmd: str) -> str: ) # Populate broadcom_nic_* fields from bcmcli results keyed by device_id + broadcom_firmware = self._collect_broadcom_nic_firmware(results) broadcom_support_rdma: Dict[int, str] = {} broadcom_performance_profile: Dict[int, str] = {} broadcom_pcie_relaxed_ordering: Dict[int, str] = {} @@ -991,6 +1008,7 @@ def _bcmcli_cmd(cmd: str) -> str: broadcom_performance_profile, broadcom_pcie_relaxed_ordering, ) = self._collect_broadcom_nic_structured(results, niccli_version=niccli_version) + broadcom_firmware = self._collect_broadcom_nic_firmware(results) ( pensando_cards, pensando_dcqcn, @@ -1055,6 +1073,7 @@ def _truncate(s: str, max_len: int) -> str: broadcom_nic_support_rdma=broadcom_support_rdma, broadcom_nic_performance_profile=broadcom_performance_profile, broadcom_nic_pcie_relaxed_ordering=broadcom_pcie_relaxed_ordering, + broadcom_nic_firmware=broadcom_firmware, pensando_nic_cards=pensando_cards, pensando_nic_dcqcn=pensando_dcqcn, pensando_nic_environment=pensando_environment, @@ -1067,6 +1086,27 @@ def _truncate(s: str, max_len: int) -> str: pensando_nic_version_firmware=pensando_version_firmware, ) + def _collect_broadcom_nic_firmware( + self, results: Dict[str, NicCommandResult] + ) -> Dict[int, str]: + """Parse Broadcom bcmcli/niccli firmware commands into device records.""" + firmware: Dict[int, str] = {} + for command, result in results.items(): + lowered = command.lower() + if not ( + "fwmanager show fwpackage" in lowered + or "show --pkg_ver" in lowered + or "show -pkg_ver" in lowered + ): + continue + if not result.succeeded: + continue + device_match = re.search(r"(?:-d|--dev|-dev)\s+(\d+)", command) + version = _extract_firmware_version(result.stdout) + if device_match and version: + firmware[int(device_match.group(1))] = version + return firmware + def _resolve_bcmcli_bin_dir(self) -> str: res = self._run_sut_cmd("which bcmcli_show", sudo=False, log_artifact=False) if res.exit_code == 0 and (res.stdout or "").strip(): diff --git a/nodescraper/plugins/inband/nic/nic_data.py b/nodescraper/plugins/inband/nic/nic_data.py index 9720c4c1..e82172d5 100644 --- a/nodescraper/plugins/inband/nic/nic_data.py +++ b/nodescraper/plugins/inband/nic/nic_data.py @@ -381,6 +381,10 @@ class NicDataModel(DataModel): default_factory=dict, description="Per-device output of 'niccli -dev X nvm -getoption pcie_relaxed_ordering' (device_num -> raw stdout).", ) + broadcom_nic_firmware: Dict[int, str] = Field( + default_factory=dict, + description="Per-device firmware version parsed from bcmcli fwpackage or niccli pkg_ver output.", + ) pensando_nic_cards: List[PensandoNicCard] = Field(default_factory=list) pensando_nic_dcqcn: List[PensandoNicDcqcn] = Field(default_factory=list) pensando_nic_environment: List[PensandoNicEnvironment] = Field(default_factory=list) diff --git a/nodescraper/plugins/inband/nvme/analyzer_args.py b/nodescraper/plugins/inband/nvme/analyzer_args.py new file mode 100644 index 00000000..24281034 --- /dev/null +++ b/nodescraper/plugins/inband/nvme/analyzer_args.py @@ -0,0 +1,38 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class NvmeAnalyzerArgs(AnalyzerArgs): + maximum_smart_error_count: Optional[int] = Field( + default=None, + ge=0, + description="Maximum allowed NVMe SMART media error count per device.", + ) diff --git a/nodescraper/plugins/inband/nvme/nvme_analyzer.py b/nodescraper/plugins/inband/nvme/nvme_analyzer.py new file mode 100644 index 00000000..3453f114 --- /dev/null +++ b/nodescraper/plugins/inband/nvme/nvme_analyzer.py @@ -0,0 +1,118 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from typing import Optional + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import NvmeAnalyzerArgs +from .nvmedata import NvmeDataModel + + +class NvmeAnalyzer(DataAnalyzer[NvmeDataModel, NvmeAnalyzerArgs]): + """Check NVMe SMART health data.""" + + DATA_MODEL = NvmeDataModel + + @staticmethod + def _parse_media_errors(smart_log: Optional[str]) -> Optional[int]: + if not smart_log: + return None + match = re.search(r"(?im)^\s*media_errors\s*:\s*(\d+)\s*$", smart_log) + return int(match.group(1)) if match else None + + def analyze_data( + self, data: NvmeDataModel, args: Optional[NvmeAnalyzerArgs] = None + ) -> TaskResult: + """Check each NVMe device's SMART media error count.""" + if args is None or args.maximum_smart_error_count is None: + self.result.status = ExecutionStatus.NOT_RAN + self.result.message = "Maximum NVMe SMART error count not provided" + return self.result + + if not data.devices: + self.result.status = ExecutionStatus.NOT_RAN + self.result.message = "No NVMe data available" + return self.result + + failures = [] + unavailable = [] + for device, device_data in data.devices.items(): + media_errors = self._parse_media_errors(device_data.smart_log) + if media_errors is None: + unavailable.append(device) + continue + + if media_errors > args.maximum_smart_error_count: + failures.append((device, media_errors)) + + if unavailable and not failures and len(unavailable) == len(data.devices): + for device in unavailable: + self._log_event( + category=EventCategory.STORAGE, + description=f"NVMe SMART media error count unavailable for {device}", + priority=EventPriority.ERROR, + data={"device": device}, + console_log=True, + ) + self.result.status = ExecutionStatus.ERROR + self.result.message = "NVMe SMART media error count unavailable" + return self.result + + for device in unavailable: + self._log_event( + category=EventCategory.STORAGE, + description=f"NVMe SMART media error count unavailable for {device}", + priority=EventPriority.WARNING, + data={"device": device}, + console_log=True, + ) + + if failures: + for device, media_errors in failures: + self._log_event( + category=EventCategory.STORAGE, + description=( + f"NVMe SMART media errors exceeded threshold for {device}: " + f"{media_errors} > {args.maximum_smart_error_count}" + ), + priority=EventPriority.ERROR, + data={ + "device": device, + "media_errors": media_errors, + "maximum_smart_error_count": args.maximum_smart_error_count, + }, + console_log=True, + ) + self.result.status = ExecutionStatus.ERROR + self.result.message = "NVMe SMART media error threshold exceeded" + else: + self.result.status = ExecutionStatus.OK + self.result.message = "NVMe SMART media error counts are within limits" + + return self.result diff --git a/nodescraper/plugins/inband/nvme/nvme_plugin.py b/nodescraper/plugins/inband/nvme/nvme_plugin.py index 29557290..35ecb1bc 100644 --- a/nodescraper/plugins/inband/nvme/nvme_plugin.py +++ b/nodescraper/plugins/inband/nvme/nvme_plugin.py @@ -25,13 +25,19 @@ ############################################################################### from nodescraper.base import InBandDataPlugin +from .analyzer_args import NvmeAnalyzerArgs +from .nvme_analyzer import NvmeAnalyzer from .nvme_collector import NvmeCollector from .nvmedata import NvmeDataModel -class NvmePlugin(InBandDataPlugin[NvmeDataModel, None, None]): +class NvmePlugin(InBandDataPlugin[NvmeDataModel, None, NvmeAnalyzerArgs]): """Plugin for collection and analysis of nvme data""" DATA_MODEL = NvmeDataModel COLLECTOR = NvmeCollector + + ANALYZER = NvmeAnalyzer + + ANALYZER_ARGS = NvmeAnalyzerArgs diff --git a/nodescraper/plugins/inband/os/analyzer_args.py b/nodescraper/plugins/inband/os/analyzer_args.py index 58565123..4fd4ed95 100644 --- a/nodescraper/plugins/inband/os/analyzer_args.py +++ b/nodescraper/plugins/inband/os/analyzer_args.py @@ -23,7 +23,7 @@ # SOFTWARE. # ############################################################################### -from typing import Union +from typing import Optional, Union from pydantic import Field, field_validator @@ -32,6 +32,11 @@ class OsAnalyzerArgs(AnalyzerArgs): + maximum_load_per_cpu_core: Optional[float] = Field( + default=None, + ge=0, + description="Maximum allowed 1-minute load average per CPU core.", + ) exp_os: Union[str, list] = Field( default_factory=list, description="Expected OS name/version string(s) to match (e.g. from lsb_release or /etc/os-release).", diff --git a/nodescraper/plugins/inband/os/os_analyzer.py b/nodescraper/plugins/inband/os/os_analyzer.py index 235b4aab..f168d016 100644 --- a/nodescraper/plugins/inband/os/os_analyzer.py +++ b/nodescraper/plugins/inband/os/os_analyzer.py @@ -48,26 +48,81 @@ def analyze_data(self, data: OsDataModel, args: Optional[OsAnalyzerArgs] = None) Returns: TaskResult: Result of the analysis containing status and message. """ - if not args or not args.exp_os: + if args is None: + args = OsAnalyzerArgs() + + if not args.exp_os and args.maximum_load_per_cpu_core is None: self.result.message = "Expected OS name not provided" self.result.status = ExecutionStatus.NOT_RAN return self.result - for os_name in args.exp_os: - if (os_name == data.os_name and args.exact_match) or ( - os_name in data.os_name and not args.exact_match - ): - self.result.message = "OS name matches expected" - self.result.status = ExecutionStatus.OK - return self.result + os_name_matches = True + load_check_warning = False + load_check_error = False + + if args.exp_os: + os_name_matches = False + for os_name in args.exp_os: + if (os_name == data.os_name and args.exact_match) or ( + os_name in data.os_name and not args.exact_match + ): + os_name_matches = True + break + + if not os_name_matches: + self._log_event( + category=EventCategory.OS, + description=f"OS name mismatch! Expected: {args.exp_os}, actual: {data.os_name}", + data={"expected": args.exp_os, "actual": data.os_name}, + priority=EventPriority.CRITICAL, + console_log=True, + ) + + if args.maximum_load_per_cpu_core is not None: + if data.load_average_1m is None or data.cpu_count is None or data.cpu_count <= 0: + load_check_warning = True + self._log_event( + category=EventCategory.OS, + description="Cannot validate maximum load per CPU core", + data={ + "load_average_1m": data.load_average_1m, + "cpu_count": data.cpu_count, + }, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + load_per_cpu_core = data.load_average_1m / data.cpu_count + if load_per_cpu_core > args.maximum_load_per_cpu_core: + load_check_error = True + self._log_event( + category=EventCategory.OS, + description=( + f"Load per CPU core is {load_per_cpu_core:.2f} " + f"(maximum {args.maximum_load_per_cpu_core:.2f})" + ), + data={ + "load_average_1m": data.load_average_1m, + "cpu_count": data.cpu_count, + "load_per_cpu_core": load_per_cpu_core, + "maximum_load_per_cpu_core": args.maximum_load_per_cpu_core, + }, + priority=EventPriority.CRITICAL, + console_log=True, + ) - self.result.message = "OS name mismatch!" - self.result.status = ExecutionStatus.ERROR - self._log_event( - category=EventCategory.OS, - description=f"OS name mismatch! Expected: {args.exp_os}, actual: {data.os_name}", - data={"expected": args.exp_os, "actual": data.os_name}, - priority=EventPriority.CRITICAL, - console_log=True, - ) + if not os_name_matches: + self.result.message = "OS name mismatch!" + self.result.status = ExecutionStatus.ERROR + elif load_check_error: + self.result.message = "Maximum load per CPU core exceeded" + self.result.status = ExecutionStatus.ERROR + elif load_check_warning: + self.result.message = "CPU load data is not available" + self.result.status = ExecutionStatus.WARNING + else: + self.result.message = ( + "OS name matches expected" if args.exp_os else "Load per CPU core is within limit" + ) + self.result.status = ExecutionStatus.OK return self.result diff --git a/nodescraper/plugins/inband/os/os_collector.py b/nodescraper/plugins/inband/os/os_collector.py index 42e435a9..8f5543ac 100644 --- a/nodescraper/plugins/inband/os/os_collector.py +++ b/nodescraper/plugins/inband/os/os_collector.py @@ -36,12 +36,17 @@ class OsCollector(InBandDataCollector[OsDataModel, None]): """Collect OS details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = OsDataModel CMD_VERSION_WINDOWS = "wmic os get Version /value" CMD_VERSION = "cat /etc/*release | grep VERSION_ID" + CMD_VERSION_ESXI = "esxcli system version get" CMD_WINDOWS = "wmic os get Caption /Value" + CMD_ESXI = "vmware -v" PRETTY_STR = "PRETTY_NAME" # noqa: N806 CMD = f"sh -c '( lsb_release -ds || (cat /etc/*release | grep {PRETTY_STR}) || uname -om ) 2>/dev/null | head -n1'" + CMD_LOAD_AVERAGE = "cat /proc/loadavg" + CMD_CPU_COUNT = "grep -c ^processor /proc/cpuinfo" def collect_version(self) -> str: """Collect OS version. @@ -60,6 +65,22 @@ def collect_version(self) -> str: priority=EventPriority.ERROR, ) os_version = "" + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_VERSION_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + if "Version:" in line: + os_version = line.split(":", 1)[1].strip() + break + else: + os_version = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS version not found", + priority=EventPriority.ERROR, + ) + os_version = "" else: res = self._run_sut_cmd(self.CMD_VERSION) if res.exit_code == 0: @@ -86,6 +107,16 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[OsDataModel]]: res = self._run_sut_cmd(self.CMD_WINDOWS) if res.exit_code == 0: os_name = re.search(r"Caption=([\w\s]+)", res.stdout).group(1) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + os_name = res.stdout.strip() + else: + self._log_event( + category=EventCategory.OS, + description="OS name not found", + priority=EventPriority.ERROR, + ) else: res = self._run_sut_cmd(self.CMD) # search for PRETTY_NAME in res @@ -106,9 +137,36 @@ def collect_data(self, args=None) -> tuple[TaskResult, Optional[OsDataModel]]: if os_name: os_version = self.collect_version() + load_average_1m = None + cpu_count = None + if self.system_info.os_family == OSFamily.LINUX: + load_res = self._run_sut_cmd(self.CMD_LOAD_AVERAGE) + if load_res.exit_code == 0 and load_res.stdout.strip(): + try: + load_average_1m = float(load_res.stdout.split()[0]) + except (ValueError, IndexError): + self._log_event( + category=EventCategory.OS, + description="Invalid 1-minute load average", + priority=EventPriority.WARNING, + ) + + cpu_res = self._run_sut_cmd(self.CMD_CPU_COUNT) + if cpu_res.exit_code == 0 and cpu_res.stdout.strip(): + try: + cpu_count = int(cpu_res.stdout.strip()) + except ValueError: + self._log_event( + category=EventCategory.OS, + description="Invalid CPU count", + priority=EventPriority.WARNING, + ) + os_data = OsDataModel( os_name=os_name, os_version=os_version, + load_average_1m=load_average_1m, + cpu_count=cpu_count, ) self._log_event( category="OS_NAME_READ", diff --git a/nodescraper/plugins/inband/os/osdata.py b/nodescraper/plugins/inband/os/osdata.py index fd861349..7bd6e56d 100644 --- a/nodescraper/plugins/inband/os/osdata.py +++ b/nodescraper/plugins/inband/os/osdata.py @@ -23,9 +23,13 @@ # SOFTWARE. # ############################################################################### +from typing import Optional + from nodescraper.models import DataModel class OsDataModel(DataModel): os_name: str os_version: str = "" + load_average_1m: Optional[float] = None + cpu_count: Optional[int] = None diff --git a/nodescraper/plugins/inband/pcie/collector_args.py b/nodescraper/plugins/inband/pcie/collector_args.py new file mode 100644 index 00000000..21260fe2 --- /dev/null +++ b/nodescraper/plugins/inband/pcie/collector_args.py @@ -0,0 +1,24 @@ +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class PcieCollectorArgs(CollectorArgs): + """Collector args for PCIe data. + + On ESXi, GPU/VF BDFs are resolved from ``esxcli hardware pci list`` by matching + the expected PF/VF PCI device IDs (esxcli has no device filter). Provide them + here; when both are unset no ESXi GPU BDFs are resolved. The caller populates + these (e.g. from the system SKU). + """ + + devid_ep: Optional[int] = Field( + default=None, + description="Expected GPU PF PCI device ID (int, e.g. 0x75a3) for ESXi BDF resolution.", + ) + devid_ep_vf: Optional[int] = Field( + default=None, + description="Expected GPU VF PCI device ID (int) for ESXi VF BDF resolution.", + ) diff --git a/nodescraper/plugins/inband/pcie/pcie_collector.py b/nodescraper/plugins/inband/pcie/pcie_collector.py index 624122ec..1085bf2a 100755 --- a/nodescraper/plugins/inband/pcie/pcie_collector.py +++ b/nodescraper/plugins/inband/pcie/pcie_collector.py @@ -41,6 +41,7 @@ from nodescraper.models import TaskResult from nodescraper.utils import get_all_subclasses, get_exception_details +from .collector_args import PcieCollectorArgs from .pcie_data import ( MAX_CAP_ID, MAX_ECAP_ID, @@ -54,7 +55,7 @@ ) -class PcieCollector(InBandDataCollector[PcieDataModel, None]): +class PcieCollector(InBandDataCollector[PcieDataModel, PcieCollectorArgs]): """class for collection of PCIe data only supports Linux OS type. This class collects the PCIE config space using the lspci hex dump and then parses the hex dump to get the @@ -80,7 +81,10 @@ class PcieCollector(InBandDataCollector[PcieDataModel, None]): """ - SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX} + SUPPORTED_OS_FAMILY: Set[OSFamily] = {OSFamily.LINUX, OSFamily.ESXI} + + # A bare BDF line that begins an esxcli/lspci device block, e.g. "0000:05:00.0". + _BDF_LINE = re.compile(r"^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]+", re.IGNORECASE) DATA_MODEL = PcieDataModel @@ -518,17 +522,8 @@ def get_cap_cfg( return cap_structure # type: ignore[return-value] - def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: - """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" - hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) - if hex_data_raw is None: - self._log_event( - category=EventCategory.IO, - description="Failed to get hex data for BDF.", - data={"bdf": bdf}, - priority=EventPriority.ERROR, - ) - return PcieCfgSpace() + def _cfg_space_from_hex(self, hex_data_raw: str, bdf: str) -> PcieCfgSpace: + """Parse a raw lspci hex dump (Linux ``-xxxx`` or ESXi ``-e``) into a PcieCfgSpace.""" hex_data: List[int] = self.parse_hex_dump(hex_data_raw) if len(hex_data) < 64: # Expect at least 256 bytes of data, for the first 256 bytes of the PCIe config space @@ -542,6 +537,19 @@ def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: cap_data, ecap_data = self.discover_capability_structure(hex_data) return self.get_pcie_cfg(hex_data, cap_data, ecap_data) + def get_cfg_by_bdf(self, bdf: str, sudo=True) -> PcieCfgSpace: + """Will fill out a PcieCfgSpace object with the PCIe configuration space for a given BDF""" + hex_data_raw = self.show_lspci_hex(bdf, sudo=sudo) + if hex_data_raw is None: + self._log_event( + category=EventCategory.IO, + description="Failed to get hex data for BDF.", + data={"bdf": bdf}, + priority=EventPriority.ERROR, + ) + return PcieCfgSpace() + return self._cfg_space_from_hex(hex_data_raw, bdf) + def get_pcie_cfg( self, config_data: List[int], @@ -595,8 +603,132 @@ def _log_pcie_artifacts( if data is not None: self.result.artifacts.append(TextFileArtifact(filename=name, contents=data)) + def _get_gpu_vf_bdfs_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Tuple[List[str], List[str]]: + """Return (pf_bdfs, vf_bdfs) for the GPUs on an ESXi host via esxcli. + + ESXi busybox lspci has no device filter, so GPU/VF BDFs are resolved from + ``esxcli hardware pci list`` by matching the expected PF/VF device IDs + (``pf_devid`` / ``vf_devid``, from the collector args). Each device block + starts with a bare BDF line followed by indented fields incl. "Device ID". + """ + pf_bdfs: List[str] = [] + vf_bdfs: List[str] = [] + if pf_devid is None and vf_devid is None: + return pf_bdfs, vf_bdfs + + out = self._run_os_cmd("esxcli hardware pci list", sudo=False) + if not out: + return pf_bdfs, vf_bdfs + + current_bdf: Optional[str] = None + for line in out.splitlines(): + stripped = line.strip() + if self._BDF_LINE.match(stripped) and ":" in stripped and " " not in stripped: + # Bare BDF header line (anchors the block). + current_bdf = stripped + elif current_bdf and stripped.lower().startswith("device id:"): + # Compare by integer value so case ("0x744C") and zero-padding + # ("0x0000744c") both match the expected device ID. + raw = stripped.split(":", 1)[1].strip() + try: + devid = int(raw, 16) + except ValueError: + continue + if pf_devid is not None and devid == pf_devid: + pf_bdfs.append(current_bdf) + elif vf_devid is not None and devid == vf_devid: + vf_bdfs.append(current_bdf) + return pf_bdfs, vf_bdfs + + def _get_all_cfg_space_esxi(self) -> Dict[str, str]: + """Return {bdf: hex_dump_text} for every device from a single ``lspci -e``. + + ESXi has no per-device dump; ``lspci -e`` emits the full extended (4096-byte) + config space for all devices in one blob. Each device section starts with a + header line " " followed by "NN: .." hex lines. + """ + blob = self._run_os_cmd("lspci -e", sudo=False) + if not blob: + return {} + self.result.artifacts.append(TextFileArtifact(filename="lspci_e.txt", contents=blob)) + sections: Dict[str, List[str]] = {} + current_bdf: Optional[str] = None + for line in blob.splitlines(): + header = self._BDF_LINE.match(line) + if header and " " in line: + # Device header line: " ". + current_bdf = line.split(" ", 1)[0] + sections[current_bdf] = [] + elif current_bdf is not None: + sections[current_bdf].append(line) + return {bdf: "\n".join(lines) for bdf, lines in sections.items()} + + def _get_pcie_data_esxi( + self, pf_devid: Optional[int], vf_devid: Optional[int] + ) -> Optional[PcieDataModel]: + """Collect GPU + VF PCIe config space on ESXi. + + ESXi busybox lspci lacks ``-s`` (per-device) and ``-PP`` (bus-path), so dump + all extended config space once via ``lspci -e``, split it by BDF, and select the + GPU/VF BDFs resolved from esxcli (matching ``pf_devid`` / ``vf_devid`` from the + collector args). Upstream-bridge traversal is not available on ESXi and is + intentionally skipped (GPU + VF only). + """ + pf_bdfs, vf_bdfs = self._get_gpu_vf_bdfs_esxi(pf_devid, vf_devid) + if not pf_bdfs and not vf_bdfs: + self._log_event( + category=EventCategory.IO, + description="No GPU/VF BDFs found on ESXi host for this SKU.", + data={"devid_ep": pf_devid, "devid_ep_vf": vf_devid}, + priority=EventPriority.WARNING, + ) + return None + + cfg_by_bdf = self._get_all_cfg_space_esxi() + if not cfg_by_bdf: + self.result.status = ExecutionStatus.ERROR + return None + + self._log_event( + category=EventCategory.IO, + description=( + "Upstream-bridge PCIe collection is not supported on ESXi; " + "collecting GPU + VF only." + ), + priority=EventPriority.INFO, + ) + + try: + pcie_cfg_dict: Dict[str, PcieCfgSpace] = {} + for bdf in pf_bdfs: + if bdf in cfg_by_bdf: + pcie_cfg_dict[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + vf_pcie_cfg_data: Dict[str, PcieCfgSpace] = {} + for bdf in vf_bdfs: + if bdf in cfg_by_bdf: + vf_pcie_cfg_data[bdf] = self._cfg_space_from_hex(cfg_by_bdf[bdf], bdf) + pcie_data = PcieDataModel( + pcie_cfg_space=pcie_cfg_dict, + vf_pcie_cfg_space=vf_pcie_cfg_data, + ) + except ValidationError as e: + self._log_event( + category=EventCategory.OS, + description="Failed to build model for PCIe data", + data=get_exception_details(e), + priority=EventPriority.ERROR, + ) + self.result.status = ExecutionStatus.ERROR + return None + return pcie_data + def _get_pcie_data( - self, upstream_steps_to_collect: Optional[int] = None + self, + upstream_steps_to_collect: Optional[int] = None, + pf_devid: Optional[int] = None, + vf_devid: Optional[int] = None, ) -> Optional[PcieDataModel]: """Will return all PCIe data in a PcieDataModel object. @@ -605,6 +737,9 @@ def _get_pcie_data( Optional[PcieDataModel] The data in a PcieDataModel object or None on failure """ + if self.system_info.os_family == OSFamily.ESXI: + return self._get_pcie_data_esxi(pf_devid, vf_devid) + minimum_system_interaction_level_required_for_sudo = SystemInteractionLevel.INTERACTIVE try: @@ -702,19 +837,24 @@ def discover_capability_structure( return cap, ecap def collect_data( - self, args=None, upstream_steps_to_collect: Optional[int] = None, **kwargs + self, + args: Optional[PcieCollectorArgs] = None, + upstream_steps_to_collect: Optional[int] = None, + **kwargs, ) -> Tuple[TaskResult, Optional[PcieDataModel]]: """Read PCIe data. Args: - args: Optional collector arguments (not used) + args: Optional collector arguments (devid_ep / devid_ep_vf for ESXi GPU BDF resolution) upstream_steps_to_collect: Number of upstream devices to collect **kwargs: Additional keyword arguments Returns: Tuple[TaskResult, Optional[PcieDataModel]]: tuple containing the result of the task and the PCIe data if available """ - pcie_data = self._get_pcie_data(upstream_steps_to_collect) + if args is None: + args = PcieCollectorArgs() + pcie_data = self._get_pcie_data(upstream_steps_to_collect, args.devid_ep, args.devid_ep_vf) if pcie_data: self._log_event( category=EventCategory.IO, diff --git a/nodescraper/plugins/inband/pcie/pcie_plugin.py b/nodescraper/plugins/inband/pcie/pcie_plugin.py index 0e4f3eb0..9d894ade 100644 --- a/nodescraper/plugins/inband/pcie/pcie_plugin.py +++ b/nodescraper/plugins/inband/pcie/pcie_plugin.py @@ -26,18 +26,21 @@ from nodescraper.base import InBandDataPlugin from .analyzer_args import PcieAnalyzerArgs +from .collector_args import PcieCollectorArgs from .pcie_analyzer import PcieAnalyzer from .pcie_collector import PcieCollector from .pcie_data import PcieDataModel -class PciePlugin(InBandDataPlugin[PcieDataModel, None, PcieAnalyzerArgs]): +class PciePlugin(InBandDataPlugin[PcieDataModel, PcieCollectorArgs, PcieAnalyzerArgs]): """Plugin for collection and analysis of PCIe data""" DATA_MODEL = PcieDataModel COLLECTOR = PcieCollector + COLLECTOR_ARGS = PcieCollectorArgs + ANALYZER = PcieAnalyzer ANALYZER_ARGS = PcieAnalyzerArgs diff --git a/nodescraper/plugins/inband/process/analyzer_args.py b/nodescraper/plugins/inband/process/analyzer_args.py index 39375ed9..658aedd2 100644 --- a/nodescraper/plugins/inband/process/analyzer_args.py +++ b/nodescraper/plugins/inband/process/analyzer_args.py @@ -31,13 +31,12 @@ class ProcessAnalyzerArgs(AnalyzerArgs): - max_kfd_processes: int = Field( - default=0, - description="Maximum allowed number of KFD (Kernel Fusion Driver) processes; 0 disables the check.", - ) max_cpu_usage: float = Field( default=20.0, - description="Maximum allowed CPU usage (percent) for process checks.", + description=( + "Maximum allowed aggregate CPU execution percentage; idle and I/O-wait time " + "are excluded." + ), ) @classmethod @@ -50,4 +49,4 @@ def build_from_model(cls, datamodel: ProcessDataModel) -> "ProcessAnalyzerArgs": Returns: ProcessAnalyzerArgs: instance of analyzer args class """ - return cls(max_kfd_processes=datamodel.kfd_process, max_cpu_usage=datamodel.cpu_usage) + return cls(max_cpu_usage=datamodel.cpu_usage) diff --git a/nodescraper/plugins/inband/process/collector_args.py b/nodescraper/plugins/inband/process/collector_args.py index 74099144..909d36a9 100644 --- a/nodescraper/plugins/inband/process/collector_args.py +++ b/nodescraper/plugins/inband/process/collector_args.py @@ -31,7 +31,10 @@ class ProcessCollectorArgs(CollectorArgs): top_n_process: int = Field( default=10, - description=( - "Number of top processes by CPU usage to collect " "(e.g. for top -b -n 1 -o %%CPU)." - ), + description="Number of top processes by CPU usage to collect.", + ) + sample_interval_seconds: float = Field( + default=1.0, + gt=0, + description="Wall-clock interval between procfs CPU samples.", ) diff --git a/nodescraper/plugins/inband/process/process_analyzer.py b/nodescraper/plugins/inband/process/process_analyzer.py index 8700f1c1..67f32646 100644 --- a/nodescraper/plugins/inband/process/process_analyzer.py +++ b/nodescraper/plugins/inband/process/process_analyzer.py @@ -34,7 +34,7 @@ class ProcessAnalyzer(DataAnalyzer[ProcessDataModel, ProcessAnalyzerArgs]): - """Check cpu and kfd processes are within allowed maximum cpu and gpu usage""" + """Check aggregate CPU usage against the configured maximum.""" DATA_MODEL = ProcessDataModel @@ -42,8 +42,7 @@ def analyze_data( self, data: ProcessDataModel, args: Optional[ProcessAnalyzerArgs] = None ) -> TaskResult: """ - Analyze the process data to check if the number of KFD processes and CPU usage - are within the allowed limits. + Analyze whether aggregate CPU usage is within the allowed limit. Args: data (ProcessDataModel): The process data to analyze. @@ -56,19 +55,6 @@ def analyze_data( args = ProcessAnalyzerArgs() has_errors = False - if data.kfd_process is not None and data.kfd_process > args.max_kfd_processes: - has_errors = True - self._log_event( - category=EventCategory.OS, - description=f"Kfd processes {data.kfd_process} exeed max limit {args.max_kfd_processes}", - data={ - "kfd_process": data.kfd_process, - "kfd_process_limit": args.max_kfd_processes, - }, - priority=EventPriority.CRITICAL, - console_log=True, - ) - if data.cpu_usage is not None and data.cpu_usage > args.max_cpu_usage: has_errors = True self._log_event( diff --git a/nodescraper/plugins/inband/process/process_collector.py b/nodescraper/plugins/inband/process/process_collector.py index 72feb8c5..4655468b 100644 --- a/nodescraper/plugins/inband/process/process_collector.py +++ b/nodescraper/plugins/inband/process/process_collector.py @@ -23,7 +23,7 @@ # SOFTWARE. # ############################################################################### -import re +import time from typing import Optional from nodescraper.base import InBandDataCollector @@ -34,15 +34,226 @@ from .processdata import ProcessDataModel +def _parse_aggregate_cpu_from_proc_stat(proc_stat: str) -> Optional[tuple[int, int]]: + """Return total and idle-plus-I/O-wait jiffies from ``/proc/stat``. + + Guest fields are excluded from the total because Linux includes them in + the user and nice fields. + """ + for line in proc_stat.splitlines(): + if not line.startswith("cpu "): + continue + + parts = line.split() + if len(parts) < 6: + return None + try: + values = [int(value) for value in parts[1:]] + except ValueError: + return None + + # guest and guest_nice are already included in user and nice. + return sum(values[:8]), values[3] + values[4] + return None + + +def _global_non_idle_percent(total1: int, idle1: int, total2: int, idle2: int) -> float: + """Return aggregate non-idle CPU percentage between two samples.""" + total_delta = total2 - total1 + if total_delta <= 0: + return 0.0 + + idle_delta = idle2 - idle1 + percentage = 100.0 * (1.0 - idle_delta / total_delta) + return round(max(0.0, min(100.0, percentage)), 6) + + +def _parse_proc_pid_stat(stat_line: str) -> Optional[tuple[int, int]]: + """Return a process PID and its user-plus-system jiffies.""" + stat_line = stat_line.strip() + if not stat_line: + return None + + try: + paren_end = stat_line.rfind(") ") + if paren_end < 0: + return None + pid = int(stat_line[:paren_end].split(maxsplit=1)[0]) + fields = stat_line[paren_end + 2 :].split() + if len(fields) < 13: + return None + return pid, int(fields[11]) + int(fields[12]) + except ValueError: + return None + + +def _parse_proc_stat_dump(dump_stdout: str) -> tuple[dict[int, int], set[int]]: + """Parse bulk ``pid|stat`` output and sampler PIDs.""" + jiffies_by_pid: dict[int, int] = {} + sampler_pids: set[int] = set() + + for line in dump_stdout.splitlines(): + stripped = line.strip() + if stripped.startswith("__SAMPLER__:"): + try: + sampler_pids.add(int(stripped.split(":", 1)[1])) + except ValueError: + pass + continue + if "|" not in line: + continue + + pid_text, stat_line = line.split("|", 1) + try: + prefixed_pid = int(pid_text) + except ValueError: + continue + + parsed = _parse_proc_pid_stat(stat_line) + if parsed is None: + continue + stat_pid, jiffies = parsed + if stat_pid == prefixed_pid: + jiffies_by_pid[prefixed_pid] = jiffies + + return jiffies_by_pid, sampler_pids + + +def _top_process_cpu_shares( + sample1: dict[int, int], + sample2: dict[int, int], + total_delta: int, + top_n: int, + exclude_pids: set[int], +) -> list[tuple[int, float]]: + """Rank processes by their share of aggregate CPU jiffies.""" + if top_n <= 0: + return [] + + rows: list[tuple[int, float, int]] = [] + for pid in sample2: + if pid in exclude_pids: + continue + delta = max(0, sample2.get(pid, 0) - sample1.get(pid, 0)) + percentage = 100.0 * delta / total_delta if total_delta > 0 else 0.0 + rows.append((pid, percentage, delta)) + + rows.sort(key=lambda row: (row[2], row[1]), reverse=True) + return [(pid, percentage) for pid, percentage, _ in rows[:top_n]] + + +def _parse_comm_dump(dump_stdout: str) -> dict[int, str]: + """Parse ``pid:comm`` lines into process names keyed by PID.""" + processes: dict[int, str] = {} + for line in dump_stdout.splitlines(): + if ":" not in line: + continue + pid_text, command = line.split(":", 1) + try: + processes[int(pid_text)] = command.strip() + except ValueError: + continue + return processes + + +_STAT_DUMP_SHELL = ( + 'printf "__SAMPLER__:%s\\n" "$$"; ' + "for f in /proc/[0-9]*/stat; do " + '[ -r "$f" ] || continue; ' + 'pid="${f#/proc/}"; pid="${pid%/stat}"; ' + '[ "$pid" = "$$" ] && continue; ' + 'printf "%s|" "$pid"; cat "$f" 2>/dev/null || true; printf "\\n"; ' + "done" +) + + class ProcessCollector(InBandDataCollector[ProcessDataModel, ProcessCollectorArgs]): - """Collect Process details""" + """Collect aggregate CPU usage and top processes from Linux procfs.""" SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.LINUX} DATA_MODEL = ProcessDataModel - CMD_KFD = "rocm-smi --showpids" - CMD_CPU_USAGE = "top -b -n 1" - CMD_PROCESS = "top -b -n 1 -o %CPU " + CMD_PROC_STAT = "cat /proc/stat" + CMD_PROCESS_STAT = _STAT_DUMP_SHELL + CMD_PROCESS_NAMES = ( + "for p in {pids}; do " + 'printf "%s:" "$p"; ' + "cat /proc/$p/comm 2>/dev/null || true; " + 'printf "\\n"; ' + "done" + ) + + def _collect_procfs_cpu( + self, top_n_process: int, sample_interval_seconds: float + ) -> tuple[Optional[float], list[tuple[str, str]]]: + """Collect aggregate CPU usage and top process CPU shares.""" + stat1 = self._run_sut_cmd(self.CMD_PROC_STAT) + if stat1.exit_code != 0: + self.logger.warning( + "Unable to read first aggregate CPU sample (exit code %s)", stat1.exit_code + ) + return None, [] + dump1 = self._run_sut_cmd(self.CMD_PROCESS_STAT, log_artifact=False) + if dump1.exit_code != 0: + self.logger.warning( + "Unable to read first process CPU sample (exit code %s)", dump1.exit_code + ) + return None, [] + + time.sleep(sample_interval_seconds) + + stat2 = self._run_sut_cmd(self.CMD_PROC_STAT) + if stat2.exit_code != 0: + self.logger.warning( + "Unable to read second aggregate CPU sample (exit code %s)", stat2.exit_code + ) + return None, [] + dump2 = self._run_sut_cmd(self.CMD_PROCESS_STAT, log_artifact=False) + if dump2.exit_code != 0: + self.logger.warning( + "Unable to read second process CPU sample (exit code %s)", dump2.exit_code + ) + return None, [] + + aggregate1 = _parse_aggregate_cpu_from_proc_stat(stat1.stdout) + aggregate2 = _parse_aggregate_cpu_from_proc_stat(stat2.stdout) + if aggregate1 is None or aggregate2 is None: + sample = "first" if aggregate1 is None else "second" + output = stat1.stdout if aggregate1 is None else stat2.stdout + self._log_event( + category=EventCategory.OS, + description=f"Could not parse aggregate cpu line from /proc/stat ({sample} sample)", + data={"proc_stat_preview": output[:200]}, + priority=EventPriority.ERROR, + ) + return None, [] + + total1, idle1 = aggregate1 + total2, idle2 = aggregate2 + cpu_usage = _global_non_idle_percent(total1, idle1, total2, idle2) + total_delta = max(0, total2 - total1) + + sample1, excluded1 = _parse_proc_stat_dump(dump1.stdout) + sample2, excluded2 = _parse_proc_stat_dump(dump2.stdout) + top_processes = _top_process_cpu_shares( + sample1, + sample2, + total_delta, + top_n_process, + excluded1 | excluded2, + ) + + if not top_processes: + return cpu_usage, [] + + pids = " ".join(str(pid) for pid, _percentage in top_processes) + process_names = self._run_sut_cmd(self.CMD_PROCESS_NAMES.format(pids=pids)) + names = _parse_comm_dump(process_names.stdout) if process_names.exit_code == 0 else {} + processes = [ + (names.get(pid) or f"pid_{pid}", f"{percentage:.1f}") + for pid, percentage in top_processes + ] + return cpu_usage, processes def collect_data( self, args: Optional[ProcessCollectorArgs] = None @@ -59,58 +270,26 @@ def collect_data( args = ProcessCollectorArgs() process_data = ProcessDataModel() - process_data.processes = [] - - kfd_process = self._run_sut_cmd(self.CMD_KFD) - if kfd_process.exit_code == 0: - if "No KFD PIDs currently running" in kfd_process.stdout: - process_data.kfd_process = 0 - else: - kfd_process = re.findall( - r"^\s*\d+\s+[\w]+\s+\d+\s+\d+\s+\d+\s+\d+", - kfd_process.stdout, - re.MULTILINE, - ) - process_data.kfd_process = len(kfd_process) - - cpu_usage = self._run_sut_cmd(self.CMD_CPU_USAGE) - if cpu_usage.exit_code == 0: - cpu_idle = ( - [line for line in cpu_usage.stdout.splitlines() if "Cpu(s)" in line][0] - .split(",")[3] - .split()[0] - .replace("%id", "") - ) - process_data.cpu_usage = 100 - float(cpu_idle) - - last_line = args.top_n_process + 7 - processes = self._run_sut_cmd( - f"{self.CMD_PROCESS} | sed -n '8,{last_line}p'" - ) # Remove system header - if processes.exit_code == 0: - for line in processes.stdout.splitlines(): - columns = line.split() - process_cpu_usage = columns[8] - process_name = columns[11] - process_data.processes.append((process_name, process_cpu_usage)) - - process_check = bool(process_data.model_fields_set) - if process_check: + cpu_usage, processes = self._collect_procfs_cpu( + args.top_n_process, args.sample_interval_seconds + ) + if cpu_usage is not None: + process_data.cpu_usage = cpu_usage + process_data.processes = processes self._log_event( - category="PROCESS_READ", + category=EventCategory.PROCESS_READ, description="Process data collected", - data=process_data.model_dump(), priority=EventPriority.INFO, ) self.result.message = "Process data collected" self.result.status = ExecutionStatus.OK return self.result, process_data - else: - self._log_event( - category=EventCategory.OS, - description="Process data not found", - priority=EventPriority.ERROR, - ) - self.result.message = "Process data not found" - self.result.status = ExecutionStatus.ERROR - return self.result, None + + self._log_event( + category=EventCategory.OS, + description="Process data not found", + priority=EventPriority.ERROR, + ) + self.result.message = "Process data not found" + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result, None diff --git a/nodescraper/plugins/inband/process/processdata.py b/nodescraper/plugins/inband/process/processdata.py index 683ad948..3073e250 100644 --- a/nodescraper/plugins/inband/process/processdata.py +++ b/nodescraper/plugins/inband/process/processdata.py @@ -25,10 +25,16 @@ ############################################################################### from typing import Optional +from pydantic import Field + from nodescraper.models import DataModel class ProcessDataModel(DataModel): - kfd_process: Optional[int] = None - cpu_usage: Optional[float] = None + """CPU execution usage (excluding idle and I/O-wait) and top processes from procfs.""" + + cpu_usage: Optional[float] = Field( + default=None, + description="Aggregate CPU execution percentage; idle and I/O-wait time are excluded.", + ) processes: Optional[list[tuple[str, str]]] = None diff --git a/nodescraper/plugins/inband/rdma/analyzer_args.py b/nodescraper/plugins/inband/rdma/analyzer_args.py index 0507ed15..282ab9cf 100644 --- a/nodescraper/plugins/inband/rdma/analyzer_args.py +++ b/nodescraper/plugins/inband/rdma/analyzer_args.py @@ -33,6 +33,10 @@ class RdmaAnalyzerArgs(AnalyzerArgs): """Arguments for the RDMA analyzer.""" + expected_nic_firmware: Optional[str] = Field( + default=None, + description="Exact firmware version expected from each collected RDMA NIC.", + ) exclusion_regex: Optional[list[str]] = Field( default=None, description="Regex patterns matched against an interface netdev; matching interfaces are skipped.", diff --git a/nodescraper/plugins/inband/rdma/rdma_analyzer.py b/nodescraper/plugins/inband/rdma/rdma_analyzer.py index ba48c20e..441327a4 100644 --- a/nodescraper/plugins/inband/rdma/rdma_analyzer.py +++ b/nodescraper/plugins/inband/rdma/rdma_analyzer.py @@ -29,6 +29,7 @@ from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus from nodescraper.interfaces import DataAnalyzer from nodescraper.models import TaskResult +from nodescraper.utils import _validate_firmware_policy from .analyzer_args import RdmaAnalyzerArgs from .rdmadata import RdmaDataModel @@ -54,14 +55,41 @@ def analyze_data( Returns: TaskResult with status OK if no errors, ERROR if any error counter > 0. """ - if not data.statistic_list: + if not args: + args = RdmaAnalyzerArgs() + + firmware_records = [ + { + "identity": device.device, + "version": device.firmware_version, + "source": "rdma", + "vendor": _vendor_for_rdma_device(device.device), + "driver": device.device, + } + for device in data.dev_list + ] + firmware_policy_issues = _validate_firmware_policy( + firmware_records, + ( + {"expected_nic_firmware": args.expected_nic_firmware} + if args.expected_nic_firmware + else None + ), + ) + for issue in firmware_policy_issues: + self._log_event( + category=EventCategory.NETWORK, + description="RDMA adapter firmware policy mismatch", + data=issue, + priority=EventPriority.WARNING, + console_log=True, + ) + + if not data.statistic_list and not data.dev_list: self.result.message = "No RDMA devices found" self.result.status = ExecutionStatus.WARNING return self.result - if not args: - args = RdmaAnalyzerArgs() - compiled_exclusions = [re.compile(pattern) for pattern in (args.exclusion_regex or [])] error_detected = False @@ -124,6 +152,9 @@ def analyze_data( if error_detected or critical_detected: self.result.message = "RDMA errors detected in statistics" self.result.status = ExecutionStatus.ERROR + elif firmware_policy_issues: + self.result.message = "RDMA adapter firmware policy mismatch" + self.result.status = ExecutionStatus.WARNING else: self.result.message = "No RDMA errors detected in statistics" self.result.status = ExecutionStatus.OK @@ -132,3 +163,14 @@ def analyze_data( self.result.message += f" ({skipped_count} skipped)" return self.result + + +def _vendor_for_rdma_device(device: str) -> Optional[str]: + normalized = device.lower() + if normalized.startswith(("mlx", "ib_")): + return "Mellanox" + if normalized.startswith(("bnxt", "bnx2")): + return "Broadcom" + if normalized.startswith("ionic"): + return "Pensando" + return None diff --git a/nodescraper/plugins/inband/rdma/rdma_collector.py b/nodescraper/plugins/inband/rdma/rdma_collector.py index 90d68aa4..18e1344b 100644 --- a/nodescraper/plugins/inband/rdma/rdma_collector.py +++ b/nodescraper/plugins/inband/rdma/rdma_collector.py @@ -125,6 +125,9 @@ def _parse_rdma_dev(self, output: str) -> list[RdmaDevice]: device.node_type = parts[i + 1] i += 2 elif parts[i] == "fw" and i + 1 < len(parts): + device.firmware_version = parts[i + 1] + # Keep the legacy attribute for consumers of older + # serialized RDMA data models. device.attributes["fw_version"] = parts[i + 1] i += 2 elif parts[i] == "node_guid" and i + 1 < len(parts): @@ -281,6 +284,23 @@ def _get_rdma_link(self) -> Optional[list[RdmaLink]]: ) continue links.append(RdmaLink(**link)) + self._log_event( + category=EventCategory.NETWORK, + description=f"Collected {len(links)} RDMA links from 'rdma link -j'", + data={ + "links": [ + { + "ifname": link.ifname, + "port": link.port, + "state": link.state, + "physical_state": link.physical_state, + "netdev": link.netdev, + } + for link in links + ] + }, + priority=EventPriority.INFO, + ) return links except ValidationError as e: self._log_event( @@ -318,6 +338,11 @@ def collect_data( self._log_event( category=EventCategory.NETWORK, description=f"Collected {len(dev_list)} RDMA devices from 'rdma dev'", + data={ + "devices": [ + {"device": device.device, "state": device.state} for device in dev_list + ] + }, priority=EventPriority.INFO, ) else: @@ -335,6 +360,18 @@ def collect_data( self._log_event( category=EventCategory.NETWORK, description=f"Collected {len(link_list_text)} RDMA links from 'rdma link'", + data={ + "links": [ + { + "device": link.device, + "port": link.port, + "state": link.state, + "physical_state": link.physical_state, + "netdev": link.netdev, + } + for link in link_list_text + ] + }, priority=EventPriority.INFO, ) else: diff --git a/nodescraper/plugins/inband/rdma/rdmadata.py b/nodescraper/plugins/inband/rdma/rdmadata.py index 2859d325..7207209f 100644 --- a/nodescraper/plugins/inband/rdma/rdmadata.py +++ b/nodescraper/plugins/inband/rdma/rdmadata.py @@ -407,6 +407,7 @@ class RdmaDevice(BaseModel): node_guid: Optional[str] = None sys_image_guid: Optional[str] = None state: Optional[str] = None + firmware_version: Optional[str] = None attributes: dict[str, str] = Field(default_factory=dict) diff --git a/nodescraper/plugins/inband/storage/analyzer_args.py b/nodescraper/plugins/inband/storage/analyzer_args.py index fcd5ed10..827b0e70 100644 --- a/nodescraper/plugins/inband/storage/analyzer_args.py +++ b/nodescraper/plugins/inband/storage/analyzer_args.py @@ -51,3 +51,7 @@ class StorageAnalyzerArgs(AnalyzerArgs): default=False, description="If True, match device/mount names with regex; otherwise exact match.", ) + skip_loop_devices: bool = Field( + default=True, + description="If True, skip /dev/loopN devices (read-only image mounts that always report 100 percent used) unless listed in check_devices.", + ) diff --git a/nodescraper/plugins/inband/storage/storage_analyzer.py b/nodescraper/plugins/inband/storage/storage_analyzer.py index 6c2f8334..7c06d1f2 100644 --- a/nodescraper/plugins/inband/storage/storage_analyzer.py +++ b/nodescraper/plugins/inband/storage/storage_analyzer.py @@ -40,6 +40,19 @@ class StorageAnalyzer(DataAnalyzer[StorageDataModel, StorageAnalyzerArgs]): DATA_MODEL = StorageDataModel + LOOP_DEVICE_REGEX = re.compile(r"^/dev/loop\d+$") + + def _is_loop_device(self, device_name: str) -> bool: + """Check if the device name is a loop device (e.g. /dev/loop0) + + Args: + device_name (str): device name to check + + Returns: + bool: True if the device name is a /dev/loopN device, False otherwise + """ + return bool(self.LOOP_DEVICE_REGEX.match(device_name)) + def _matches_device_filter( self, device_name: str, exp_devices: list[str], regex_match: bool ) -> bool: @@ -104,8 +117,12 @@ def analyze_data( device_name, args.check_devices, args.regex_match ): continue - elif args.ignore_devices: - if self._matches_device_filter(device_name, args.ignore_devices, args.regex_match): + else: + if args.skip_loop_devices and self._is_loop_device(device_name): + continue + if args.ignore_devices and self._matches_device_filter( + device_name, args.ignore_devices, args.regex_match + ): continue condition = False diff --git a/nodescraper/plugins/inband/storage/storage_collector.py b/nodescraper/plugins/inband/storage/storage_collector.py index e5373ebc..3ab43583 100644 --- a/nodescraper/plugins/inband/storage/storage_collector.py +++ b/nodescraper/plugins/inband/storage/storage_collector.py @@ -37,9 +37,11 @@ class StorageCollector(InBandDataCollector[StorageDataModel, None]): """Collect disk usage details""" + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.WINDOWS, OSFamily.LINUX, OSFamily.ESXI} DATA_MODEL = StorageDataModel CMD_WINDOWS = """wmic LogicalDisk Where DriveType="3" Get DeviceId,Size,FreeSpace""" CMD = """sh -c 'df -lH -B1 | grep -v 'boot''""" + CMD_ESXI = "esxcli storage filesystem list" def collect_data( self, args: Optional[StorageCollectorArgs] = None @@ -61,6 +63,28 @@ def collect_data( used=int(size) - int(free_space), percent=round((int(size) - int(free_space)) / int(size) * 100, 2), ) + elif self.system_info.os_family == OSFamily.ESXI: + res = self._run_sut_cmd(self.CMD_ESXI) + if res.exit_code == 0: + for line in res.stdout.splitlines(): + # esxcli columns (fixed order): [0] Mount Point [1] Volume Name + # [2] UUID [3] Mounted [4] Type [5] Size [6] Free + fields = re.split(r"\s{2,}", line.strip()) + if len(fields) >= 7 and fields[5].isdigit() and fields[6].isdigit(): + device_id = fields[0] + total_bytes = int(fields[5]) + free_bytes = int(fields[6]) + used_bytes = total_bytes - free_bytes + if total_bytes: + usage_percent = round(used_bytes / total_bytes * 100, 2) + else: + usage_percent = 0.0 + storage_data[device_id] = DeviceStorageData( + total=total_bytes, + free=free_bytes, + used=used_bytes, + percent=usage_percent, + ) else: if args.skip_sudo: self.result.message = "Skipping sudo plugin" @@ -98,7 +122,6 @@ def collect_data( self._log_event( category="STORAGE_READ", description="Available storage read", - data=storage_model.model_dump(), priority=EventPriority.INFO, ) self.result.message = f"{len(storage_model.storage_data)} storage devices collected" diff --git a/nodescraper/plugins/inband/switch/__init__.py b/nodescraper/plugins/inband/switch/__init__.py index ad8bbd8d..41405fe6 100644 --- a/nodescraper/plugins/inband/switch/__init__.py +++ b/nodescraper/plugins/inband/switch/__init__.py @@ -25,5 +25,6 @@ ############################################################################### from .scale_out_arista import ScaleOutAristaPlugin from .scale_out_dell import ScaleOutDellPlugin +from .scale_up_sonic import ScaleUpSonicPlugin -__all__ = ["ScaleOutAristaPlugin", "ScaleOutDellPlugin"] +__all__ = ["ScaleOutAristaPlugin", "ScaleOutDellPlugin", "ScaleUpSonicPlugin"] diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/__init__.py b/nodescraper/plugins/inband/switch/scale_up_sonic/__init__.py new file mode 100644 index 00000000..bee20831 --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/__init__.py @@ -0,0 +1,28 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from .scale_up_sonic_plugin import ScaleUpSonicPlugin + +__all__ = ["ScaleUpSonicPlugin"] diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/analyzer_args.py b/nodescraper/plugins/inband/switch/scale_up_sonic/analyzer_args.py new file mode 100644 index 00000000..4985d2ee --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/analyzer_args.py @@ -0,0 +1,47 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import List, Optional + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class ScaleUpSonicAnalyzerArgs(AnalyzerArgs): + """Arguments for the scale up SONiC switch analyzer.""" + + analysis_ports: Optional[List[str]] = Field( + default=None, + description=( + "Restrict per-port analysis to the given full port names " + "(e.g. ['Ethernet1', 'Ethernet31']). " + "When omitted, every port present in the data is analyzed." + ), + ) + expected_port_speed: str = Field( + default="400G", + description=("Expected interface speed from show interface status "), + ) diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/collector_args.py b/nodescraper/plugins/inband/switch/scale_up_sonic/collector_args.py new file mode 100644 index 00000000..4222cd5c --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/collector_args.py @@ -0,0 +1,40 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class ScaleUpSonicCollectorArgs(CollectorArgs): + """Arguments for the scale up SONiC switch collector.""" + + html_view: bool = Field( + default=True, + description=( + "When true, include logged command artifacts in command_artifacts.html " + "using human-readable output." + ), + ) diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/port_names.py b/nodescraper/plugins/inband/switch/scale_up_sonic/port_names.py new file mode 100644 index 00000000..ad9541d6 --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/port_names.py @@ -0,0 +1,29 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import re + +PORT_TOKEN_RE = re.compile(r"^(Ethernet\d+)$") diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_analyzer.py b/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_analyzer.py new file mode 100644 index 00000000..d751dd79 --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_analyzer.py @@ -0,0 +1,77 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import re +from typing import Any, ClassVar + +from pydantic import BaseModel + +from nodescraper.interfaces import DataAnalyzer + +from ..switch_analyzer_base import SwitchAnalyzerBase +from .analyzer_args import ScaleUpSonicAnalyzerArgs +from .port_names import PORT_TOKEN_RE +from .scaleupsonicdata import ScaleUpSonicDataModel, SonicPortData + + +class ScaleUpSonicAnalyzer( + SwitchAnalyzerBase[ScaleUpSonicDataModel], + DataAnalyzer[ScaleUpSonicDataModel, ScaleUpSonicAnalyzerArgs], +): + """Check scale up SONiC switch data for errors and warnings. + + Walks every model in the collected :class:`ScaleUpSonicDataModel` and checks + each ``error_fields`` / ``warning_fields`` ClassVar against an optional + ``ports`` filter. + """ + + VENDOR_NAME: ClassVar[str] = "SONiC" + DATA_MODEL = ScaleUpSonicDataModel + + PORT_NAME_RE: ClassVar[re.Pattern] = PORT_TOKEN_RE + PORT_FORMAT_HINT: ClassVar[str] = "expected full port name 'Ethernet' (e.g. 'Ethernet1')" + + def _extra_port_findings(self, port_name: str, port_data: BaseModel) -> list[dict[str, Any]]: + if not isinstance(port_data, SonicPortData): + return [] + + args = self._analyzer_args + if not isinstance(args, ScaleUpSonicAnalyzerArgs): + args = ScaleUpSonicAnalyzerArgs() + + status = port_data.interface_status + if status is None: + return [] + + finding = self._port_field_mismatch( + port_name, + "interface_status", + "speed", + status.speed, + args.expected_port_speed, + "SonicInterfaceStatus", + ) + return [finding] if finding else [] diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_collector.py b/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_collector.py new file mode 100644 index 00000000..5930b3d2 --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_collector.py @@ -0,0 +1,490 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import re +from typing import Dict, List, Optional, TypedDict + +from pydantic import ValidationError + +from nodescraper.base import InBandDataCollector +from nodescraper.connection.inband import CommandArtifact +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily +from nodescraper.models import TaskResult +from nodescraper.utils import get_exception_details, get_exception_traceback + +from .collector_args import ScaleUpSonicCollectorArgs +from .scaleupsonicdata import ( + ScaleUpSonicDataModel, + SonicFecStatus, + SonicInterfaceCounters, + SonicInterfaceStatus, + SonicPortData, + SonicQueueCounter, +) + + +class _ParsedInterfaceStatusLine(TypedDict): + name: str + lanes: str + speed: str + mtu: Optional[int] + fec: str + alias: str + vlan: str + oper: str + admin: str + type: str + asym_pfc: str + + +# Counter columns are comma grouped and may be reported as ``N/A`` +_COUNT = r"[\d,]+|N/A" + + +class ScaleUpSonicCollector(InBandDataCollector[ScaleUpSonicDataModel, ScaleUpSonicCollectorArgs]): + """Collect scale up SONiC switch data. + + Runs SONiC ``show`` commands over SSH and parses their text output into a + :class:`ScaleUpSonicDataModel`. Unlike the Dell collector, commands are + issued directly and are **not** wrapped in ``sonic-cli -c``. + """ + + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.SONIC, OSFamily.LINUX, OSFamily.UNKNOWN} + + DATA_MODEL = ScaleUpSonicDataModel + + CMD_VERSION = "show version" + CMD_INTERFACE_STATUS = "show interface status" + CMD_INTERFACE_COUNTERS = "show interface counters" + CMD_FEC_STATUS = "show interface fec status" + CMD_QUEUE_COUNTERS = "show queue counters" + + # Commands run for diagnostics, not parsed into a data model. + CMD_CLOCK = "show clock" + CMD_PLATFORM_SYSEEPROM = "show platform syseeprom" + CMD_PLATFORM_FIRMWARE_DETAIL = "show platform firmware version" + CMD_IP_INTERFACES = "show ip interfaces" + CMD_LLDP_TABLE = "show lldp table" + CMD_LLDP_NEIGHBOR = "show lldp neighbors" + CMD_QUEUE_WATERMARK_UNICAST = "show queue watermark unicast" + CMD_QUEUE_WATERMARK_MULTICAST = "show queue watermark multicast" + CMD_QUEUE_PERSISTENT_WATERMARK_UNICAST = "show queue persistent-watermark unicast" + CMD_QUEUE_PERSISTENT_WATERMARK_MULTICAST = "show queue persistent-watermark multicast" + CMD_PLATFORM_ENVIRONMENT = "show platform summary" + + _INTERFACE_STATUS_LINE_RE = re.compile( + r"^" + r"(?PEthernet\S*)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s+(?P\S+)" + r"\s*$" + ) + + # Aggregate of the diagnostic CMD_* commands above + ARTIFACT_COMMANDS: list[str] = [ + CMD_CLOCK, + CMD_PLATFORM_SYSEEPROM, + CMD_PLATFORM_FIRMWARE_DETAIL, + CMD_IP_INTERFACES, + CMD_LLDP_TABLE, + CMD_LLDP_NEIGHBOR, + CMD_QUEUE_WATERMARK_UNICAST, + CMD_QUEUE_WATERMARK_MULTICAST, + CMD_QUEUE_PERSISTENT_WATERMARK_UNICAST, + CMD_QUEUE_PERSISTENT_WATERMARK_MULTICAST, + CMD_PLATFORM_ENVIRONMENT, + ] + + # helpers + @staticmethod + def _is_sonic_output(text: str) -> bool: + return "sonic" in text.lower() + + @staticmethod + def _to_int(value: str) -> Optional[int]: + """Convert a comma grouped counter value to an int, or ``None`` if not numeric.""" + try: + return int(value.replace(",", "")) + except ValueError: + return None + + def _run_sonic_command(self, command: str) -> Optional[str]: + """Run a SONiC CLI command directly, without a ``sonic-cli -c`` wrapper. + + Args: + command: The full CLI command to run. + + Returns: + The command stdout, or ``None`` on error. + """ + cmd_ret: CommandArtifact = self._run_sut_cmd(command) + if cmd_ret.exit_code != 0: + self._log_event( + category=EventCategory.SWITCH, + description=f"Error running SONiC command: `{command}`", + data={ + "command": command, + "exit_code": cmd_ret.exit_code, + "stderr": cmd_ret.stderr, + }, + priority=EventPriority.ERROR, + console_log=True, + ) + return None + return cmd_ret.stdout or "" + + # sub-collectors + @classmethod + def _parse_interface_status_line(cls, line: str) -> Optional[_ParsedInterfaceStatusLine]: + """Parse one ``show interface status`` row.""" + stripped = line.strip() + if not stripped.startswith("Ethernet"): + return None + match = cls._INTERFACE_STATUS_LINE_RE.match(stripped) + if not match: + return None + return { + "name": match.group("name"), + "lanes": match.group("lanes"), + "speed": match.group("speed"), + "mtu": cls._to_int(match.group("mtu")), + "fec": match.group("fec"), + "alias": match.group("alias"), + "vlan": match.group("vlan"), + "oper": match.group("oper").lower(), + "admin": match.group("admin").lower(), + "type": match.group("type"), + "asym_pfc": match.group("asym_pfc"), + } + + def get_interface_status(self) -> Optional[Dict[str, SonicInterfaceStatus]]: + """Parse ``show interface status`` into per-port status models. + + Returns: + Mapping of port name to :class:`SonicInterfaceStatus`, or ``None``. + """ + text = self._run_sonic_command(self.CMD_INTERFACE_STATUS) + if text is None: + return None + result: Dict[str, SonicInterfaceStatus] = {} + for line in text.splitlines(): + parsed = self._parse_interface_status_line(line) + if parsed is None: + continue + name = str(parsed["name"]) + try: + result[name] = SonicInterfaceStatus(**parsed) + except (ValidationError, TypeError) as e: + self._log_event( + category=EventCategory.SWITCH, + description=f"Failed to build SonicInterfaceStatus for {name}", + data=get_exception_details(e), + priority=EventPriority.WARNING, + ) + return result or None + + def get_interface_counters(self) -> Optional[Dict[str, SonicInterfaceCounters]]: + """Parse ``show interface counters`` into per-port counter models. + + Returns: + Mapping of port name to :class:`SonicInterfaceCounters`, or ``None``. + """ + text = self._run_sonic_command(self.CMD_INTERFACE_COUNTERS) + if text is None: + return None + line_pattern = re.compile( + r"^(?PEthernet\S*)" + r"\s+(?P\S+)" + rf"\s+(?P{_COUNT})" + r"\s+(?P[\d.,]+\s+\S*B/s)" + r"\s+(?P\S+)" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + r"\s+(?P[\d.,]+\s+\S*B/s)" + r"\s+(?P\S+)" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + r"\s*$" + ) + result: Dict[str, SonicInterfaceCounters] = {} + for line in text.splitlines(): + match = line_pattern.match(line.strip()) + if not match: + continue + name = match.group("name") + try: + result[name] = SonicInterfaceCounters( + state=match.group("state"), + rx_ok=self._to_int(match.group("rx_ok")), + rx_bps=match.group("rx_bps"), + rx_util=match.group("rx_util"), + rx_err=self._to_int(match.group("rx_err")), + rx_drp=self._to_int(match.group("rx_drp")), + rx_ovr=self._to_int(match.group("rx_ovr")), + tx_ok=self._to_int(match.group("tx_ok")), + tx_bps=match.group("tx_bps"), + tx_util=match.group("tx_util"), + tx_err=self._to_int(match.group("tx_err")), + tx_drp=self._to_int(match.group("tx_drp")), + tx_ovr=self._to_int(match.group("tx_ovr")), + ) + except (ValidationError, TypeError) as e: + self._log_event( + category=EventCategory.SWITCH, + description=f"Failed to build SonicInterfaceCounters for {name}", + data=get_exception_details(e), + priority=EventPriority.WARNING, + ) + return result or None + + def get_fec_status(self) -> Optional[Dict[str, SonicFecStatus]]: + """Parse ``show interface fec status`` into per-port FEC models. + + Returns: + Mapping of port name to :class:`SonicFecStatus`, or ``None``. + """ + text = self._run_sonic_command(self.CMD_FEC_STATUS) + if text is None: + return None + line_pattern = re.compile(r"^(?PEthernet\S*)\s+(?P\S+)\s+(?P\S+)\s*$") + result: Dict[str, SonicFecStatus] = {} + for line in text.splitlines(): + match = line_pattern.match(line.strip()) + if not match: + continue + name = match.group("name") + try: + result[name] = SonicFecStatus( + oper=match.group("oper"), + admin=match.group("admin"), + ) + except (ValidationError, TypeError) as e: + self._log_event( + category=EventCategory.SWITCH, + description=f"Failed to build SonicFecStatus for {name}", + data=get_exception_details(e), + priority=EventPriority.WARNING, + ) + return result or None + + def get_queue_counters(self) -> Optional[Dict[str, List[SonicQueueCounter]]]: + """Parse ``show queue counters`` (``Eth*`` rows only) per port. + + Returns: + Mapping of port name to a list of :class:`SonicQueueCounter`, + or ``None``. + """ + text = self._run_sonic_command(self.CMD_QUEUE_COUNTERS) + if text is None: + return None + line_pattern = re.compile( + r"^(?PEthernet\S*)" + r"\s+(?P\S+)" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + rf"\s+(?P{_COUNT})" + r"\s*$" + ) + result: Dict[str, List[SonicQueueCounter]] = {} + for line in text.splitlines(): + match = line_pattern.match(line.strip()) + if not match: + continue + name = match.group("port") + try: + entry = SonicQueueCounter( + txq=match.group("txq"), + counter_pkts=self._to_int(match.group("pkts")), + counter_bytes=self._to_int(match.group("bytes")), + drop_pkts=self._to_int(match.group("drop_pkts")), + drop_bytes=self._to_int(match.group("drop_bytes")), + ) + result.setdefault(name, []).append(entry) + except (ValidationError, TypeError) as e: + self._log_event( + category=EventCategory.SWITCH, + description=f"Failed to build SonicQueueCounter for {name}", + data=get_exception_details(e), + priority=EventPriority.WARNING, + ) + return result or None + + # artifact-only collectors + + def collect_artifact_commands(self) -> None: + """Run diagnostic commands so their output is captured in ``command_artifacts.json``. + + Each command is executed via :meth:`_run_sut_cmd`, which records the + command, stdout, stderr and exit code as a ``CommandArtifact`` in + ``command_artifacts.json``. No separate per-command ``.log`` files are + written. Failures are logged but do **not** cause the overall + collection to fail. + """ + for command in self.ARTIFACT_COMMANDS: + try: + cmd_ret = self._run_sut_cmd(command) + if cmd_ret.exit_code != 0: + self._log_event( + category=EventCategory.SWITCH, + description=f"Error running artifact command: `{command}`", + data={ + "command": command, + "exit_code": cmd_ret.exit_code, + "stderr": cmd_ret.stderr, + }, + priority=EventPriority.ERROR, + console_log=True, + ) + continue + except Exception as e: + self._log_event( + category=EventCategory.SWITCH, + description=f"Error collecting artifact for command: `{command}`", + data={ + "command": command, + "exception": get_exception_traceback(e), + }, + priority=EventPriority.WARNING, + console_log=True, + ) + + def _preflight_check(self) -> bool: + """Verify the device is a reachable SONiC switch. + + Ensures the device responds and identifies as SONiC. + + On failure this sets ``self.result.status`` to + :attr:`ExecutionStatus.NOT_RAN` and returns ``False``. + + Returns: + ``True`` if the pre-flight check passed, ``False`` otherwise. + """ + version_text = self._run_sonic_command(self.CMD_VERSION) + if version_text is None: + self._log_event( + category=EventCategory.SWITCH, + description="ScaleUpSonicCollector pre-flight check failed", + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.NOT_RAN + return False + + if not self._is_sonic_output(version_text): + self._log_event( + category=EventCategory.SWITCH, + description="Not a SONiC switch", + data={"raw_output": version_text}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.NOT_RAN + return False + + return True + + # main entry point + + def collect_data( + self, args: Optional[ScaleUpSonicCollectorArgs] = None + ) -> tuple[TaskResult, Optional[ScaleUpSonicDataModel]]: + """Run all SONiC collectors and assemble the switch data model. + + Args: + args: Optional :class:`ScaleUpSonicCollectorArgs`. + + Returns: + Tuple of ``(TaskResult, ScaleUpSonicDataModel | None)``. + """ + + if not self._preflight_check(): + return self.result, None + + try: + interface_status = self.get_interface_status() + interface_counters = self.get_interface_counters() + fec_status = self.get_fec_status() + queue_counters = self.get_queue_counters() + + self.collect_artifact_commands() + except Exception as e: + self._log_event( + category=EventCategory.SWITCH, + description="Error running SONiC collector sub commands", + data={"exception": get_exception_traceback(e)}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result, None + + all_port_names: set[str] = set() + for source in (interface_status, interface_counters, fec_status, queue_counters): + if source: + all_port_names.update(source.keys()) + + port_data: Optional[Dict[str, SonicPortData]] = None + if all_port_names: + port_data = {} + for name in sorted(all_port_names): + port_data[name] = SonicPortData( + interface_status=interface_status.get(name) if interface_status else None, + interface_counters=( + interface_counters.get(name) if interface_counters else None + ), + fec_status=fec_status.get(name) if fec_status else None, + queue_counters=queue_counters.get(name) if queue_counters else None, + ) + + try: + sonic_data = ScaleUpSonicDataModel( + port_list=sorted(all_port_names) if all_port_names else None, + port=port_data, + ) + except (ValidationError, TypeError) as e: + self._log_event( + category=EventCategory.SWITCH, + description="Failed to build ScaleUpSonicDataModel", + data=get_exception_details(e), + priority=EventPriority.ERROR, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result, None + + self.result.message = "SONiC switch data collected" + return self.result, sonic_data diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_plugin.py b/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_plugin.py new file mode 100644 index 00000000..10a8ee09 --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/scale_up_sonic_plugin.py @@ -0,0 +1,48 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.base import InBandDataPlugin + +from .analyzer_args import ScaleUpSonicAnalyzerArgs +from .collector_args import ScaleUpSonicCollectorArgs +from .scale_up_sonic_analyzer import ScaleUpSonicAnalyzer +from .scale_up_sonic_collector import ScaleUpSonicCollector +from .scaleupsonicdata import ScaleUpSonicDataModel + + +class ScaleUpSonicPlugin( + InBandDataPlugin[ScaleUpSonicDataModel, ScaleUpSonicCollectorArgs, ScaleUpSonicAnalyzerArgs] +): + """Plugin for collection and analysis of scale up SONiC switch data""" + + DATA_MODEL = ScaleUpSonicDataModel + + COLLECTOR = ScaleUpSonicCollector + + COLLECTOR_ARGS = ScaleUpSonicCollectorArgs + + ANALYZER = ScaleUpSonicAnalyzer + + ANALYZER_ARGS = ScaleUpSonicAnalyzerArgs diff --git a/nodescraper/plugins/inband/switch/scale_up_sonic/scaleupsonicdata.py b/nodescraper/plugins/inband/switch/scale_up_sonic/scaleupsonicdata.py new file mode 100644 index 00000000..f283c292 --- /dev/null +++ b/nodescraper/plugins/inband/switch/scale_up_sonic/scaleupsonicdata.py @@ -0,0 +1,132 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +from typing import ClassVar, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel + +from nodescraper.models import DataModel + + +class SonicInterfaceStatus(BaseModel): + """Per-port info from ``show interface status``""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + name: Optional[str] = None + lanes: Optional[str] = None + speed: Optional[str] = None + mtu: Optional[int] = None + fec: Optional[str] = None + alias: Optional[str] = None + vlan: Optional[str] = None + oper: Optional[str] = None + admin: Optional[str] = None + type: Optional[str] = None + asym_pfc: Optional[str] = None + + error_fields: ClassVar[dict[str, List[str]]] = { + "oper": ["up"], + } + + +class SonicFecStatus(BaseModel): + """Per-port FEC status from ``show interface fec status``""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + oper: Optional[str] = None + admin: Optional[str] = None + + +class SonicInterfaceCounters(BaseModel): + """Per-port summary counters from ``show interface counters``""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + state: Optional[str] = None + rx_ok: Optional[int] = None + rx_bps: Optional[str] = None + rx_util: Optional[str] = None + rx_err: Optional[int] = None + rx_drp: Optional[int] = None + rx_ovr: Optional[int] = None + tx_ok: Optional[int] = None + tx_bps: Optional[str] = None + tx_util: Optional[str] = None + tx_err: Optional[int] = None + tx_drp: Optional[int] = None + tx_ovr: Optional[int] = None + + error_fields: ClassVar[dict[str, List[str]]] = { + "state": ["U"], + "rx_err": ["0"], + "rx_ovr": ["0"], + "tx_err": ["0"], + "tx_ovr": ["0"], + } + + warning_fields: ClassVar[dict[str, List[str]]] = { + "rx_drp": ["0"], + "tx_drp": ["0"], + } + + +class SonicQueueCounter(BaseModel): + """Single queue counter entry from ``show queue counters``""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + txq: Optional[str] = None + counter_pkts: Optional[int] = None + counter_bytes: Optional[int] = None + drop_pkts: Optional[int] = None + drop_bytes: Optional[int] = None + + warning_fields: ClassVar[dict[str, List[str]]] = { + "drop_pkts": ["0"], + "drop_bytes": ["0"], + } + + +class SonicPortData(BaseModel): + """All collected per-port data for a SONiC switch""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + interface_status: Optional[SonicInterfaceStatus] = None + interface_counters: Optional[SonicInterfaceCounters] = None + fec_status: Optional[SonicFecStatus] = None + queue_counters: Optional[List[SonicQueueCounter]] = None + + +class ScaleUpSonicDataModel(DataModel): + """Collected output of SONiC switch commands""" + + port_list: Optional[List[str]] = None + + port: Optional[Dict[str, SonicPortData]] = None diff --git a/nodescraper/plugins/inband/sys_settings/analyzer_args.py b/nodescraper/plugins/inband/sys_settings/analyzer_args.py index 594a4ca7..6778e639 100644 --- a/nodescraper/plugins/inband/sys_settings/analyzer_args.py +++ b/nodescraper/plugins/inband/sys_settings/analyzer_args.py @@ -1,92 +1,92 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Optional - -from pydantic import BaseModel, Field - -from nodescraper.models import AnalyzerArgs - - -class SysfsCheck(BaseModel): - """One sysfs check: path to read, acceptable values or pattern, and display name. - - For file paths: use expected (list of acceptable values); if empty, check passes. - For directory paths: use pattern (regex); at least one directory entry must match (e.g. ^hsn[0-9]+). - """ - - path: str = Field(description="Sysfs path to read (file) or list (directory).") - expected: list[str] = Field( - default_factory=list, - description="For file paths: list of acceptable values; if empty, check passes.", - ) - name: str = Field(description="Display name for this check in reports.") - pattern: Optional[str] = Field( - default=None, - description="For directory paths: regex; at least one entry must match (e.g. ^hsn[0-9]+).", - ) - - -class SysSettingsAnalyzerArgs(AnalyzerArgs): - """Sysfs settings for analysis via a list of checks (path, expected values, name). - - The path in each check is the sysfs path to read; the collector uses these paths - when collection_args is derived from analysis_args (e.g. by the plugin). - """ - - checks: Optional[list[SysfsCheck]] = Field( - default=None, - description="List of sysfs checks (path, expected values or pattern, display name).", - ) - - def paths_to_collect(self) -> list[str]: - """Return unique sysfs file paths from checks (those without pattern), for use by the collector.""" - if not self.checks: - return [] - seen = set() - out = [] - for c in self.checks: - if c.pattern: - continue - p = c.path.rstrip("/") - if p not in seen: - seen.add(p) - out.append(c.path) - return out - - def paths_to_list(self) -> list[str]: - """Return unique sysfs directory paths from checks (those with pattern), for listing (ls).""" - if not self.checks: - return [] - seen = set() - out = [] - for c in self.checks: - if not c.pattern: - continue - p = c.path.rstrip("/") - if p not in seen: - seen.add(p) - out.append(c.path) - return out +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import BaseModel, Field + +from nodescraper.models import AnalyzerArgs + + +class SysfsCheck(BaseModel): + """One sysfs check: path to read, acceptable values or pattern, and display name. + + For file paths: use expected (list of acceptable values); if empty, check passes. + For directory paths: use pattern (regex); at least one directory entry must match (e.g. ^hsn[0-9]+). + """ + + path: str = Field(description="Sysfs path to read (file) or list (directory).") + expected: list[str] = Field( + default_factory=list, + description="For file paths: list of acceptable values; if empty, check passes.", + ) + name: str = Field(description="Display name for this check in reports.") + pattern: Optional[str] = Field( + default=None, + description="For directory paths: regex; at least one entry must match (e.g. ^hsn[0-9]+).", + ) + + +class SysSettingsAnalyzerArgs(AnalyzerArgs): + """Sysfs settings for analysis via a list of checks (path, expected values, name). + + The path in each check is the sysfs path to read; the collector uses these paths + when collection_args is derived from analysis_args (e.g. by the plugin). + """ + + checks: Optional[list[SysfsCheck]] = Field( + default=None, + description="List of sysfs checks (path, expected values or pattern, display name).", + ) + + def paths_to_collect(self) -> list[str]: + """Return unique sysfs file paths from checks (those without pattern), for use by the collector.""" + if not self.checks: + return [] + seen = set() + out = [] + for c in self.checks: + if c.pattern: + continue + p = c.path.rstrip("/") + if p not in seen: + seen.add(p) + out.append(c.path) + return out + + def paths_to_list(self) -> list[str]: + """Return unique sysfs directory paths from checks (those with pattern), for listing (ls).""" + if not self.checks: + return [] + seen = set() + out = [] + for c in self.checks: + if not c.pattern: + continue + p = c.path.rstrip("/") + if p not in seen: + seen.add(p) + out.append(c.path) + return out diff --git a/nodescraper/plugins/inband/sys_settings/collector_args.py b/nodescraper/plugins/inband/sys_settings/collector_args.py index 7e49f228..40ef94ab 100644 --- a/nodescraper/plugins/inband/sys_settings/collector_args.py +++ b/nodescraper/plugins/inband/sys_settings/collector_args.py @@ -1,45 +1,50 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from pydantic import Field - -from nodescraper.models import CollectorArgs - - -class SysSettingsCollectorArgs(CollectorArgs): - """Collection args for SysSettingsCollector. - - paths: sysfs paths to read (cat). If a path contains '*', collect with ls -l instead (e.g. class/net/*/device). - directory_paths: sysfs paths to list (ls -1); use for checks that match entry names by regex. - """ - - paths: list[str] = Field( - default_factory=list, - description="Sysfs paths to read (cat). Paths with '*' are collected with ls -l (e.g. class/net/*/device).", - ) - directory_paths: list[str] = Field( - default_factory=list, - description="Sysfs paths to list (ls -1); used for checks that match entry names by regex.", - ) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class SysSettingsCollectorArgs(CollectorArgs): + """Collection args for SysSettingsCollector. + + paths: sysfs paths to read (cat). If a path contains '*', collect with + ``cat`` first (e.g. class/drm/card*/device/vbios_version). If cat fails, + fall back to ls -l (e.g. class/net/*/device). + directory_paths: sysfs paths to list (ls -1); use for checks that match entry names by regex. + """ + + paths: list[str] = Field( + default_factory=list, + description=( + "Sysfs paths to read (cat). Paths with '*' are cat'd via bash " + "(e.g. class/drm/card*/device/vbios_version); if cat fails, ls -l is used." + ), + ) + directory_paths: list[str] = Field( + default_factory=list, + description="Sysfs paths to list (ls -1); used for checks that match entry names by regex.", + ) diff --git a/nodescraper/plugins/inband/sys_settings/sys_settings_analyzer.py b/nodescraper/plugins/inband/sys_settings/sys_settings_analyzer.py index 276b73ce..f31f7432 100644 --- a/nodescraper/plugins/inband/sys_settings/sys_settings_analyzer.py +++ b/nodescraper/plugins/inband/sys_settings/sys_settings_analyzer.py @@ -1,161 +1,161 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import re -from typing import List, Optional, Union, cast - -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.interfaces import DataAnalyzer -from nodescraper.models import TaskResult - -from .analyzer_args import SysSettingsAnalyzerArgs -from .sys_settings_data import SysSettingsDataModel - - -def _get_actual_for_path(data: SysSettingsDataModel, path: str) -> Optional[str]: - """Return the actual value from the data model for the given sysfs path. - - Args: - data: Collected sysfs readings (path -> value). - path: Sysfs path (with or without trailing slash). - - Returns: - Normalized value for that path, or None if not present. - """ - value = data.readings.get(path) or data.readings.get(path.rstrip("/")) - return (value or "").strip().lower() if value is not None else None - - -class SysSettingsAnalyzer(DataAnalyzer[SysSettingsDataModel, SysSettingsAnalyzerArgs]): - """Check sysfs settings against expected values from the checks list.""" - - DATA_MODEL = SysSettingsDataModel - - def analyze_data( - self, data: SysSettingsDataModel, args: Optional[SysSettingsAnalyzerArgs] = None - ) -> TaskResult: - """Compare sysfs data to expected settings from args.checks. - - Args: - data: Collected sysfs readings to check. - args: Analyzer args with checks (path, expected, name). If None or no checks, returns OK. - - Returns: - TaskResult with status OK if all checks pass, ERROR if any mismatch or missing path. - """ - mismatches: dict[str, dict[str, Union[Optional[str], List[str]]]] = {} - - if not args or not args.checks: - self.result.status = ExecutionStatus.OK - self.result.message = "No checks configured." - return self.result - - for check in args.checks: - raw_reading = data.readings.get(check.path) or data.readings.get(check.path.rstrip("/")) - - if check.pattern: - # Directory-listing check: at least one line must match the regex (e.g. ^hsn[0-9]+) - if raw_reading is None: - mismatches[check.name] = { - "path": check.path, - "pattern": check.pattern, - "actual": None, - "reason": "path not collected by this plugin", - } - continue - try: - pat = re.compile(check.pattern) - except re.error: - mismatches[check.name] = { - "path": check.path, - "pattern": check.pattern, - "reason": "invalid regex", - } - continue - lines = [ln.strip() for ln in raw_reading.splitlines() if ln.strip()] - if not any(pat.search(ln) for ln in lines): - mismatches[check.name] = { - "path": check.path, - "pattern": check.pattern, - "actual": lines, - } - continue - - actual = _get_actual_for_path(data, check.path) - if actual is None: - mismatches[check.name] = { - "path": check.path, - "expected": check.expected, - "actual": None, - "reason": "path not collected by this plugin", - } - continue - - if not check.expected: - continue - expected_normalized = [e.strip().lower() for e in check.expected] - if actual not in expected_normalized: - raw = data.readings.get(check.path) or data.readings.get(check.path.rstrip("/")) - mismatches[check.name] = { - "path": check.path, - "expected": check.expected, - "actual": raw, - } - - if mismatches: - self.result.status = ExecutionStatus.ERROR - parts = [] - for name, info in mismatches.items(): - path = info.get("path", "") - reason = info.get("reason") - pattern = info.get("pattern") - if reason: - part = f"{name} ({path}): {reason}" - elif pattern is not None: - part = f"{name} ({path}): no entry matching pattern {pattern!r}" - else: - expected = info.get("expected") - actual = cast(Optional[str], info.get("actual")) - part = f"{name} ({path}): expected one of {expected}, actual {repr(actual)}" - parts.append(part) - self.result.message = "Sysfs mismatch: " + "; ".join(parts) - self._log_event( - category=EventCategory.OS, - description="Sysfs mismatch detected", - data=mismatches, - priority=EventPriority.ERROR, - console_log=True, - ) - else: - self._log_event( - category=EventCategory.OS, - description="Sysfs settings match expected", - priority=EventPriority.INFO, - console_log=True, - ) - self.result.status = ExecutionStatus.OK - self.result.message = "Sysfs settings as expected." - - return self.result +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from typing import List, Optional, Union, cast + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import SysSettingsAnalyzerArgs +from .sys_settings_data import SysSettingsDataModel + + +def _get_actual_for_path(data: SysSettingsDataModel, path: str) -> Optional[str]: + """Return the actual value from the data model for the given sysfs path. + + Args: + data: Collected sysfs readings (path -> value). + path: Sysfs path (with or without trailing slash). + + Returns: + Normalized value for that path, or None if not present. + """ + value = data.readings.get(path) or data.readings.get(path.rstrip("/")) + return (value or "").strip().lower() if value is not None else None + + +class SysSettingsAnalyzer(DataAnalyzer[SysSettingsDataModel, SysSettingsAnalyzerArgs]): + """Check sysfs settings against expected values from the checks list.""" + + DATA_MODEL = SysSettingsDataModel + + def analyze_data( + self, data: SysSettingsDataModel, args: Optional[SysSettingsAnalyzerArgs] = None + ) -> TaskResult: + """Compare sysfs data to expected settings from args.checks. + + Args: + data: Collected sysfs readings to check. + args: Analyzer args with checks (path, expected, name). If None or no checks, returns OK. + + Returns: + TaskResult with status OK if all checks pass, ERROR if any mismatch or missing path. + """ + mismatches: dict[str, dict[str, Union[Optional[str], List[str]]]] = {} + + if not args or not args.checks: + self.result.status = ExecutionStatus.OK + self.result.message = "No checks configured." + return self.result + + for check in args.checks: + raw_reading = data.readings.get(check.path) or data.readings.get(check.path.rstrip("/")) + + if check.pattern: + # Directory-listing check: at least one line must match the regex (e.g. ^hsn[0-9]+) + if raw_reading is None: + mismatches[check.name] = { + "path": check.path, + "pattern": check.pattern, + "actual": None, + "reason": "path not collected by this plugin", + } + continue + try: + pat = re.compile(check.pattern) + except re.error: + mismatches[check.name] = { + "path": check.path, + "pattern": check.pattern, + "reason": "invalid regex", + } + continue + lines = [ln.strip() for ln in raw_reading.splitlines() if ln.strip()] + if not any(pat.search(ln) for ln in lines): + mismatches[check.name] = { + "path": check.path, + "pattern": check.pattern, + "actual": lines, + } + continue + + actual = _get_actual_for_path(data, check.path) + if actual is None: + mismatches[check.name] = { + "path": check.path, + "expected": check.expected, + "actual": None, + "reason": "path not collected by this plugin", + } + continue + + if not check.expected: + continue + expected_normalized = [e.strip().lower() for e in check.expected] + if actual not in expected_normalized: + raw = data.readings.get(check.path) or data.readings.get(check.path.rstrip("/")) + mismatches[check.name] = { + "path": check.path, + "expected": check.expected, + "actual": raw, + } + + if mismatches: + self.result.status = ExecutionStatus.ERROR + parts = [] + for name, info in mismatches.items(): + path = info.get("path", "") + reason = info.get("reason") + pattern = info.get("pattern") + if reason: + part = f"{name} ({path}): {reason}" + elif pattern is not None: + part = f"{name} ({path}): no entry matching pattern {pattern!r}" + else: + expected = info.get("expected") + actual = cast(Optional[str], info.get("actual")) + part = f"{name} ({path}): expected one of {expected}, actual {repr(actual)}" + parts.append(part) + self.result.message = "Sysfs mismatch: " + "; ".join(parts) + self._log_event( + category=EventCategory.OS, + description="Sysfs mismatch detected", + data=mismatches, + priority=EventPriority.ERROR, + console_log=True, + ) + else: + self._log_event( + category=EventCategory.OS, + description="Sysfs settings match expected", + priority=EventPriority.INFO, + console_log=True, + ) + self.result.status = ExecutionStatus.OK + self.result.message = "Sysfs settings as expected." + + return self.result diff --git a/nodescraper/plugins/inband/sys_settings/sys_settings_collector.py b/nodescraper/plugins/inband/sys_settings/sys_settings_collector.py index af51a021..6739a6fc 100644 --- a/nodescraper/plugins/inband/sys_settings/sys_settings_collector.py +++ b/nodescraper/plugins/inband/sys_settings/sys_settings_collector.py @@ -1,220 +1,242 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import re -from typing import Optional - -from nodescraper.base import InBandDataCollector -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily -from nodescraper.models import TaskResult - -from .collector_args import SysSettingsCollectorArgs -from .sys_settings_data import SysSettingsDataModel - -BRACKETED_RE = re.compile(r"\[(\w+)\]") - - -def _parse_bracketed_setting(content: str) -> Optional[str]: - """Extract the active setting from sysfs content. - - Args: - content: Raw sysfs file content (e.g. "[always] madvise never"). - - Returns: - The bracketed value if present, else None. - """ - if not content: - return None - match = BRACKETED_RE.search(content.strip()) - return match.group(1).strip() if match else None - - -def _paths_from_args(args: Optional[SysSettingsCollectorArgs]) -> list[str]: - """Extract list of sysfs paths from collection args. - - Args: - args: Collector args containing paths to read, or None. - - Returns: - List of sysfs paths; empty if args is None or args.paths is empty. - """ - if args is None: - return [] - return list(args.paths) if args.paths else [] - - -def _directory_paths_from_args(args: Optional[SysSettingsCollectorArgs]) -> list[str]: - """Extract list of sysfs directory paths to list from collection args.""" - if args is None: - return [] - return list(args.directory_paths) if args.directory_paths else [] - - -def _path_under_sys(path: str) -> Optional[str]: - """Normalize path to the suffix under /sys/ for use in 'cat /sys/{}'. - - Accepts paths like '/sys/kernel/...' or 'kernel/...'. Returns the relative - part (e.g. 'kernel/mm/transparent_hugepage/enabled'). Returns None if path - contains '..' (e.g. /sys/../etc/passwd, /sys/something/../../etc) or is not under /sys. - """ - if ".." in path: - return None - p = path.strip().lstrip("/") - if p.startswith("sys/"): - p = p[4:] - if p.startswith("/"): - return None - return p if p else None - - -def _sysfs_full_path(suffix: str) -> str: - """Return full path /sys/{suffix} for use as readings key.""" - return f"/sys/{suffix}" - - -class SysSettingsCollector(InBandDataCollector[SysSettingsDataModel, SysSettingsCollectorArgs]): - """Collect sysfs settings from user-specified paths.""" - - DATA_MODEL = SysSettingsDataModel - SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.LINUX} - - CMD = "cat /sys/{}" - CMD_LS = "ls -1 /sys/{}" - CMD_LS_LONG = "ls -l /sys/{}" - - def collect_data( - self, args: Optional[SysSettingsCollectorArgs] = None - ) -> tuple[TaskResult, Optional[SysSettingsDataModel]]: - """Collect sysfs values for each path in args.paths. - - Args: - args: Collector args with paths to read; if None or empty paths, returns NOT_RAN. - - Returns: - Tuple of (TaskResult, SysSettingsDataModel or None). Data is None on NOT_RAN or ERROR. - """ - if self.system_info.os_family != OSFamily.LINUX: - self._log_event( - category=EventCategory.OS, - description="Sysfs collection is only supported on Linux.", - priority=EventPriority.WARNING, - console_log=True, - ) - return self.result, None - - paths = _paths_from_args(args) - directory_paths = _directory_paths_from_args(args) - if directory_paths: - self._log_event( - category=EventCategory.OS, - description=f"Sysfs directory_paths to list: {directory_paths}", - data={"directory_paths": directory_paths}, - priority=EventPriority.INFO, - console_log=True, - ) - if not paths and not directory_paths: - self.result.message = "No paths configured for sysfs collection" - self.result.status = ExecutionStatus.NOT_RAN - return self.result, None - - readings: dict[str, str] = {} - for path in paths: - suffix = _path_under_sys(path) - if suffix is None: - self._log_event( - category=EventCategory.OS, - description=f"Skipping path not under /sys or invalid: {path!r}", - data={"path": path}, - priority=EventPriority.WARNING, - console_log=True, - ) - continue - full_path = _sysfs_full_path(suffix) - if "*" in suffix: - cmd = self.CMD_LS_LONG.format(suffix) - res = self._run_sut_cmd(f"bash -c {cmd!r}", sudo=False) - if res.exit_code == 0: - readings[full_path] = res.stdout.strip() if res.stdout else "" - else: - self._log_event( - category=EventCategory.OS, - description=f"Failed to run ls -l for sysfs path: {full_path}", - data={"exit_code": res.exit_code}, - priority=EventPriority.WARNING, - console_log=True, - ) - else: - res = self._run_sut_cmd(self.CMD.format(suffix), sudo=False) - if res.exit_code == 0 and res.stdout: - value = _parse_bracketed_setting(res.stdout) or res.stdout.strip() - readings[full_path] = value - else: - self._log_event( - category=EventCategory.OS, - description=f"Failed to read sysfs path: {full_path}", - data={"exit_code": res.exit_code}, - priority=EventPriority.WARNING, - console_log=True, - ) - - for path in directory_paths: - suffix = _path_under_sys(path) - if suffix is None: - self._log_event( - category=EventCategory.OS, - description=f"Skipping directory path not under /sys or invalid: {path!r}", - data={"path": path}, - priority=EventPriority.WARNING, - console_log=True, - ) - continue - full_path = _sysfs_full_path(suffix) - res = self._run_sut_cmd(self.CMD_LS.format(suffix), sudo=False) - if res.exit_code == 0: - readings[full_path] = res.stdout.strip() if res.stdout else "" - else: - self._log_event( - category=EventCategory.OS, - description=f"Failed to list sysfs path: {full_path}", - data={"exit_code": res.exit_code}, - priority=EventPriority.WARNING, - console_log=True, - ) - - if not readings: - self.result.message = "Sysfs settings not read" - self.result.status = ExecutionStatus.ERROR - return self.result, None - - sys_settings_data = SysSettingsDataModel(readings=readings) - self._log_event( - category=EventCategory.OS, - description="Sysfs settings collected", - data=sys_settings_data.model_dump(), - priority=EventPriority.INFO, - ) - self.result.message = f"Sysfs collected {len(readings)} path(s)" - self.result.status = ExecutionStatus.OK - return self.result, sys_settings_data +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from typing import Optional + +from nodescraper.base import InBandDataCollector +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus, OSFamily +from nodescraper.models import TaskResult + +from .collector_args import SysSettingsCollectorArgs +from .sys_settings_data import SysSettingsDataModel + +BRACKETED_RE = re.compile(r"\[(\w+)\]") + + +def _parse_bracketed_setting(content: str) -> Optional[str]: + """Extract the active setting from sysfs content. + + Args: + content: Raw sysfs file content (e.g. "[always] madvise never"). + + Returns: + The bracketed value if present, else None. + """ + if not content: + return None + match = BRACKETED_RE.search(content.strip()) + return match.group(1).strip() if match else None + + +def _paths_from_args(args: Optional[SysSettingsCollectorArgs]) -> list[str]: + """Extract list of sysfs paths from collection args. + + Args: + args: Collector args containing paths to read, or None. + + Returns: + List of sysfs paths; empty if args is None or args.paths is empty. + """ + if args is None: + return [] + return list(args.paths) if args.paths else [] + + +def _directory_paths_from_args(args: Optional[SysSettingsCollectorArgs]) -> list[str]: + """Extract list of sysfs directory paths to list from collection args.""" + if args is None: + return [] + return list(args.directory_paths) if args.directory_paths else [] + + +def _path_under_sys(path: str) -> Optional[str]: + """Normalize path to the suffix under /sys/ for use in 'cat /sys/{}'. + + Accepts paths like '/sys/kernel/...' or 'kernel/...'. Returns the relative + part (e.g. 'kernel/mm/transparent_hugepage/enabled'). Returns None if path + contains '..' (e.g. /sys/../etc/passwd, /sys/something/../../etc) or is not under /sys. + """ + if ".." in path: + return None + p = path.strip().lstrip("/") + if p.startswith("sys/"): + p = p[4:] + if p.startswith("/"): + return None + return p if p else None + + +def _sysfs_full_path(suffix: str) -> str: + """Return full path /sys/{suffix} for use as readings key.""" + return f"/sys/{suffix}" + + +def _collapse_glob_cat(stdout: str) -> str: + """Keep unique non-empty lines; store a single value when every match is identical.""" + lines = [line.strip() for line in (stdout or "").splitlines() if line.strip()] + unique = list(dict.fromkeys(lines)) + if not unique: + return "" + if len(unique) == 1: + return unique[0] + return "\n".join(unique) + + +class SysSettingsCollector(InBandDataCollector[SysSettingsDataModel, SysSettingsCollectorArgs]): + """Collect sysfs settings from user-specified paths.""" + + DATA_MODEL = SysSettingsDataModel + SUPPORTED_OS_FAMILY: set[OSFamily] = {OSFamily.LINUX} + + CMD = "cat /sys/{}" + CMD_LS = "ls -1 /sys/{}" + CMD_LS_LONG = "ls -l /sys/{}" + + def collect_data( + self, args: Optional[SysSettingsCollectorArgs] = None + ) -> tuple[TaskResult, Optional[SysSettingsDataModel]]: + """Collect sysfs values for each path in args.paths. + + Glob paths (containing '*') are read with ``bash -c 'cat /sys/...'`` first. + Identical cat lines are collapsed to one value. If cat fails, ``ls -l`` is used. + + Args: + args: Collector args with paths to read; if None or empty paths, returns NOT_RAN. + + Returns: + Tuple of (TaskResult, SysSettingsDataModel or None). Data is None on NOT_RAN or ERROR. + """ + if self.system_info.os_family != OSFamily.LINUX: + self._log_event( + category=EventCategory.OS, + description="Sysfs collection is only supported on Linux.", + priority=EventPriority.WARNING, + console_log=True, + ) + return self.result, None + + paths = _paths_from_args(args) + directory_paths = _directory_paths_from_args(args) + if directory_paths: + self._log_event( + category=EventCategory.OS, + description=f"Sysfs directory_paths to list: {directory_paths}", + data={"directory_paths": directory_paths}, + priority=EventPriority.INFO, + console_log=True, + ) + if not paths and not directory_paths: + self.result.message = "No paths configured for sysfs collection" + self.result.status = ExecutionStatus.NOT_RAN + return self.result, None + + readings: dict[str, str] = {} + for path in paths: + suffix = _path_under_sys(path) + if suffix is None: + self._log_event( + category=EventCategory.OS, + description=f"Skipping path not under /sys or invalid: {path!r}", + data={"path": path}, + priority=EventPriority.WARNING, + console_log=True, + ) + continue + full_path = _sysfs_full_path(suffix) + if "*" in suffix: + cat_cmd = self.CMD.format(suffix) + res = self._run_sut_cmd(f"bash -c {cat_cmd!r}", sudo=False) + if res.exit_code == 0 and res.stdout and res.stdout.strip(): + value = _collapse_glob_cat(res.stdout) + if "\n" not in value: + value = _parse_bracketed_setting(value) or value + readings[full_path] = value + continue + cmd = self.CMD_LS_LONG.format(suffix) + res = self._run_sut_cmd(f"bash -c {cmd!r}", sudo=False) + if res.exit_code == 0: + readings[full_path] = res.stdout.strip() if res.stdout else "" + else: + self._log_event( + category=EventCategory.OS, + description=f"Failed to read sysfs path: {full_path}", + data={"exit_code": res.exit_code}, + priority=EventPriority.WARNING, + console_log=True, + ) + else: + res = self._run_sut_cmd(self.CMD.format(suffix), sudo=False) + if res.exit_code == 0 and res.stdout: + value = _parse_bracketed_setting(res.stdout) or res.stdout.strip() + readings[full_path] = value + else: + self._log_event( + category=EventCategory.OS, + description=f"Failed to read sysfs path: {full_path}", + data={"exit_code": res.exit_code}, + priority=EventPriority.WARNING, + console_log=True, + ) + + for path in directory_paths: + suffix = _path_under_sys(path) + if suffix is None: + self._log_event( + category=EventCategory.OS, + description=f"Skipping directory path not under /sys or invalid: {path!r}", + data={"path": path}, + priority=EventPriority.WARNING, + console_log=True, + ) + continue + full_path = _sysfs_full_path(suffix) + res = self._run_sut_cmd(self.CMD_LS.format(suffix), sudo=False) + if res.exit_code == 0: + readings[full_path] = res.stdout.strip() if res.stdout else "" + else: + self._log_event( + category=EventCategory.OS, + description=f"Failed to list sysfs path: {full_path}", + data={"exit_code": res.exit_code}, + priority=EventPriority.WARNING, + console_log=True, + ) + + if not readings: + self.result.message = "Sysfs settings not read" + self.result.status = ExecutionStatus.ERROR + return self.result, None + + sys_settings_data = SysSettingsDataModel(readings=readings) + self._log_event( + category=EventCategory.OS, + description="Sysfs settings collected", + data=sys_settings_data.model_dump(), + priority=EventPriority.INFO, + ) + self.result.message = f"Sysfs collected {len(readings)} path(s)" + self.result.status = ExecutionStatus.OK + return self.result, sys_settings_data diff --git a/nodescraper/plugins/inband/sys_settings/sys_settings_data.py b/nodescraper/plugins/inband/sys_settings/sys_settings_data.py index acec4b4f..eadc1216 100644 --- a/nodescraper/plugins/inband/sys_settings/sys_settings_data.py +++ b/nodescraper/plugins/inband/sys_settings/sys_settings_data.py @@ -1,38 +1,38 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from pydantic import Field - -from nodescraper.models import DataModel - - -class SysSettingsDataModel(DataModel): - """Data model for sysfs settings: path -> parsed value. - - Values are parsed from user-specified sysfs paths (bracketed value extracted - when present, e.g. '[always] madvise never' -> 'always'). - """ - - readings: dict[str, str] = Field(default_factory=dict) # sysfs path (as given) -> parsed value +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import DataModel + + +class SysSettingsDataModel(DataModel): + """Data model for sysfs settings: path -> parsed value. + + Values are parsed from user-specified sysfs paths (bracketed value extracted + when present, e.g. '[always] madvise never' -> 'always'). + """ + + readings: dict[str, str] = Field(default_factory=dict) # sysfs path (as given) -> parsed value diff --git a/nodescraper/plugins/inband/sys_settings/sys_settings_plugin.py b/nodescraper/plugins/inband/sys_settings/sys_settings_plugin.py index 158ac6ff..be81b10a 100644 --- a/nodescraper/plugins/inband/sys_settings/sys_settings_plugin.py +++ b/nodescraper/plugins/inband/sys_settings/sys_settings_plugin.py @@ -1,44 +1,44 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from nodescraper.base import InBandDataPlugin - -from .analyzer_args import SysSettingsAnalyzerArgs -from .collector_args import SysSettingsCollectorArgs -from .sys_settings_analyzer import SysSettingsAnalyzer -from .sys_settings_collector import SysSettingsCollector -from .sys_settings_data import SysSettingsDataModel - - -class SysSettingsPlugin( - InBandDataPlugin[SysSettingsDataModel, SysSettingsCollectorArgs, SysSettingsAnalyzerArgs] -): - """Plugin to collect and analyze sysfs settings from user-specified paths.""" - - DATA_MODEL = SysSettingsDataModel - COLLECTOR = SysSettingsCollector - ANALYZER = SysSettingsAnalyzer - COLLECTOR_ARGS = SysSettingsCollectorArgs - ANALYZER_ARGS = SysSettingsAnalyzerArgs +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.base import InBandDataPlugin + +from .analyzer_args import SysSettingsAnalyzerArgs +from .collector_args import SysSettingsCollectorArgs +from .sys_settings_analyzer import SysSettingsAnalyzer +from .sys_settings_collector import SysSettingsCollector +from .sys_settings_data import SysSettingsDataModel + + +class SysSettingsPlugin( + InBandDataPlugin[SysSettingsDataModel, SysSettingsCollectorArgs, SysSettingsAnalyzerArgs] +): + """Plugin to collect and analyze sysfs settings from user-specified paths.""" + + DATA_MODEL = SysSettingsDataModel + COLLECTOR = SysSettingsCollector + ANALYZER = SysSettingsAnalyzer + COLLECTOR_ARGS = SysSettingsCollectorArgs + ANALYZER_ARGS = SysSettingsAnalyzerArgs diff --git a/nodescraper/plugins/ooband/__init__.py b/nodescraper/plugins/ooband/__init__.py index b75b2eb0..e83d846e 100644 --- a/nodescraper/plugins/ooband/__init__.py +++ b/nodescraper/plugins/ooband/__init__.py @@ -1,26 +1,26 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Out-of-band (OOB) plugins: Redfish and other BMC/remote management plugins.""" +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Out-of-band (OOB) plugins: Redfish and other BMC/remote management plugins.""" diff --git a/nodescraper/plugins/ooband/amc_redfish_diag/__init__.py b/nodescraper/plugins/ooband/amc_redfish_diag/__init__.py new file mode 100644 index 00000000..2c342f0d --- /dev/null +++ b/nodescraper/plugins/ooband/amc_redfish_diag/__init__.py @@ -0,0 +1,17 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +############################################################################### +from .amc_diag_collector import AmcRedfishDiagCollector +from .amc_diag_plugin import AmcRedfishDiagPlugin +from .collector_args import AmcDiagCollectionSpec, AmcRedfishDiagCollectorArgs + +__all__ = [ + "AmcDiagCollectionSpec", + "AmcRedfishDiagCollector", + "AmcRedfishDiagCollectorArgs", + "AmcRedfishDiagPlugin", +] diff --git a/nodescraper/plugins/ooband/amc_redfish_diag/amc_diag_collector.py b/nodescraper/plugins/ooband/amc_redfish_diag/amc_diag_collector.py new file mode 100644 index 00000000..09425b85 --- /dev/null +++ b/nodescraper/plugins/ooband/amc_redfish_diag/amc_diag_collector.py @@ -0,0 +1,275 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pathlib import Path +from typing import Optional + +from nodescraper.base import RedfishDataCollector +from nodescraper.connection.redfish import collect_oem_diagnostic_data +from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogResult, + _parse_clear_endpoint, + clear_redfish_logs, +) +from nodescraper.connection.redfish.redfish_constants import RF_MEMBERS, RF_ODATA_ID +from nodescraper.enums import ( + EventCategory, + EventPriority, + ExecutionStatus, + SystemInteractionLevel, +) +from nodescraper.models import TaskResult +from nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_data import ( + OemDiagTypeResult, + RedfishOemDiagDataModel, +) + +from .collector_args import AmcDiagCollectionSpec, AmcRedfishDiagCollectorArgs + +_COLLECT_ACTION_KEYS = ( + "LogService.CollectDiagnosticData", + "#LogService.CollectDiagnosticData", +) + + +def _collection_key(spec: AmcDiagCollectionSpec) -> str: + """Build a stable results-dict key for one CollectDiagnosticData job. + + Args: + spec: Collection request. + + Returns: + Root, diagnostic type, and optional OEM type joined by colons. + """ + if spec.oem_data_type: + return f"{spec.root}:{spec.diagnostic_data_type}:{spec.oem_data_type}" + return f"{spec.root}:{spec.diagnostic_data_type}" + + +class AmcRedfishDiagCollector( + RedfishDataCollector[RedfishOemDiagDataModel, AmcRedfishDiagCollectorArgs] +): + """Collect AMC Manager and Systems diagnostic bundles through SSH-proxy Redfish.""" + + DATA_MODEL = RedfishOemDiagDataModel + + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "SSH-proxy Redfish GET of Managers/Systems LogServices that advertise CollectDiagnosticData.", + "CollectDiagnosticData for each collection_args.collections entry (Manager dump and OEM AllLogs by default).", + "Optional binary archives under the plugin log path when log_path is set.", + ) + + def collect_data( + self, args: Optional[AmcRedfishDiagCollectorArgs] = None + ) -> tuple[TaskResult, Optional[RedfishOemDiagDataModel]]: + """Discover AMC log services and run CollectDiagnosticData jobs. + + Args: + args: Collection args. Defaults to the AMC manager dump plus systems AllLogs. + + Returns: + TaskResult and RedfishOemDiagDataModel keyed by collection spec. + """ + if args is None: + args = AmcRedfishDiagCollectorArgs() + jobs = list(args.collections) if args.collections else [] + if not jobs: + self.result.message = "No AMC diagnostic collections configured" + self.result.status = ExecutionStatus.NOT_RAN + return self.result, None + + if self.log_path: + output_dir = (Path(self.log_path) / "diag_logs").resolve() + self.logger.info( + "(AmcRedfishDiagPlugin) Diagnostic archives will be written to: %s", + output_dir, + ) + else: + output_dir = None + + results: dict[str, OemDiagTypeResult] = {} + successful_log_services: list[str] = [] + for spec in jobs: + key = _collection_key(spec) + log_service = self._find_log_service(spec.root, args) + if not log_service: + missing_err = f"No CollectDiagnosticData LogService under {spec.root}" + self._log_event( + category=EventCategory.RUNTIME, + description=f"AMC diag {key}: {missing_err}", + priority=EventPriority.WARNING, + console_log=True, + ) + results[key] = OemDiagTypeResult(success=False, error=missing_err, metadata=None) + continue + _log_bytes, metadata, collect_err = collect_oem_diagnostic_data( + self.connection, + log_service_path=log_service, + oem_diagnostic_type=spec.oem_data_type or None, + diagnostic_data_type=spec.diagnostic_data_type, + task_timeout_s=args.task_timeout_s, + output_dir=output_dir, + logger=self.logger, + ) + if collect_err: + self._log_event( + category=EventCategory.RUNTIME, + description=f"AMC diag {key}: error", + priority=EventPriority.WARNING, + data={"response_body": collect_err}, + console_log=True, + ) + results[key] = OemDiagTypeResult(success=False, error=collect_err, metadata=None) + else: + results[key] = OemDiagTypeResult(success=True, error=None, metadata=metadata) + if log_service not in successful_log_services: + successful_log_services.append(log_service) + + success_count = sum(1 for r in results.values() if r.success) + self.result.message = f"AMC diag: {success_count}/{len(results)} collections succeeded" + if success_count == len(results): + self.result.status = ExecutionStatus.OK + elif success_count: + self.result.status = ExecutionStatus.WARNING + else: + self.result.status = ExecutionStatus.ERROR + + if ( + args.clear_logs_after_collection + and successful_log_services + and self.system_interaction_level >= SystemInteractionLevel.INTERACTIVE + ): + self._clear_log_stores(successful_log_services) + elif args.clear_logs_after_collection and not successful_log_services: + self.logger.debug("Skipping log clear: no AMC diag collections succeeded") + elif args.clear_logs_after_collection: + self.logger.debug( + "Skipping log clear: system_interaction_level is %s (requires INTERACTIVE)", + self.system_interaction_level.name, + ) + + return self.result, RedfishOemDiagDataModel(results=results) + + def _clear_log_stores(self, log_service_paths: list[str]) -> None: + """Clear only the log services whose collection succeeded. + + GETs each log service path, checks for a ClearLog action, and POSTs to it. + Failures downgrade the overall result to WARNING. + + Args: + log_service_paths: Deduplicated Redfish paths for successfully collected + log services. + """ + all_cleared: list[ClearLogResult] = [] + all_failed: list[ClearLogResult] = [] + + for log_service_path in log_service_paths: + svc_res = self._run_redfish_get(log_service_path, log_artifact=False) + if not svc_res.success or not svc_res.data: + self.logger.debug( + "Skipping log clear: could not GET log service %s", log_service_path + ) + continue + endpoint = _parse_clear_endpoint(svc_res.data) + if endpoint is None: + self.logger.debug( + "LogService %s does not advertise ClearLog action", log_service_path + ) + continue + clear_results = clear_redfish_logs(self.connection, [endpoint], self.logger) + for r in clear_results: + if r.success: + all_cleared.append(r) + else: + all_failed.append(r) + + if all_cleared: + self.result.message += f"; cleared {len(all_cleared)} log store(s)" + if all_failed: + descriptions = "; ".join(f"{r.path}: {r.error}" for r in all_failed) + self._log_event( + category=EventCategory.RUNTIME, + description=f"Failed to clear {len(all_failed)} log store(s): {descriptions}", + priority=EventPriority.WARNING, + console_log=True, + ) + if self.result.status == ExecutionStatus.OK: + self.result.status = ExecutionStatus.WARNING + self.result.message += f"; {len(all_failed)} log store(s) failed to clear" + + def _member_paths(self, root: str, args: AmcRedfishDiagCollectorArgs) -> list[str]: + """Resolve Systems or Managers member URIs to probe. + + Args: + root: Managers or Systems. + args: Collection args with optional member Ids. + + Returns: + Redfish paths without a leading slash. + """ + api_root = (getattr(self.connection, "api_root", None) or "redfish/v1").strip("/") + ids = args.manager_ids if root == "Managers" else args.system_ids + if ids: + return [f"{api_root}/{root}/{member_id}" for member_id in ids] + listing = self._run_redfish_get(f"/{api_root}/{root}") + if not listing.success or not listing.data: + return [] + paths: list[str] = [] + for member in listing.data.get(RF_MEMBERS) or []: + if not isinstance(member, dict): + continue + odata_id = member.get(RF_ODATA_ID) + if isinstance(odata_id, str) and odata_id.strip(): + paths.append(odata_id.strip().lstrip("/")) + return paths + + def _find_log_service(self, root: str, args: AmcRedfishDiagCollectorArgs) -> Optional[str]: + """Find the first LogService under root that advertises CollectDiagnosticData. + + Args: + root: Managers or Systems. + args: Collection args with optional member Ids. + + Returns: + LogService path, or None if none found. + """ + for member_path in self._member_paths(root, args): + ls_list = self._run_redfish_get(f"/{member_path}/LogServices") + if not ls_list.success or not ls_list.data: + continue + for member in ls_list.data.get(RF_MEMBERS) or []: + if not isinstance(member, dict): + continue + odata_id = member.get(RF_ODATA_ID) + if not isinstance(odata_id, str) or not odata_id.strip(): + continue + ls_path = odata_id.strip().lstrip("/") + ls_body = self._run_redfish_get(f"/{ls_path}") + if not ls_body.success or not ls_body.data: + continue + actions = ls_body.data.get("Actions") or {} + if any(key in actions for key in _COLLECT_ACTION_KEYS): + return ls_path + return None diff --git a/nodescraper/plugins/ooband/amc_redfish_diag/amc_diag_plugin.py b/nodescraper/plugins/ooband/amc_redfish_diag/amc_diag_plugin.py new file mode 100644 index 00000000..8804b018 --- /dev/null +++ b/nodescraper/plugins/ooband/amc_redfish_diag/amc_diag_plugin.py @@ -0,0 +1,61 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.connection.redfish import ( + RedfishSshProxyConnectionManager, + RedfishSshProxyConnectionParams, +) +from nodescraper.interfaces import DataPlugin +from nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_data import ( + RedfishOemDiagDataModel, +) + +from .amc_diag_collector import AmcRedfishDiagCollector +from .collector_args import AmcRedfishDiagCollectorArgs + + +class AmcRedfishDiagPlugin( + DataPlugin[ + RedfishSshProxyConnectionManager, + RedfishSshProxyConnectionParams, + RedfishOemDiagDataModel, + AmcRedfishDiagCollectorArgs, + None, + ] +): + """AMC CollectDiagnosticData over SSH-proxy Redfish for manager and system diagnostic bundles. + + Collection only, there is no analyzer. Each failed collection logs a WARNING event and the + collector reports ERROR when no collection succeeded, OK when at least one succeeded. + + Configure RedfishSshProxyConnectionManager: ssh to the BMC, host/port of the AMC Redfish + URL reachable from that BMC. A targets list collects once per BMC. collection_args selects + Managers Manager dump and Systems OEM AllLogs. + """ + + CONNECTION_TYPE = RedfishSshProxyConnectionManager + DATA_MODEL = RedfishOemDiagDataModel + COLLECTOR = AmcRedfishDiagCollector + COLLECTOR_ARGS = AmcRedfishDiagCollectorArgs diff --git a/nodescraper/plugins/ooband/amc_redfish_diag/collector_args.py b/nodescraper/plugins/ooband/amc_redfish_diag/collector_args.py new file mode 100644 index 00000000..751dc32f --- /dev/null +++ b/nodescraper/plugins/ooband/amc_redfish_diag/collector_args.py @@ -0,0 +1,94 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from pydantic import BaseModel, Field + +from nodescraper.models import CollectorArgs + +DEFAULT_TASK_TIMEOUT_S = 1800 + + +class AmcDiagCollectionSpec(BaseModel): + """One CollectDiagnosticData request for an AMC telemetry bundle.""" + + root: str = Field( + description="Redfish root collection to search: Managers or Systems.", + ) + diagnostic_data_type: str = Field( + description="DMTF DiagnosticDataType (Manager, OEM, and similar).", + ) + oem_data_type: str = Field( + default="", + description="OEMDiagnosticDataType when diagnostic_data_type is OEM.", + ) + + +def _default_collections() -> list[AmcDiagCollectionSpec]: + """Return the AMC manager dump plus systems AllLogs jobs. + + Returns: + Default CollectDiagnosticData specs. + """ + return [ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + AmcDiagCollectionSpec( + root="Systems", + diagnostic_data_type="OEM", + oem_data_type="AllLogs", + ), + ] + + +class AmcRedfishDiagCollectorArgs(CollectorArgs): + """Collector args for AMC Redfish diagnostic dumps via SSH-proxy curl.""" + + manager_ids: list[str] = Field( + default_factory=lambda: ["AMC"], + description="Manager member Ids to probe; empty walks the Managers collection.", + ) + system_ids: list[str] = Field( + default_factory=lambda: ["MI450", "Accelerators"], + description="System member Ids to probe; empty walks the Systems collection.", + ) + collections: list[AmcDiagCollectionSpec] = Field( + default_factory=_default_collections, + description="CollectDiagnosticData jobs to run (Managers manager dump then Systems OEM AllLogs).", + ) + task_timeout_s: int = Field( + default=DEFAULT_TASK_TIMEOUT_S, + ge=1, + le=3600, + description="Max seconds to wait for each CollectDiagnosticData task.", + ) + clear_logs_after_collection: bool = Field( + default=False, + description=( + "If True, discover and POST LogService.ClearLog to all log stores under the " + "configured Systems/Managers members after a successful collection. " + "Requires system_interaction_level >= INTERACTIVE." + ), + ) diff --git a/nodescraper/plugins/ooband/redfish_endpoint/__init__.py b/nodescraper/plugins/ooband/redfish_endpoint/__init__.py index 84293e72..6c769742 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/__init__.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/__init__.py @@ -1,40 +1,41 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from .analyzer_args import RedfishEndpointAnalyzerArgs -from .collector_args import RedfishEndpointCollectorArgs -from .endpoint_analyzer import RedfishEndpointAnalyzer -from .endpoint_collector import RedfishEndpointCollector -from .endpoint_data import RedfishEndpointDataModel -from .endpoint_plugin import RedfishEndpointPlugin - -__all__ = [ - "RedfishEndpointAnalyzer", - "RedfishEndpointAnalyzerArgs", - "RedfishEndpointCollector", - "RedfishEndpointCollectorArgs", - "RedfishEndpointDataModel", - "RedfishEndpointPlugin", -] +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from .analyzer_args import RedfishEndpointAnalyzerArgs +from .collector_args import RedfishEndpointCollectorArgs +from .endpoint_analyzer import RedfishEndpointAnalyzer +from .endpoint_collector import RedfishEndpointCollector +from .endpoint_data import RedfishEndpointDataModel +from .endpoint_plugin import AmcRedfishEndpointPlugin, RedfishEndpointPlugin + +__all__ = [ + "AmcRedfishEndpointPlugin", + "RedfishEndpointAnalyzer", + "RedfishEndpointAnalyzerArgs", + "RedfishEndpointCollector", + "RedfishEndpointCollectorArgs", + "RedfishEndpointDataModel", + "RedfishEndpointPlugin", +] diff --git a/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py b/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py index 5bb5acf9..5b495bb5 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/analyzer_args.py @@ -1,67 +1,71 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from enum import Enum -from typing import Any, Union - -from pydantic import Field - -from nodescraper.models import AnalyzerArgs - - -class ConstraintKey(str, Enum): - """Keys used in Redfish constraint dicts (e.g. in analyzer checks config). - - Naming aligns with JSON Schema combining: anyOf = value must match any of the list (OR). - oneOf in JSON Schema means exactly one (XOR); we use anyOf for \"value in allowed list\". - """ - - EQ = "eq" - MIN = "min" - MAX = "max" - ANY_OF = "anyOf" - - -RedfishConstraint = Union[int, float, str, bool, dict[str, Any]] - - -class RedfishEndpointAnalyzerArgs(AnalyzerArgs): - """Analyzer args for config-driven Redfish checks.""" - - checks: dict[str, dict[str, RedfishConstraint]] = Field( - default_factory=dict, - description=( - "Map: URI or `*` -> { property_path: constraint }. " - "URI keys must match a key in the collected responses (exact match). " - "Use `*` as the key to apply the inner constraints to every collected response body. " - "Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. " - "Constraints: " - "'eq' — value must equal the given literal (int, float, str, bool). " - "'min' — value must be numeric and >= the given number. " - "'max' — value must be numeric and <= the given number. " - "'anyOf' — value must be in the given list (OR; any match passes). " - 'Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "*": { "Status/Health": { "anyOf": ["OK"] } } }.' - ), - ) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from enum import Enum +from typing import Any, Union + +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class ConstraintKey(str, Enum): + """Keys used in Redfish constraint dicts (e.g. in analyzer checks config). + + Naming aligns with JSON Schema combining: anyOf = value must match any of the list (OR). + oneOf in JSON Schema means exactly one (XOR); we use anyOf for \"value in allowed list\". + """ + + EQ = "eq" + MIN = "min" + MAX = "max" + ANY_OF = "anyOf" + REGEX = "regex" + EXISTS = "exists" + + +RedfishConstraint = Union[int, float, str, bool, dict[str, Any]] + + +class RedfishEndpointAnalyzerArgs(AnalyzerArgs): + """Analyzer args for config-driven Redfish checks.""" + + checks: dict[str, dict[str, RedfishConstraint]] = Field( + default_factory=dict, + description=( + "Map: URI or `*` -> { property_path: constraint }. " + "URI keys must match a key in the collected responses (exact match). " + "Use `*` as the key to apply the inner constraints to every collected response body. " + "Property paths use '/' for nesting and indices, e.g. 'Status/Health', 'PowerControl/0/PowerConsumedWatts'. " + "Constraints: " + "'eq' — value must equal the given literal (int, float, str, bool). " + "'min' — value must be numeric and >= the given number. " + "'max' — value must be numeric and <= the given number. " + "'anyOf' — value must be in the given list (OR; any match passes). " + "'regex' — value must be a string matching the pattern (Python re.search; anchor with ^ and $ for a full match). " + "'exists' — true: property must be present and not null; false: property must be absent or null. " + 'Example: { "/redfish/v1/Systems/1": { "Status/Health": { "anyOf": ["OK", "Warning"] }, "PowerState": "On" }, "*": { "Status/Health": { "anyOf": ["OK"] } } }.' + ), + ) diff --git a/nodescraper/plugins/ooband/redfish_endpoint/collector_args.py b/nodescraper/plugins/ooband/redfish_endpoint/collector_args.py index 6583075e..b4a38aba 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/collector_args.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/collector_args.py @@ -42,7 +42,7 @@ class RedfishEndpointCollectorArgs(CollectorArgs): tree_max_depth: int = Field( default=2, ge=1, - le=10, + le=20, description="When discover_tree is True: max traversal depth (1=service root only, 2=root + collections, 3=+ members).", ) tree_max_endpoints: int = Field( @@ -51,6 +51,21 @@ class RedfishEndpointCollectorArgs(CollectorArgs): le=10_000, description="When discover_tree is True: max endpoints to discover (0=no limit).", ) + tree_max_collection_pages: int = Field( + default=1, + ge=1, + le=10_000, + description="When discover_tree is True: max pages to collect for each paginated collection.", + ) + tree_max_log_entry_members: int = Field( + default=0, + ge=0, + le=10_000, + description=( + "When discover_tree is True: max individual LogService entries to collect per collection. " + "Zero keeps every referenced entry within the depth limit." + ), + ) max_workers: int = Field( default=1, ge=1, @@ -71,6 +86,15 @@ class RedfishEndpointCollectorArgs(CollectorArgs): description="When follow_next_link is True: safety cap on the number of pages to follow per URI (default 200).", ) + clear_log_uris: list[str] = Field( + default_factory=list, + description=( + "Redfish LogService.ClearLog action URIs to POST {} to after successful collection. " + "Each URI is posted with an empty JSON body. " + "Requires system_interaction_level >= INTERACTIVE." + ), + ) + @field_validator("uris", mode="before") @classmethod def strip_uris(cls, v: list[str]) -> list[str]: diff --git a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py index 26412f94..d0e51b1b 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_analyzer.py @@ -1,151 +1,165 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Any, Optional - -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.interfaces import DataAnalyzer -from nodescraper.models import TaskResult - -from .analyzer_args import ConstraintKey, RedfishConstraint, RedfishEndpointAnalyzerArgs -from .endpoint_data import RedfishEndpointDataModel - - -def _get_by_path(obj: Any, path: str) -> Any: - """Get a value from a nested dict/list.""" - if not path.strip(): - return obj - current: Any = obj - for segment in path.strip().split("/"): - if not segment: - continue - if current is None: - return None - if isinstance(current, list): - try: - idx = int(segment) - current = current[idx] if 0 <= idx < len(current) else None - except ValueError: - return None - elif isinstance(current, dict): - current = current.get(segment) - else: - return None - return current - - -def _check_constraint(actual: Any, constraint: RedfishConstraint) -> tuple[bool, str]: - """Compare actual value to constraint.""" - if isinstance(constraint, dict): - if ConstraintKey.EQ in constraint: - ok = actual == constraint[ConstraintKey.EQ] - return ok, f"expected eq {constraint[ConstraintKey.EQ]}, got {actual!r}" - if ConstraintKey.MIN in constraint or ConstraintKey.MAX in constraint: - try: - val = float(actual) if actual is not None else None - if val is None: - return False, f"expected numeric, got {actual!r}" - if ConstraintKey.MIN in constraint and val < constraint[ConstraintKey.MIN]: - return False, f"value {val} below min {constraint[ConstraintKey.MIN]}" - if ConstraintKey.MAX in constraint and val > constraint[ConstraintKey.MAX]: - return False, f"value {val} above max {constraint[ConstraintKey.MAX]}" - return True, "" - except (TypeError, ValueError): - return False, f"expected numeric, got {actual!r}" - if ConstraintKey.ANY_OF in constraint: - allowed = constraint[ConstraintKey.ANY_OF] - if not isinstance(allowed, list): - return False, "anyOf must be a list" - ok = actual in allowed - return ok, f"expected any of {allowed}, got {actual!r}" - ok = actual == constraint - return ok, f"expected {constraint!r}, got {actual!r}" - - -class RedfishEndpointAnalyzer(DataAnalyzer[RedfishEndpointDataModel, RedfishEndpointAnalyzerArgs]): - """Checks Redfish endpoint responses against configured thresholds and key/value rules.""" - - DATA_MODEL = RedfishEndpointDataModel - - DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( - "For each entry in analysis_args.checks, reads JSON paths in collected responses and " - "compares values to constraints (eq, min/max, anyOf, regex, etc.).", - "URI key `*` runs checks against every collected response body.", - ) - - def analyze_data( - self, - data: RedfishEndpointDataModel, - args: Optional[RedfishEndpointAnalyzerArgs] = None, - ) -> TaskResult: - """Evaluate each configured check against the collected Redfish responses.""" - if not args or not args.checks: - self.result.status = ExecutionStatus.OK - self.result.message = "No checks configured" - return self.result - - failed: list[dict[str, Any]] = [] - for uri, path_constraints in args.checks.items(): - if uri == "*": - bodies = list(data.responses.values()) - else: - body = data.responses.get(uri) - bodies = [body] if body is not None else [] - if not bodies: - if uri != "*": - failed.append( - {"uri": uri, "path": None, "reason": "URI not in collected responses"} - ) - continue - for resp in bodies: - for path, constraint in path_constraints.items(): - actual = _get_by_path(resp, path) - ok, msg = _check_constraint(actual, constraint) - if not ok: - failed.append( - { - "uri": uri, - "path": path, - "expected": constraint, - "actual": actual, - "reason": msg, - } - ) - - if failed: - description = f"Redfish endpoint checks failed: {len(failed)} failure(s)" - self._log_event( - category=EventCategory.TELEMETRY, - description=description, - data={"failures": failed}, - priority=EventPriority.WARNING, - console_log=True, - ) - self.result.status = ExecutionStatus.ERROR - self.result.message = f"{len(failed)} check(s) failed" - else: - self.result.status = ExecutionStatus.OK - self.result.message = "All Redfish endpoint checks passed" - return self.result +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from typing import Any, Optional + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import ConstraintKey, RedfishConstraint, RedfishEndpointAnalyzerArgs +from .endpoint_data import RedfishEndpointDataModel + + +def _get_by_path(obj: Any, path: str) -> Any: + """Get a value from a nested dict/list.""" + if not path.strip(): + return obj + current: Any = obj + for segment in path.strip().split("/"): + if not segment: + continue + if current is None: + return None + if isinstance(current, list): + try: + idx = int(segment) + current = current[idx] if 0 <= idx < len(current) else None + except ValueError: + return None + elif isinstance(current, dict): + current = current.get(segment) + else: + return None + return current + + +def _check_constraint(actual: Any, constraint: RedfishConstraint) -> tuple[bool, str]: + """Compare actual value to constraint.""" + if isinstance(constraint, dict): + if ConstraintKey.EXISTS in constraint: + want = bool(constraint[ConstraintKey.EXISTS]) + ok = (actual is not None) == want + return ok, f"expected {'present' if want else 'absent'}, got {actual!r}" + if ConstraintKey.REGEX in constraint: + pattern = constraint[ConstraintKey.REGEX] + if not isinstance(actual, str): + return False, f"expected string matching {pattern!r}, got {actual!r}" + try: + ok = re.search(pattern, actual) is not None + except re.error as exc: + return False, f"invalid regex {pattern!r}: {exc}" + return ok, f"expected match for {pattern!r}, got {actual!r}" + if ConstraintKey.EQ in constraint: + ok = actual == constraint[ConstraintKey.EQ] + return ok, f"expected eq {constraint[ConstraintKey.EQ]}, got {actual!r}" + if ConstraintKey.MIN in constraint or ConstraintKey.MAX in constraint: + try: + val = float(actual) if actual is not None else None + if val is None: + return False, f"expected numeric, got {actual!r}" + if ConstraintKey.MIN in constraint and val < constraint[ConstraintKey.MIN]: + return False, f"value {val} below min {constraint[ConstraintKey.MIN]}" + if ConstraintKey.MAX in constraint and val > constraint[ConstraintKey.MAX]: + return False, f"value {val} above max {constraint[ConstraintKey.MAX]}" + return True, "" + except (TypeError, ValueError): + return False, f"expected numeric, got {actual!r}" + if ConstraintKey.ANY_OF in constraint: + allowed = constraint[ConstraintKey.ANY_OF] + if not isinstance(allowed, list): + return False, "anyOf must be a list" + ok = actual in allowed + return ok, f"expected any of {allowed}, got {actual!r}" + ok = actual == constraint + return ok, f"expected {constraint!r}, got {actual!r}" + + +class RedfishEndpointAnalyzer(DataAnalyzer[RedfishEndpointDataModel, RedfishEndpointAnalyzerArgs]): + """Checks Redfish endpoint responses against configured thresholds and key/value rules.""" + + DATA_MODEL = RedfishEndpointDataModel + + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "For each entry in analysis_args.checks, reads JSON paths in collected responses and " + "compares values to constraints (eq, min/max, anyOf, regex, exists).", + "URI key `*` runs checks against every collected response body.", + ) + + def analyze_data( + self, + data: RedfishEndpointDataModel, + args: Optional[RedfishEndpointAnalyzerArgs] = None, + ) -> TaskResult: + """Evaluate each configured check against the collected Redfish responses.""" + if not args or not args.checks: + self.result.status = ExecutionStatus.OK + self.result.message = "No checks configured" + return self.result + + failed: list[dict[str, Any]] = [] + for uri, path_constraints in args.checks.items(): + if uri == "*": + bodies = list(data.responses.values()) + else: + body = data.responses.get(uri) + bodies = [body] if body is not None else [] + if not bodies: + if uri != "*": + failed.append( + {"uri": uri, "path": None, "reason": "URI not in collected responses"} + ) + continue + for resp in bodies: + for path, constraint in path_constraints.items(): + actual = _get_by_path(resp, path) + ok, msg = _check_constraint(actual, constraint) + if not ok: + failed.append( + { + "uri": uri, + "path": path, + "expected": constraint, + "actual": actual, + "reason": msg, + } + ) + + if failed: + description = f"Redfish endpoint checks failed: {len(failed)} failure(s)" + self._log_event( + category=EventCategory.TELEMETRY, + description=description, + data={"failures": failed}, + priority=EventPriority.WARNING, + console_log=True, + ) + self.result.status = ExecutionStatus.ERROR + self.result.message = f"{len(failed)} check(s) failed" + else: + self.result.status = ExecutionStatus.OK + self.result.message = "All Redfish endpoint checks passed" + return self.result diff --git a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_collector.py b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_collector.py index d76cb33d..e9076069 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_collector.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_collector.py @@ -1,282 +1,492 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from collections import deque -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Any, Optional -from urllib.parse import urlparse - -from nodescraper.base import RedfishDataCollector -from nodescraper.connection.redfish import ( - RF_MEMBERS, - RF_ODATA_ID, - RedfishConnection, - RedfishGetResult, -) -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.models import TaskResult - -from .collector_args import RedfishEndpointCollectorArgs -from .endpoint_data import RedfishEndpointDataModel - - -def _normalize_path(odata_id: str, api_root: str) -> str: - """Convert @odata.id value (URL or path) to a normalized path under api_root.""" - if not odata_id or not isinstance(odata_id, str): - return "" - s = odata_id.strip() - if s.startswith(("http://", "https://")): - parsed = urlparse(s) - s = parsed.path or "/" - if not s.startswith("/"): - s = "/" + s - s = s.rstrip("/") or "/" - api_root_norm = api_root.strip("/") - if api_root_norm and not s.startswith("/" + api_root_norm): - return "" - return s - - -def _extract_odata_ids(obj: Any) -> list[str]: - """Recursively extract all @odata.id values from a Redfish JSON body.""" - out: list[str] = [] - if isinstance(obj, dict): - if RF_ODATA_ID in obj and isinstance(obj[RF_ODATA_ID], str): - out.append(obj[RF_ODATA_ID]) - for k, v in obj.items(): - if k == RF_MEMBERS and isinstance(v, list): - for item in v: - if ( - isinstance(item, dict) - and RF_ODATA_ID in item - and isinstance(item[RF_ODATA_ID], str) - ): - out.append(item[RF_ODATA_ID]) - elif isinstance(v, dict): - out.extend(_extract_odata_ids(v)) - elif isinstance(v, list): - for item in v: - if isinstance(item, dict): - out.extend(_extract_odata_ids(item)) - return out - - -def _discover_tree( - connection: RedfishConnection, - api_root: str, - max_depth: int, - max_endpoints: int, -) -> tuple[list[str], dict[str, dict], list[RedfishGetResult]]: - """ - Traverse the Redfish resource tree from the service root. - - max_depth matches collection_args.tree_max_depth: 1 = service root only; 2 = root + one link - level; child links are only enqueued when depth + 1 < max_depth (root is depth 0). - """ - root_path = _normalize_path(api_root, api_root) or ("/" + api_root.strip("/")) - seen: set[str] = set() - to_visit: deque[tuple[str, int]] = deque([(root_path, 0)]) - responses: dict[str, dict] = {} - results: list[RedfishGetResult] = [] - while to_visit: - if max_endpoints and len(seen) >= max_endpoints: - break - path, depth = to_visit.popleft() - if path in seen or depth > max_depth: - continue - seen.add(path) - res = connection.run_get(path) - results.append(res) - if res.success and res.data is not None: - responses[path] = res.data - for odata_id in _extract_odata_ids(res.data): - link_path = _normalize_path(odata_id, api_root) - # Follow only if the child depth stays strictly below max_depth (1 = root only). - if link_path and link_path not in seen and depth + 1 < max_depth: - to_visit.append((link_path, depth + 1)) - return sorted(seen), responses, results - - -def _uris_from_args(args: Optional[RedfishEndpointCollectorArgs]) -> list[str]: - """Return list of URIs from collector args.uris.""" - if args is None: - return [] - return list(args.uris) if args.uris else [] - - -def _discover_tree_enabled(args: Optional[RedfishEndpointCollectorArgs]) -> bool: - """True only when tree discovery is explicitly enabled (avoids string/other truthy junk).""" - if args is None: - return False - return getattr(args, "discover_tree", False) is True - - -def _fetch_one(connection_copy: RedfishConnection, path: str) -> RedfishGetResult: - """Run a single GET on a connection copy (used from worker threads).""" - return connection_copy.run_get(path) - - -def _fetch_one_paged( - connection_copy: RedfishConnection, path: str, max_pages: int -) -> RedfishGetResult: - """Run a paged GET on a connection copy, following Members@odata.nextLink (used from worker threads).""" - return connection_copy.run_get_paged(path, max_pages=max_pages) - - -class RedfishEndpointCollector( - RedfishDataCollector[RedfishEndpointDataModel, RedfishEndpointCollectorArgs] -): - """Collects Redfish endpoint responses for URIs specified in config.""" - - DATA_MODEL = RedfishEndpointDataModel - - DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( - "Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).", - "Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.", - "Redfish GET tree: when discover_tree is true, walks from api_root using OData resource id links and " - "Members navigation (depth and endpoint caps from collection_args).", - ) - - def collect_data( - self, args: Optional[RedfishEndpointCollectorArgs] = None - ) -> tuple[TaskResult, Optional[RedfishEndpointDataModel]]: - """Collect via tree discovery, or via explicit URIs, or skip if neither is configured.""" - uris = _uris_from_args(args) - use_tree = _discover_tree_enabled(args) - - # 1) Tree discovery: only when discover_tree is explicitly true - if use_tree: - api_root = getattr(self.connection, "api_root", "redfish/v1") - max_depth = getattr(args, "tree_max_depth", 2) if args else 2 - max_endpoints = (getattr(args, "tree_max_endpoints", 0) or 0) if args else 0 - _paths, responses, results = _discover_tree( - self.connection, - api_root, - max_depth=max_depth, - max_endpoints=max_endpoints, - ) - for res in results: - self._append_redfish_artifact(res) - if not res.success and res.error: - self._log_event( - category=EventCategory.RUNTIME, - description=f"Redfish GET failed during tree discovery for {res.path}: {res.error}", - priority=EventPriority.WARNING, - console_log=True, - ) - if not responses: - self.result.message = "No Redfish endpoints discovered from tree" - self.result.status = ExecutionStatus.ERROR - return self.result, None - data = RedfishEndpointDataModel(responses=responses) - self.result.message = f"Collected {len(responses)} Redfish endpoint(s) from tree" - self.result.status = ExecutionStatus.OK - return self.result, data - - # 2) URI list: when discover_tree is false/absent and uris are provided - if not uris: - self.result.message = ( - "No collection mode configured: set collection_args.discover_tree to true " - "or provide collection_args.uris" - ) - self.result.status = ExecutionStatus.NOT_RAN - return self.result, None - - paths = [] - for uri in uris: - path = uri.strip() if uri else "" - if not path: - continue - if not path.startswith("/"): - path = "/" + path - paths.append(path) - - max_workers = getattr(args, "max_workers", 1) if args else 1 - max_workers = min(max_workers, len(paths)) - follow_next_link = getattr(args, "follow_next_link", False) is True - max_pages = getattr(args, "max_pages", 200) if args else 200 - - if max_workers <= 1: - # Sequential - responses = {} - for path in paths: - if follow_next_link: - res = self._run_redfish_get_paged(path, max_pages=max_pages, log_artifact=True) - else: - res = self._run_redfish_get(path, log_artifact=True) - if res.success and res.data is not None: - responses[res.path] = res.data - else: - self._log_event( - category=EventCategory.RUNTIME, - description=f"Redfish GET failed for {path}: {res.error or 'unknown'}", - priority=EventPriority.WARNING, - console_log=True, - ) - else: - # Concurrent: one connection copy per worker, collect results in main thread - responses = {} - results = [] - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = {} - for path in paths: - conn = self.connection.copy() - if follow_next_link: - futures[executor.submit(_fetch_one_paged, conn, path, max_pages)] = path - else: - futures[executor.submit(_fetch_one, conn, path)] = path - for future in as_completed(futures): - path = futures[future] - try: - res = future.result() - results.append(res) - if res.success and res.data is not None: - responses[res.path] = res.data - else: - self._log_event( - category=EventCategory.RUNTIME, - description=f"Redfish GET failed for {path}: {res.error or 'unknown'}", - priority=EventPriority.WARNING, - console_log=True, - ) - except Exception as e: - self._log_event( - category=EventCategory.RUNTIME, - description=f"Redfish GET failed for {path}: {e!s}", - priority=EventPriority.WARNING, - console_log=True, - ) - for res in results: - self._append_redfish_artifact(res) - - if not responses: - self.result.message = "No Redfish endpoints could be read" - self.result.status = ExecutionStatus.ERROR - return self.result, None - - data = RedfishEndpointDataModel(responses=responses) - self.result.message = f"Collected {len(responses)} Redfish endpoint(s)" - self.result.status = ExecutionStatus.OK - return self.result, data +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from collections import deque +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Optional +from urllib.parse import urlparse + +from nodescraper.base import RedfishDataCollector +from nodescraper.connection.redfish import ( + RF_MEMBERS, + RF_MEMBERS_NEXT_LINK, + RF_ODATA_ID, + RedfishConnection, + RedfishGetResult, +) +from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + clear_redfish_logs, +) +from nodescraper.enums import ( + EventCategory, + EventPriority, + ExecutionStatus, + SystemInteractionLevel, +) +from nodescraper.models import TaskResult + +from .collector_args import RedfishEndpointCollectorArgs +from .endpoint_data import RedfishEndpointDataModel + +_REFERENCE_KEYS = frozenset( + { + RF_ODATA_ID, + "Uri", + "@Redfish.ActionInfo", + "DataSourceUri", + "TargetComponentURI", + } +) +_SKIPPED_REFERENCE_SUBTREES = frozenset({"Oem", "OriginOfCondition"}) +# Individual LogServices entry resources, not the Entries collection. +_LOG_ENTRY_URI_RE = re.compile(r"LogServices/.*/Entries/[^/]+$") + + +def _normalize_path(odata_id: str, api_root: str) -> str: + """Convert @odata.id value (URL or path) to a normalized path under api_root.""" + if not odata_id or not isinstance(odata_id, str): + return "" + s = odata_id.strip() + if s.startswith(("http://", "https://")): + parsed = urlparse(s) + s = parsed.path or "/" + if parsed.query: + s = f"{s}?{parsed.query}" + if not s.startswith("/"): + s = "/" + s + s = s.rstrip("/") or "/" + api_root_norm = api_root.strip("/") + if api_root_norm and not s.startswith("/" + api_root_norm): + return "" + return s + + +def _is_fetchable_reference(value: Any) -> bool: + """Return whether a value points at a separate Redfish resource. + + Args: + value: A candidate Redfish reference. + + Returns: + True when value is an absolute resource path without a fragment. + """ + return isinstance(value, str) and value.startswith("/") and "#" not in value + + +def _extract_odata_ids(obj: Any) -> list[str]: + """Extract Redfish resource references from a JSON body. + + Args: + obj: A Redfish JSON value. + + Returns: + Resource paths and resource ids referenced by the body. + """ + out: list[str] = [] + if not isinstance(obj, dict): + return out + for key, value in obj.items(): + if key in _SKIPPED_REFERENCE_SUBTREES: + continue + if key == RF_ODATA_ID and isinstance(value, str): + out.append(value) + elif key in _REFERENCE_KEYS and _is_fetchable_reference(value): + out.append(value) + for key, value in obj.items(): + if key in _SKIPPED_REFERENCE_SUBTREES: + continue + if key == RF_MEMBERS and isinstance(value, list): + for item in value: + if isinstance(item, dict) and isinstance(item.get(RF_ODATA_ID), str): + out.append(item[RF_ODATA_ID]) + elif isinstance(value, dict): + out.extend(_extract_odata_ids(value)) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + out.extend(_extract_odata_ids(item)) + return out + + +def _normalize_next_link(odata_id: Any, api_root: str) -> str: + """Normalize a collection next link and keep its query string. + + Args: + odata_id: A Members nextLink value. + api_root: Redfish service root. + + Returns: + The path and query under the service root, or an empty string. + """ + if not isinstance(odata_id, str) or not odata_id.strip(): + return "" + source = odata_id.strip() + query = "" + if source.startswith(("http://", "https://")): + parsed = urlparse(source) + source = parsed.path or "/" + query = f"?{parsed.query}" if parsed.query else "" + elif "?" in source: + source, raw_query = source.split("?", 1) + query = "?" + raw_query + path = _normalize_path(source, api_root) + if not path: + return "" + return path + query + + +def _enqueue_tree_links( + payload: Any, + depth: int, + api_root: str, + max_depth: int, + seen: set[str], + to_visit: deque[tuple[str, int]], + queued: set[str], + max_log_entry_members: int, +) -> None: + """Queue child resources linked from one Redfish payload. + + Args: + payload: Redfish JSON body. + depth: Depth of the payload that contains the links. + api_root: Redfish service root. + max_depth: Maximum resource depth. + seen: Paths already fetched. + to_visit: Paths waiting to be fetched. + queued: Paths already queued in this crawl. + max_log_entry_members: Maximum log entries from this payload. Zero means no limit. + + Returns: + None + """ + candidates: list[str] = [] + for odata_id in _extract_odata_ids(payload): + if isinstance(odata_id, str) and "#" in odata_id: + continue + link_path = _normalize_path(odata_id, api_root) + if not link_path or link_path in seen or link_path in queued or depth + 1 >= max_depth: + continue + candidates.append(link_path) + unique = list(dict.fromkeys(candidates)) + log_entries = sorted(path for path in unique if _LOG_ENTRY_URI_RE.search(path)) + others = [path for path in unique if not _LOG_ENTRY_URI_RE.search(path)] + if max_log_entry_members: + log_entries = log_entries[:max_log_entry_members] + for link_path in others + log_entries: + queued.add(link_path) + to_visit.append((link_path, depth + 1)) + + +def _discover_tree( + connection: RedfishConnection, + api_root: str, + max_depth: int, + max_endpoints: int, + max_collection_pages: int = 1, + max_log_entry_members: int = 0, +) -> tuple[list[str], dict[str, dict], list[RedfishGetResult]]: + """Traverse the Redfish resource tree from the service root. + + Args: + connection: Connection used to fetch resources. + api_root: Redfish service-root path. + max_depth: Maximum resource depth, where one is only the service root. + max_endpoints: Maximum resource count, where zero means unlimited. + max_collection_pages: Maximum pages to retain for each paginated collection. + max_log_entry_members: Maximum individual log entries taken from each payload. + + Returns: + Visited paths, successful response bodies, and every GET result. + """ + root_path = _normalize_path(api_root, api_root) or ("/" + api_root.strip("/")) + seen: set[str] = set() + queued: set[str] = {root_path} + to_visit: deque[tuple[str, int]] = deque([(root_path, 0)]) + responses: dict[str, dict] = {} + results: list[RedfishGetResult] = [] + while to_visit: + if max_endpoints and len(seen) >= max_endpoints: + break + path, depth = to_visit.popleft() + if path in seen or depth > max_depth: + continue + seen.add(path) + res = connection.run_get(path) + results.append(res) + if not res.success or not isinstance(res.data, dict): + continue + responses[path] = res.data + _enqueue_tree_links( + res.data, + depth, + api_root, + max_depth, + seen, + to_visit, + queued, + max_log_entry_members, + ) + if max_collection_pages <= 1: + continue + next_path = _normalize_next_link(res.data.get(RF_MEMBERS_NEXT_LINK), api_root) + pages = 1 + while next_path and pages < max_collection_pages: + if next_path in seen or (max_endpoints and len(seen) >= max_endpoints): + break + seen.add(next_path) + queued.add(next_path) + page_res = connection.run_get(next_path) + results.append(page_res) + pages += 1 + if not page_res.success or not isinstance(page_res.data, dict): + break + responses[next_path] = page_res.data + _enqueue_tree_links( + page_res.data, + depth, + api_root, + max_depth, + seen, + to_visit, + queued, + max_log_entry_members, + ) + next_path = _normalize_next_link(page_res.data.get(RF_MEMBERS_NEXT_LINK), api_root) + return sorted(seen), responses, results + + +def _uris_from_args(args: Optional[RedfishEndpointCollectorArgs]) -> list[str]: + """Return list of URIs from collector args.uris.""" + if args is None: + return [] + return list(args.uris) if args.uris else [] + + +def _discover_tree_enabled(args: Optional[RedfishEndpointCollectorArgs]) -> bool: + """True only when tree discovery is explicitly enabled (avoids string/other truthy junk).""" + if args is None: + return False + return getattr(args, "discover_tree", False) is True + + +def _fetch_one(connection_copy: RedfishConnection, path: str) -> RedfishGetResult: + """Run a single GET on a connection copy (used from worker threads).""" + return connection_copy.run_get(path) + + +def _fetch_one_paged( + connection_copy: RedfishConnection, path: str, max_pages: int +) -> RedfishGetResult: + """Run a paged GET on a connection copy, following Members@odata.nextLink (used from worker threads).""" + return connection_copy.run_get_paged(path, max_pages=max_pages) + + +class RedfishEndpointCollector( + RedfishDataCollector[RedfishEndpointDataModel, RedfishEndpointCollectorArgs] +): + """Collects Redfish endpoint responses for URIs specified in config.""" + + DATA_MODEL = RedfishEndpointDataModel + + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "Redfish GET: explicit paths from collection_args.uris (parallel when max_workers>1).", + "Optional paged GET following the Members collection OData nextLink field when follow_next_link is true.", + "Redfish GET tree: when discover_tree is true, walks from api_root using Redfish resource links. " + "Oem and OriginOfCondition subtrees are not followed. Collection pages and individual log entries " + "are capped by collection_args.", + ) + + def collect_data( + self, args: Optional[RedfishEndpointCollectorArgs] = None + ) -> tuple[TaskResult, Optional[RedfishEndpointDataModel]]: + """Collect via tree discovery, or via explicit URIs, or skip if neither is configured.""" + uris = _uris_from_args(args) + use_tree = _discover_tree_enabled(args) + + # 1) Tree discovery: only when discover_tree is explicitly true + if use_tree: + api_root = getattr(self.connection, "api_root", "redfish/v1") + max_depth = getattr(args, "tree_max_depth", 2) if args else 2 + max_endpoints = (getattr(args, "tree_max_endpoints", 0) or 0) if args else 0 + max_collection_pages = getattr(args, "tree_max_collection_pages", 1) if args else 1 + max_log_entry_members = getattr(args, "tree_max_log_entry_members", 0) if args else 0 + _paths, responses, results = _discover_tree( + self.connection, + api_root, + max_depth=max_depth, + max_endpoints=max_endpoints, + max_collection_pages=max_collection_pages, + max_log_entry_members=max_log_entry_members, + ) + for res in results: + self._append_redfish_artifact(res) + if not res.success and res.error: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Redfish GET failed during tree discovery for {res.path}: {res.error}", + priority=EventPriority.WARNING, + console_log=True, + ) + if not responses: + self.result.message = "No Redfish endpoints discovered from tree" + self.result.status = ExecutionStatus.ERROR + return self.result, None + data = RedfishEndpointDataModel(responses=responses) + self.result.message = f"Collected {len(responses)} Redfish endpoint(s) from tree" + self.result.status = ExecutionStatus.OK + self._apply_clear_log_uris(args) + return self.result, data + + # 2) URI list: when discover_tree is false/absent and uris are provided + if not uris: + self.result.message = ( + "No collection mode configured: set collection_args.discover_tree to true " + "or provide collection_args.uris" + ) + self.result.status = ExecutionStatus.NOT_RAN + return self.result, None + + paths = [] + for uri in uris: + path = uri.strip() if uri else "" + if not path: + continue + if not path.startswith("/"): + path = "/" + path + paths.append(path) + + max_workers = getattr(args, "max_workers", 1) if args else 1 + max_workers = min(max_workers, len(paths)) + follow_next_link = getattr(args, "follow_next_link", False) is True + max_pages = getattr(args, "max_pages", 200) if args else 200 + + if max_workers <= 1: + # Sequential + responses = {} + for path in paths: + if follow_next_link: + res = self._run_redfish_get_paged(path, max_pages=max_pages, log_artifact=True) + else: + res = self._run_redfish_get(path, log_artifact=True) + if res.success and res.data is not None: + responses[res.path] = res.data + else: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Redfish GET failed for {path}: {res.error or 'unknown'}", + priority=EventPriority.WARNING, + console_log=True, + ) + else: + # Concurrent: one connection copy per worker, collect results in main thread + responses = {} + results = [] + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {} + for path in paths: + conn = self.connection.copy() + if follow_next_link: + futures[executor.submit(_fetch_one_paged, conn, path, max_pages)] = path + else: + futures[executor.submit(_fetch_one, conn, path)] = path + for future in as_completed(futures): + path = futures[future] + try: + res = future.result() + results.append(res) + if res.success and res.data is not None: + responses[res.path] = res.data + else: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Redfish GET failed for {path}: {res.error or 'unknown'}", + priority=EventPriority.WARNING, + console_log=True, + ) + except Exception as e: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Redfish GET failed for {path}: {e!s}", + priority=EventPriority.WARNING, + console_log=True, + ) + for res in results: + self._append_redfish_artifact(res) + + if not responses: + self.result.message = "No Redfish endpoints could be read" + self.result.status = ExecutionStatus.ERROR + return self.result, None + + data = RedfishEndpointDataModel(responses=responses) + self.result.message = f"Collected {len(responses)} Redfish endpoint(s)" + self.result.status = ExecutionStatus.OK + self._apply_clear_log_uris(args) + return self.result, data + + def _apply_clear_log_uris(self, args: Optional[RedfishEndpointCollectorArgs]) -> None: + """POST {} to each URI in args.clear_log_uris (LogService.ClearLog endpoints). + + Skips when system_interaction_level < INTERACTIVE. Failures downgrade + the overall result to WARNING. + + Args: + args: Collector args carrying the clear_log_uris list. + """ + if args is None: + return + uris = [u.strip() for u in (args.clear_log_uris or []) if u and u.strip()] + if not uris: + return + if self.system_interaction_level < SystemInteractionLevel.INTERACTIVE: + self.logger.debug( + "Skipping log clear: system_interaction_level is %s (requires INTERACTIVE)", + self.system_interaction_level.name, + ) + return + + endpoints = [ClearLogEndpoint(path=uri) for uri in uris] + clear_results: list[ClearLogResult] = clear_redfish_logs( + self.connection, endpoints, self.logger + ) + cleared = [r for r in clear_results if r.success] + failed = [r for r in clear_results if not r.success] + if cleared: + self.result.message += f"; cleared {len(cleared)} log store(s)" + if failed: + descriptions = "; ".join(f"{r.path}: {r.error}" for r in failed) + self._log_event( + category=EventCategory.RUNTIME, + description=f"Failed to clear {len(failed)} log store(s): {descriptions}", + priority=EventPriority.WARNING, + console_log=True, + ) + if self.result.status == ExecutionStatus.OK: + self.result.status = ExecutionStatus.WARNING + self.result.message += f"; {len(failed)} log store(s) failed to clear" diff --git a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_data.py b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_data.py index 19145485..3a2112a9 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_data.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_data.py @@ -1,34 +1,34 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from pydantic import Field - -from nodescraper.models import DataModel - - -class RedfishEndpointDataModel(DataModel): - """Collected Redfish endpoint responses: URI -> JSON body.""" - - responses: dict[str, dict] = Field(default_factory=dict) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import DataModel + + +class RedfishEndpointDataModel(DataModel): + """Collected Redfish endpoint responses: URI -> JSON body.""" + + responses: dict[str, dict] = Field(default_factory=dict) diff --git a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_plugin.py b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_plugin.py index 029bcc88..ad8decf4 100644 --- a/nodescraper/plugins/ooband/redfish_endpoint/endpoint_plugin.py +++ b/nodescraper/plugins/ooband/redfish_endpoint/endpoint_plugin.py @@ -1,48 +1,60 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from nodescraper.base import OOBandDataPlugin - -from .analyzer_args import RedfishEndpointAnalyzerArgs -from .collector_args import RedfishEndpointCollectorArgs -from .endpoint_analyzer import RedfishEndpointAnalyzer -from .endpoint_collector import RedfishEndpointCollector -from .endpoint_data import RedfishEndpointDataModel - - -class RedfishEndpointPlugin( - OOBandDataPlugin[ - RedfishEndpointDataModel, - RedfishEndpointCollectorArgs, - RedfishEndpointAnalyzerArgs, - ] -): - """Config-driven plugin: collect from Redfish URIs and check against thresholds/key-values.""" - - DATA_MODEL = RedfishEndpointDataModel - COLLECTOR = RedfishEndpointCollector - ANALYZER = RedfishEndpointAnalyzer - COLLECTOR_ARGS = RedfishEndpointCollectorArgs - ANALYZER_ARGS = RedfishEndpointAnalyzerArgs +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.base import OOBandDataPlugin +from nodescraper.connection.redfish import RedfishSshProxyConnectionManager + +from .analyzer_args import RedfishEndpointAnalyzerArgs +from .collector_args import RedfishEndpointCollectorArgs +from .endpoint_analyzer import RedfishEndpointAnalyzer +from .endpoint_collector import RedfishEndpointCollector +from .endpoint_data import RedfishEndpointDataModel + + +class RedfishEndpointPlugin( + OOBandDataPlugin[ + RedfishEndpointDataModel, + RedfishEndpointCollectorArgs, + RedfishEndpointAnalyzerArgs, + ] +): + """Config-driven plugin: collect from Redfish URIs and check against thresholds/key-values.""" + + DATA_MODEL = RedfishEndpointDataModel + COLLECTOR = RedfishEndpointCollector + ANALYZER = RedfishEndpointAnalyzer + COLLECTOR_ARGS = RedfishEndpointCollectorArgs + ANALYZER_ARGS = RedfishEndpointAnalyzerArgs + + +class AmcRedfishEndpointPlugin(RedfishEndpointPlugin): + """Collect the same Redfish endpoint checks through the BMC SSH proxy. + + Pair this with RedfishEndpointPlugin so one run reads the BMC on the direct + Redfish connection and the AMC on RedfishSshProxyConnectionManager. collection_args + and analysis_args are the same for both plugins. + """ + + CONNECTION_TYPE = RedfishSshProxyConnectionManager # type: ignore[assignment] diff --git a/nodescraper/plugins/ooband/redfish_event_log/__init__.py b/nodescraper/plugins/ooband/redfish_event_log/__init__.py new file mode 100644 index 00000000..cc12c9f7 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/__init__.py @@ -0,0 +1,24 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +############################################################################### +from .analyzer_args import RedfishEventLogAnalyzerArgs +from .collector_args import RedfishEventLogCollectorArgs +from .event_log_analyzer import RedfishEventLogAnalyzer +from .event_log_collector import RedfishEventLogCollector +from .event_log_data import EventLogEntry, RedfishEventLogDataModel +from .event_log_plugin import AmcRedfishEventLogPlugin, RedfishEventLogPlugin + +__all__ = [ + "AmcRedfishEventLogPlugin", + "EventLogEntry", + "RedfishEventLogAnalyzer", + "RedfishEventLogAnalyzerArgs", + "RedfishEventLogCollector", + "RedfishEventLogCollectorArgs", + "RedfishEventLogDataModel", + "RedfishEventLogPlugin", +] diff --git a/nodescraper/plugins/ooband/redfish_event_log/analyzer_args.py b/nodescraper/plugins/ooband/redfish_event_log/analyzer_args.py new file mode 100644 index 00000000..7e4d15a7 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/analyzer_args.py @@ -0,0 +1,69 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from datetime import datetime, timezone +from typing import Optional + +from pydantic import Field, field_validator + +from nodescraper.models import AnalyzerArgs + + +class RedfishEventLogAnalyzerArgs(AnalyzerArgs): + """Analyzer args for Redfish LogService entries.""" + + since: Optional[datetime] = Field( + default=None, + description=( + "Only analyze entries created at or after this ISO 8601 time (naive values are UTC). " + "Entries without a parseable Created time are always analyzed." + ), + ) + message_id_include: Optional[list[str]] = Field( + default=None, + description="If set, only analyze entries whose MessageId matches one of these regexes.", + ) + message_id_exclude: list[str] = Field( + default_factory=list, + description="Skip entries whose MessageId matches any of these regexes.", + ) + + @field_validator("since") + @classmethod + def _since_utc(cls, value: Optional[datetime]) -> Optional[datetime]: + if value is not None and value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + + @field_validator("message_id_include", "message_id_exclude") + @classmethod + def _valid_patterns(cls, patterns: Optional[list[str]]) -> Optional[list[str]]: + for pattern in patterns or []: + try: + re.compile(pattern) + except re.error as exc: + raise ValueError(f"invalid regex {pattern!r}: {exc}") from exc + return patterns diff --git a/nodescraper/plugins/ooband/redfish_event_log/collector_args.py b/nodescraper/plugins/ooband/redfish_event_log/collector_args.py new file mode 100644 index 00000000..4c2c8e35 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/collector_args.py @@ -0,0 +1,57 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from pydantic import Field + +from nodescraper.models import CollectorArgs + + +class RedfishEventLogCollectorArgs(CollectorArgs): + """Collector args for reading a Redfish LogService's Entries collection.""" + + log_service_path: str = Field( + default="redfish/v1/Systems/system/LogServices/EventLog", + description="Redfish path to the LogService; entries are read from /Entries.", + ) + max_entries: Optional[int] = Field( + default=None, + ge=1, + description="Keep only the newest N entries (by Created). All pages are still read.", + ) + max_pages: int = Field( + default=100, + ge=1, + le=10000, + description="Safety cap on the number of $skip pages requested.", + ) + clear_logs_after_collection: bool = Field( + default=False, + description=( + "If True, POST LogService.ClearLog to log_service_path after a successful " + "collection. Requires system_interaction_level >= INTERACTIVE." + ), + ) diff --git a/nodescraper/plugins/ooband/redfish_event_log/event_log_analyzer.py b/nodescraper/plugins/ooband/redfish_event_log/event_log_analyzer.py new file mode 100644 index 00000000..3d3d071e --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/event_log_analyzer.py @@ -0,0 +1,119 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import re +from typing import Optional + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import RedfishEventLogAnalyzerArgs +from .event_log_data import EventLogEntry, RedfishEventLogDataModel, parse_created + +SEVERITY_PRIORITY: dict[str, EventPriority] = { + "critical": EventPriority.ERROR, + "warning": EventPriority.WARNING, +} +MAX_IDS_PER_EVENT = 20 + + +def _selected(entry: EventLogEntry, args: RedfishEventLogAnalyzerArgs) -> bool: + if args.since is not None: + created = parse_created(entry.created) + if created is not None and created < args.since: + return False + message_id = entry.message_id or "" + if args.message_id_include is not None and not any( + re.search(p, message_id) for p in args.message_id_include + ): + return False + return not any(re.search(p, message_id) for p in args.message_id_exclude) + + +class RedfishEventLogAnalyzer(DataAnalyzer[RedfishEventLogDataModel, RedfishEventLogAnalyzerArgs]): + """Reports Critical and Warning LogService entries as events.""" + + DATA_MODEL = RedfishEventLogDataModel + + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "Critical entries are reported as ERROR events and Warning entries as WARNING events; " + "OK and other severities are ignored.", + "Entries with the same Severity and MessageId are reported once with a count and the newest message.", + "analysis_args.since, message_id_include and message_id_exclude limit which entries are analyzed.", + ) + + def analyze_data( + self, + data: RedfishEventLogDataModel, + args: Optional[RedfishEventLogAnalyzerArgs] = None, + ) -> TaskResult: + """Report Critical and Warning entries, collapsed by Severity and MessageId.""" + if args is None: + args = RedfishEventLogAnalyzerArgs() + + selected = [e for e in data.entries if _selected(e, args)] + groups: dict[tuple[str, str], list[EventLogEntry]] = {} + for entry in selected: + severity = (entry.severity or "").lower() + if severity in SEVERITY_PRIORITY: + key = (severity, entry.message_id or entry.message or "") + groups.setdefault(key, []).append(entry) + + counts = {severity: 0 for severity in SEVERITY_PRIORITY} + for (severity, _), group in groups.items(): + counts[severity] += len(group) + newest = group[0] + label = newest.message_id or "LogEntry" + repeat = f" (x{len(group)})" if len(group) > 1 else "" + detail = f": {newest.message}" if newest.message else "" + self._log_event( + category=EventCategory.PLATFORM, + description=f"{newest.severity} {label}{repeat}{detail}", + data={ + "log_service_path": data.log_service_path, + "severity": newest.severity, + "message_id": newest.message_id, + "count": len(group), + "newest_created": newest.created, + "oldest_created": group[-1].created, + "message_args": newest.message_args, + "ids": [e.id for e in group[:MAX_IDS_PER_EVENT]], + }, + priority=SEVERITY_PRIORITY[severity], + console_log=True, + ) + + self.result.message = ( + f"EventLog: {counts['critical']} critical, {counts['warning']} warning " + f"in {len(selected)} of {len(data.entries)} entries" + ) + if counts["critical"]: + self.result.status = ExecutionStatus.ERROR + elif counts["warning"]: + self.result.status = ExecutionStatus.WARNING + else: + self.result.status = ExecutionStatus.OK + return self.result diff --git a/nodescraper/plugins/ooband/redfish_event_log/event_log_collector.py b/nodescraper/plugins/ooband/redfish_event_log/event_log_collector.py new file mode 100644 index 00000000..6cfbdb8e --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/event_log_collector.py @@ -0,0 +1,227 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import json +from datetime import datetime, timezone +from typing import Any, NamedTuple, Optional + +from nodescraper.base import RedfishDataCollector +from nodescraper.connection.redfish.redfish_clear_log import ( + _parse_clear_endpoint, + clear_redfish_logs, +) +from nodescraper.enums import ( + EventCategory, + EventPriority, + ExecutionStatus, + SystemInteractionLevel, +) +from nodescraper.models import TaskResult + +from .collector_args import RedfishEventLogCollectorArgs +from .event_log_data import EventLogEntry, RedfishEventLogDataModel, parse_created + +_OLDEST = datetime.min.replace(tzinfo=timezone.utc) + + +class _EntriesRead(NamedTuple): + members: list[dict[str, Any]] + reported_count: int + error: Optional[str] + complete: bool + + +def _member_key(member: dict[str, Any]) -> str: + key = member.get("@odata.id") or member.get("Id") + return str(key) if key else json.dumps(member, sort_keys=True, default=str) + + +def _reported_count(data: dict[str, Any]) -> int: + try: + return int(data.get("Members@odata.count") or 0) + except (TypeError, ValueError): + return 0 + + +class RedfishEventLogCollector( + RedfishDataCollector[RedfishEventLogDataModel, RedfishEventLogCollectorArgs] +): + """Reads every entry from a Redfish LogService's Entries collection.""" + + DATA_MODEL = RedfishEventLogDataModel + + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "Redfish GET of /Entries, paged with $skip until " + "Members@odata.count is reached; each page is saved as an artifact.", + "Entries are sorted newest first by Created; collection_args.max_entries keeps only the newest N.", + "Optional LogService.ClearLog after collection (collection_args.clear_logs_after_collection).", + ) + + def collect_data( + self, args: Optional[RedfishEventLogCollectorArgs] = None + ) -> tuple[TaskResult, Optional[RedfishEventLogDataModel]]: + """Read the LogService entries and return them newest first.""" + if args is None: + args = RedfishEventLogCollectorArgs() + entries_path = f"{args.log_service_path.rstrip('/')}/Entries" + + read = self._read_entries(entries_path, args.max_pages) + if read.error is not None: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Could not read {entries_path}: {read.error}", + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.message = f"EventLog: could not read {entries_path}" + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result, None + + entries = sorted( + (EventLogEntry.from_redfish(m) for m in read.members), + key=lambda e: parse_created(e.created) or _OLDEST, + reverse=True, + ) + if args.max_entries: + entries = entries[: args.max_entries] + total = read.reported_count or len(read.members) + + self.result.message = f"EventLog: {len(entries)} of {total} entries from {entries_path}" + self.result.status = ExecutionStatus.OK + if not read.complete: + self.result.status = ExecutionStatus.WARNING + self.result.message += f" (read {len(read.members)}; pagination stopped early)" + + if args.clear_logs_after_collection: + if self.system_interaction_level >= SystemInteractionLevel.INTERACTIVE: + self._clear_log_service(args.log_service_path) + else: + self.logger.debug( + "Skipping log clear: system_interaction_level is %s (requires INTERACTIVE)", + self.system_interaction_level.name, + ) + + return self.result, RedfishEventLogDataModel( + log_service_path=args.log_service_path, + total_count=total, + entries=entries, + ) + + def _read_entries(self, entries_path: str, max_pages: int) -> _EntriesRead: + """Page through an Entries collection with $skip. + + $skip is used instead of Members@odata.nextLink because some services return the same + nextLink on every page, which never terminates. + + Args: + entries_path: Redfish path of the Entries collection. + max_pages: Maximum number of pages to request. + + Returns: + _EntriesRead: Members read, reported count, first-page error (if any), and whether + every reported entry was read. + """ + members: list[dict[str, Any]] = [] + seen: set[str] = set() + reported = 0 + skip = 0 + for page in range(max_pages): + path = entries_path if skip == 0 else f"{entries_path}?$skip={skip}" + res = self._run_redfish_get(path) + if not res.success or res.data is None: + if page == 0: + return _EntriesRead([], 0, res.error or f"HTTP {res.status_code}", False) + self._log_event( + category=EventCategory.RUNTIME, + description=f"EventLog page {path} failed: {res.error or res.status_code}", + priority=EventPriority.WARNING, + console_log=True, + ) + return _EntriesRead(members, reported, None, False) + + if page == 0: + reported = _reported_count(res.data) + page_members = [m for m in res.data.get("Members") or [] if isinstance(m, dict)] + new = [m for m in page_members if _member_key(m) not in seen] + seen.update(_member_key(m) for m in new) + members.extend(new) + skip += len(page_members) + + if not new: + complete = not reported or len(members) >= reported + if not complete: + self._log_event( + category=EventCategory.RUNTIME, + description=( + f"EventLog {entries_path}: page at $skip={skip - len(page_members)} " + f"returned no new entries ({len(members)} of {reported} read)" + ), + priority=EventPriority.WARNING, + console_log=True, + ) + return _EntriesRead(members, reported, None, complete) + if reported and len(members) >= reported: + return _EntriesRead(members, reported, None, True) + if not reported and "Members@odata.nextLink" not in res.data: + return _EntriesRead(members, reported, None, True) + + self._log_event( + category=EventCategory.RUNTIME, + description=( + f"EventLog {entries_path}: stopped after max_pages={max_pages} " + f"({len(members)} of {reported or '?'} read)" + ), + priority=EventPriority.WARNING, + console_log=True, + ) + return _EntriesRead(members, reported, None, False) + + def _clear_log_service(self, log_service_path: str) -> None: + """POST LogService.ClearLog to the log service if it advertises the action. + + Args: + log_service_path: Redfish path to the LogService. + """ + svc_res = self._run_redfish_get(log_service_path, log_artifact=False) + if not svc_res.success or not svc_res.data: + self.logger.debug("Skipping log clear: could not GET log service %s", log_service_path) + return + endpoint = _parse_clear_endpoint(svc_res.data) + if endpoint is None: + self.logger.debug("LogService %s does not advertise ClearLog action", log_service_path) + return + clear_results = clear_redfish_logs(self.connection, [endpoint], self.logger) + if clear_results and clear_results[0].success: + self.result.message += "; cleared log store" + elif clear_results: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Failed to clear log store {endpoint.path}: {clear_results[0].error}", + priority=EventPriority.WARNING, + console_log=True, + ) + if self.result.status == ExecutionStatus.OK: + self.result.status = ExecutionStatus.WARNING + self.result.message += "; log clear failed" diff --git a/nodescraper/plugins/ooband/redfish_event_log/event_log_data.py b/nodescraper/plugins/ooband/redfish_event_log/event_log_data.py new file mode 100644 index 00000000..0181d5d5 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/event_log_data.py @@ -0,0 +1,104 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any, Optional + +from pydantic import BaseModel, Field + +from nodescraper.models import DataModel + + +def parse_created(value: Optional[str]) -> Optional[datetime]: + """Parse a LogEntry Created timestamp into an aware datetime. + + Args: + value: ISO 8601 timestamp, e.g. 2026-01-01T00:00:00+00:00 or ...Z. + + Returns: + Optional[datetime]: Parsed time (naive values are taken as UTC), or None if unparseable. + """ + if not value: + return None + text = value.strip() + # Python 3.9's fromisoformat does not accept the "Z" suffix. + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +class EventLogEntry(BaseModel): + """Subset of a Redfish LogEntry used for analysis.""" + + id: str = "" + odata_id: Optional[str] = None + created: Optional[str] = None + severity: Optional[str] = None + message_id: Optional[str] = None + message: Optional[str] = None + message_args: list[str] = Field(default_factory=list) + entry_type: Optional[str] = None + + @classmethod + def from_redfish(cls, member: dict[str, Any]) -> EventLogEntry: + """Build an entry from a LogEntry JSON object. + + Args: + member: One element of the Entries collection's Members list. + + Returns: + EventLogEntry: Parsed entry; missing properties are left empty. + """ + args = member.get("MessageArgs") or [] + return cls( + id=str(member.get("Id") or ""), + odata_id=member.get("@odata.id"), + created=member.get("Created"), + severity=member.get("Severity"), + message_id=member.get("MessageId"), + message=member.get("Message"), + message_args=[str(a) for a in args] if isinstance(args, list) else [str(args)], + entry_type=member.get("EntryType"), + ) + + +class RedfishEventLogDataModel(DataModel): + """Entries read from one Redfish LogService, newest first.""" + + log_service_path: str + total_count: int = Field( + default=0, + description=( + "Members@odata.count reported on the first page, or the number of entries read " + "when the service omits it." + ), + ) + entries: list[EventLogEntry] = Field(default_factory=list) diff --git a/nodescraper/plugins/ooband/redfish_event_log/event_log_plugin.py b/nodescraper/plugins/ooband/redfish_event_log/event_log_plugin.py new file mode 100644 index 00000000..3cfe5bf7 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_event_log/event_log_plugin.py @@ -0,0 +1,64 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.base import OOBandDataPlugin +from nodescraper.connection.redfish import RedfishSshProxyConnectionManager + +from .analyzer_args import RedfishEventLogAnalyzerArgs +from .collector_args import RedfishEventLogCollectorArgs +from .event_log_analyzer import RedfishEventLogAnalyzer +from .event_log_collector import RedfishEventLogCollector +from .event_log_data import RedfishEventLogDataModel + + +class RedfishEventLogPlugin( + OOBandDataPlugin[ + RedfishEventLogDataModel, + RedfishEventLogCollectorArgs, + RedfishEventLogAnalyzerArgs, + ] +): + """Collect a Redfish LogService's entries (e.g. EventLog) and report Critical/Warning entries. + + Uses RedfishConnectionManager. Set collection_args.log_service_path to the LogService to read. + """ + + DATA_MODEL = RedfishEventLogDataModel + COLLECTOR = RedfishEventLogCollector + ANALYZER = RedfishEventLogAnalyzer + COLLECTOR_ARGS = RedfishEventLogCollectorArgs + ANALYZER_ARGS = RedfishEventLogAnalyzerArgs + + +class AmcRedfishEventLogPlugin(RedfishEventLogPlugin): + """Collect AMC Redfish EventLog entries through the BMC SSH proxy. + + Same collection and analysis as RedfishEventLogPlugin; uses + RedfishSshProxyConnectionManager so both plugins can run in the same + node-scraper invocation (BMC EventLog on the direct connection, AMC + EventLog on the proxy). + """ + + CONNECTION_TYPE = RedfishSshProxyConnectionManager # type: ignore[assignment] diff --git a/nodescraper/plugins/ooband/redfish_inventory/__init__.py b/nodescraper/plugins/ooband/redfish_inventory/__init__.py new file mode 100644 index 00000000..c3644f93 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/__init__.py @@ -0,0 +1,41 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from .analyzer_args import RedfishInventoryAnalyzerArgs +from .collector_args import RedfishInventoryCollectorArgs +from .inventory_analyzer import RedfishInventoryAnalyzer +from .inventory_collector import RedfishInventoryCollector +from .inventory_data import RedfishInventoryDataModel +from .inventory_plugin import AmcRedfishInventoryPlugin, RedfishInventoryPlugin + +__all__ = [ + "AmcRedfishInventoryPlugin", + "RedfishInventoryAnalyzer", + "RedfishInventoryAnalyzerArgs", + "RedfishInventoryCollector", + "RedfishInventoryCollectorArgs", + "RedfishInventoryDataModel", + "RedfishInventoryPlugin", +] diff --git a/nodescraper/plugins/ooband/redfish_inventory/analyzer_args.py b/nodescraper/plugins/ooband/redfish_inventory/analyzer_args.py new file mode 100644 index 00000000..a3d0bcb7 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/analyzer_args.py @@ -0,0 +1,45 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class RedfishInventoryAnalyzerArgs(AnalyzerArgs): + """Analyzer args for comparing Redfish firmware inventory across targets.""" + + compare_firmware: bool = Field( + default=True, + description=( + "Compare FirmwareInventory across this plugin's connected targets. " + "Default true. A different Version, a member missing on a connected target, " + "or a missing Version (recorded as N/A) logs one FW warning per component. " + "The warning records Version, Status Health, Status State, SoftwareId, " + "Updateable, and Oem SVN. Ids are compared as published. " + "One connected target, a failed connection, or false skips the comparison. " + "--data loads one model and does not compare nodes." + ), + ) diff --git a/nodescraper/plugins/ooband/redfish_inventory/collector_args.py b/nodescraper/plugins/ooband/redfish_inventory/collector_args.py new file mode 100644 index 00000000..0659e873 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/collector_args.py @@ -0,0 +1,233 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Literal + +from pydantic import Field, field_validator + +from nodescraper.models import CollectorArgs + +BMC_COLLECTIONS = ("Chassis", "Managers", "TelemetryService", "UpdateService", "Systems") +BMC_LINKS = ( + "Assembly", + "PowerSubsystem", + "Power", + "FruDevices", + "MetricReports", + "MetricReportDefinitions", + "FirmwareInventory", +) +SMC_COLLECTIONS = ("Systems", "UpdateService") +SMC_LINKS = ("Memory", "Processors", "FirmwareInventory") +INVENTORY_SWITCHES = ( + "chassis", + "managers", + "assembly", + "power", + "fru", + "metrics", + "update_service", + "systems", + "memory", + "processors", +) +_ROOT_SWITCH = { + "Chassis": "chassis", + "Managers": "managers", + "TelemetryService": "metrics", + "UpdateService": "update_service", + "Systems": "systems", +} +_LINK_SWITCH = { + "Assembly": "assembly", + "PowerSubsystem": "power", + "Power": "power", + "FruDevices": "fru", + "MetricReports": "metrics", + "MetricReportDefinitions": "metrics", + "FirmwareInventory": "update_service", + "Memory": "memory", + "Processors": "processors", +} +_SWITCH_ROOTS: dict[str, tuple[str, ...]] = { + "chassis": ("Chassis",), + "managers": ("Managers",), + "metrics": ("TelemetryService",), + "update_service": ("UpdateService",), + "systems": ("Systems",), +} +_SWITCH_LINKS = { + "assembly": ("Assembly",), + "power": ("PowerSubsystem", "Power"), + "fru": ("FruDevices",), + "metrics": ("MetricReports", "MetricReportDefinitions"), + "update_service": ("FirmwareInventory",), + "memory": ("Memory",), + "processors": ("Processors",), +} + + +def _ordered_unique(names: tuple[str, ...]) -> list[str]: + """Return names with duplicates removed and the first occurrence kept. + + Args: + names: Collection or link names. + + Returns: + Names in first-seen order. + """ + return list(dict.fromkeys(names)) + + +def _apply_enabled( + names: list[str], + name_switches: dict[str, str], + forced: dict[str, tuple[str, ...]], + enabled: dict[str, bool], +) -> list[str]: + """Drop disabled inventory areas and add areas explicitly turned on. + + Args: + names: Profile or caller-selected Redfish property names. + name_switches: Property name to inventory switch. + forced: Property names added when a switch is true. + enabled: Switch overrides. A missing switch is left unchanged. + + Returns: + Property names to follow. + """ + kept: list[str] = [] + for name in names: + switch = name_switches.get(name) + if switch is not None and enabled.get(switch) is False: + continue + kept.append(name) + for switch, extra in forced.items(): + if enabled.get(switch) is True: + kept.extend(extra) + return _ordered_unique(tuple(kept)) + + +class RedfishInventoryCollectorArgs(CollectorArgs): + """Select standard Redfish inventory collections to read from the service root.""" + + profile: Literal["bmc", "smc", "all"] = Field( + default="all", + description="bmc reads chassis, managers, telemetry, firmware inventory, and systems. " + "smc reads systems, memory, processors, and firmware inventory. all reads both.", + ) + collections: list[str] = Field( + default_factory=list, + description="Extra service-root property names appended to the profile set. " + "Duplicates are ignored. Empty leaves the profile unchanged.", + ) + member_links: list[str] = Field( + default_factory=list, + description="Extra property names appended to the profile link set. " + "Duplicates are ignored. Empty leaves the profile unchanged.", + ) + max_pages: int = Field( + default=1, + ge=1, + le=10_000, + description="Maximum pages to keep for each collection, including the first page.", + ) + enabled: dict[str, bool] = Field( + default_factory=dict, + description="Turn inventory areas on or off. Omitted areas stay at the profile default, " + "and the default profile enables every area. update_service also follows " + "FirmwareInventory members. Keys: chassis, managers, assembly, power, " + "fru, metrics, update_service, systems, memory, processors.", + ) + + @field_validator("enabled") + @classmethod + def reject_unknown_switches(cls, value: dict[str, bool]) -> dict[str, bool]: + """Reject inventory switch names that this collector does not know. + + Args: + value: Requested enable flags. + + Returns: + The same flags when every key is known. + """ + unknown = sorted(set(value) - set(INVENTORY_SWITCHES)) + if unknown: + raise ValueError(f"unknown inventory switches: {', '.join(unknown)}") + return value + + def _profile_collections(self) -> list[str]: + """Return the service-root collections selected by the profile. + + Args: + None + + Returns: + Service-root property names. + """ + if self.profile == "bmc": + return list(BMC_COLLECTIONS) + if self.profile == "smc": + return list(SMC_COLLECTIONS) + return _ordered_unique(BMC_COLLECTIONS + SMC_COLLECTIONS) + + def _profile_links(self) -> list[str]: + """Return the resource links selected by the profile. + + Args: + None + + Returns: + Property names followed from collected resources. + """ + if self.profile == "bmc": + return list(BMC_LINKS) + if self.profile == "smc": + return list(SMC_LINKS) + return _ordered_unique(BMC_LINKS + SMC_LINKS) + + def selected_collections(self) -> list[str]: + """Return the service-root collections this run will follow. + + Args: + None + + Returns: + Profile names with collections appended. Duplicates are dropped. + """ + names = _ordered_unique(tuple(self._profile_collections() + list(self.collections))) + return _apply_enabled(names, _ROOT_SWITCH, _SWITCH_ROOTS, self.enabled) + + def selected_links(self) -> list[str]: + """Return the resource links this run will follow. + + Args: + None + + Returns: + Profile names with member_links appended. Duplicates are dropped. + """ + names = _ordered_unique(tuple(self._profile_links() + list(self.member_links))) + return _apply_enabled(names, _LINK_SWITCH, _SWITCH_LINKS, self.enabled) diff --git a/nodescraper/plugins/ooband/redfish_inventory/firmware_compare.py b/nodescraper/plugins/ooband/redfish_inventory/firmware_compare.py new file mode 100644 index 00000000..03bb34bb --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/firmware_compare.py @@ -0,0 +1,183 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Any, Optional, TypedDict + +from .inventory_data import RedfishInventoryDataModel + +_FIRMWARE_MEMBER = "/FirmwareInventory/" + + +class FirmwareComponent(TypedDict): + """Firmware fields recorded for one FirmwareInventory member.""" + + version: str + health: str + state: str + software_id: str + updateable: str + svn: str + + +class FirmwareMismatch(TypedDict): + """One firmware id whose version or presence is not the same on every target.""" + + firmware_id: str + targets: dict[str, Optional[FirmwareComponent]] + + +def _text_or_na(value: Any) -> str: + """Return a display string, using N/A when the value is missing. + + Args: + value: Field value from a firmware member. + + Returns: + The string value, or N/A when value is None or empty. + """ + if value is None or value == "": + return "N/A" + return str(value) + + +def _member_name(path: str) -> str: + """Return the FirmwareInventory member name from a Redfish path. + + Args: + path: Collected resource path. + + Returns: + The member name, or an empty string for collections and nested resources. + """ + clean = path.split("?", 1)[0].rstrip("/") + if _FIRMWARE_MEMBER not in clean: + return "" + tail = clean.split(_FIRMWARE_MEMBER, 1)[1] + if not tail or "/" in tail: + return "" + return tail + + +def _component(body: dict) -> FirmwareComponent: + """Read the firmware fields stored for one member. + + Args: + body: FirmwareInventory member payload. + + Returns: + Version, health, state, software id, updateable, and SVN. + """ + status = body.get("Status") + if not isinstance(status, dict): + status = {} + oem = body.get("Oem") + if not isinstance(oem, dict): + oem = {} + updateable = body.get("Updateable") + svn = oem.get("SVN") + return { + "version": _text_or_na(body.get("Version")), + "health": _text_or_na(status.get("Health")), + "state": _text_or_na(status.get("State")), + "software_id": _text_or_na(body.get("SoftwareId")), + "updateable": "N/A" if updateable is None else str(updateable), + "svn": "" if svn is None else str(svn), + } + + +def firmware_components( + model: RedfishInventoryDataModel, +) -> dict[str, FirmwareComponent]: + """Return FirmwareInventory members keyed by their raw id. + + Args: + model: Inventory collected from one target. + + Returns: + Firmware id to the recorded fields. A missing version is N/A. + """ + components: dict[str, FirmwareComponent] = {} + for path, body in model.responses.items(): + if not isinstance(path, str) or not isinstance(body, dict): + continue + member = _member_name(path) + if not member: + continue + raw_id = body.get("Id") + if isinstance(raw_id, str) and raw_id.strip(): + fw_id = raw_id.strip() + else: + fw_id = member + components[fw_id] = _component(body) + return components + + +def firmware_mismatches( + by_target: dict[str, dict[str, FirmwareComponent]], +) -> list[FirmwareMismatch]: + """Find firmware ids whose version or presence differs across targets. + + Args: + by_target: Target key to that target's firmware members. + + Returns: + Mismatches in firmware-id order. An absent member is included. + """ + names: set[str] = set() + for components in by_target.values(): + names.update(components) + mismatches: list[FirmwareMismatch] = [] + for fw_id in sorted(names): + rows: dict[str, Optional[FirmwareComponent]] = {} + versions: list[str] = [] + missing = False + for target in sorted(by_target): + record = by_target[target].get(fw_id) + rows[target] = record + if record is None: + missing = True + else: + versions.append(record["version"]) + if missing or len(set(versions)) > 1: + mismatches.append({"firmware_id": fw_id, "targets": rows}) + return mismatches + + +def firmware_target_text(record: Optional[FirmwareComponent]) -> str: + """Format one target's firmware fields for a warning. + + Args: + record: Fields for this target, or None when the member is absent. + + Returns: + missing, or the version and the other recorded fields. + """ + if record is None: + return "missing" + return ( + f"version={record['version']} health={record['health']} " + f"state={record['state']} software_id={record['software_id']} " + f"updateable={record['updateable']} svn={record['svn']}" + ) diff --git a/nodescraper/plugins/ooband/redfish_inventory/inventory_analyzer.py b/nodescraper/plugins/ooband/redfish_inventory/inventory_analyzer.py new file mode 100644 index 00000000..4a47eb04 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/inventory_analyzer.py @@ -0,0 +1,166 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import RedfishInventoryAnalyzerArgs +from .firmware_compare import ( + firmware_components, + firmware_mismatches, + firmware_target_text, +) +from .inventory_data import RedfishInventoryDataModel + + +class RedfishInventoryAnalyzer( + DataAnalyzer[RedfishInventoryDataModel, RedfishInventoryAnalyzerArgs] +): + """Compare FirmwareInventory members across the targets this plugin collected.""" + + DATA_MODEL = RedfishInventoryDataModel + + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "analysis_args.compare_firmware defaults to true and compares FirmwareInventory " + "across the targets this plugin connected.", + "RedfishInventoryPlugin compares its direct Redfish targets. " + "AmcRedfishInventoryPlugin compares its SSH-proxy targets. The two are not mixed.", + "A different Version, a member missing on a connected target, or a missing or empty " + "Version (recorded as N/A) logs one FW warning per component.", + "Each warning records Version, Status Health, Status State, SoftwareId, Updateable, " + "and Oem SVN. The component id is the published Id and is not normalized.", + "Warnings are written to the analyzer events.json. result.json has the summary count. " + "The console summary is that count, not the per-component table.", + "Fewer than two connected targets, a target that failed to connect, or " + "compare_firmware false does not warn. --data loads one model and does not compare nodes.", + "When no FirmwareInventory members were collected, one warning says the comparison did not run.", + ) + + def analyze_data( + self, + data: RedfishInventoryDataModel, + args: Optional[RedfishInventoryAnalyzerArgs] = None, + ) -> TaskResult: + """Compare firmware for a single collected inventory model. + + Args: + data: Inventory collected from one target. + args: Analyzer arguments. Defaults to comparing firmware. + + Returns: + TaskResult: OK because one model has no other target to compare. + """ + self._apply_firmware_comparison({"target": data}, args or RedfishInventoryAnalyzerArgs()) + return self.result + + def compare_firmware( + self, + targets: dict[str, RedfishInventoryDataModel], + args: Optional[RedfishInventoryAnalyzerArgs] = None, + ) -> TaskResult: + """Compare firmware across every target collected by one plugin. + + Args: + targets: Target key to the inventory collected from that target. + args: Analyzer arguments. Defaults to comparing firmware. + + Returns: + TaskResult: WARNING when firmware differs, otherwise OK or NOT_RAN. + """ + self.result = self._init_result() + self._apply_firmware_comparison(targets, args or RedfishInventoryAnalyzerArgs()) + result = self.result + result.finalize(self.logger) + sample = next(iter(targets.values()), None) + self._run_hooks(result, data=sample) + return result + + def _apply_firmware_comparison( + self, + targets: dict[str, RedfishInventoryDataModel], + args: RedfishInventoryAnalyzerArgs, + ) -> None: + """Set the analysis result for a firmware comparison. + + Args: + targets: Target key to the inventory collected from that target. + args: Analyzer arguments. + + Returns: + None. + """ + if not args.compare_firmware: + self.result.status = ExecutionStatus.OK + self.result.message = "Firmware comparison disabled" + return + if not targets: + self.result.status = ExecutionStatus.NOT_RAN + self.result.message = "No Redfish inventory available to compare" + return + if len(targets) < 2: + self.result.status = ExecutionStatus.OK + self.result.message = "Firmware comparison needs at least two connected targets" + return + + by_target = {key: firmware_components(model) for key, model in targets.items()} + if not any(by_target.values()): + self._log_event( + category=EventCategory.FW, + description=( + "Firmware comparison did not run: no FirmwareInventory members were collected" + ), + priority=EventPriority.WARNING, + console_log=True, + ) + self.result.status = ExecutionStatus.WARNING + self.result.message = "No FirmwareInventory members were collected" + return + + mismatches = firmware_mismatches(by_target) + for mismatch in mismatches: + parts = [ + f"{target}={firmware_target_text(record)}" + for target, record in mismatch["targets"].items() + ] + description = f"Firmware {mismatch['firmware_id']} differs: " + ", ".join(parts) + self._log_event( + category=EventCategory.FW, + description=description, + data={ + "firmware_id": mismatch["firmware_id"], + "targets": mismatch["targets"], + }, + priority=EventPriority.WARNING, + console_log=True, + ) + if mismatches: + self.result.status = ExecutionStatus.WARNING + self.result.message = f"Firmware differs for {len(mismatches)} component(s)" + return + self.result.status = ExecutionStatus.OK + self.result.message = f"Firmware versions match across {len(targets)} targets" diff --git a/nodescraper/plugins/ooband/redfish_inventory/inventory_collector.py b/nodescraper/plugins/ooband/redfish_inventory/inventory_collector.py new file mode 100644 index 00000000..5efb8a95 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/inventory_collector.py @@ -0,0 +1,258 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Any, Optional + +from nodescraper.base import RedfishDataCollector +from nodescraper.connection.redfish import RF_MEMBERS, RF_MEMBERS_NEXT_LINK, RF_ODATA_ID +from nodescraper.connection.redfish.ssh_proxy_connection import ( + SshProxyRedfishConnection, +) +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.models import TaskResult + +from .collector_args import RedfishInventoryCollectorArgs +from .inventory_data import RedfishInventoryDataModel + + +def _reference_path(value: Any) -> str: + """Return a fetchable Redfish path from a link value. + + Args: + value: An OData id string or a dict containing one. + + Returns: + The absolute path, or an empty string when the value is not fetchable. + """ + if isinstance(value, dict): + value = value.get(RF_ODATA_ID) + if not isinstance(value, str): + return "" + text = value.strip() + if not text or "#" in text or not text.startswith("/"): + return "" + return text.rstrip("/") or "/" + + +def _member_paths(body: dict) -> list[str]: + """Return member resource paths from a collection body. + + Args: + body: A Redfish collection payload. + + Returns: + Absolute member paths. + """ + paths: list[str] = [] + for item in body.get(RF_MEMBERS) or []: + if isinstance(item, dict): + path = _reference_path(item.get(RF_ODATA_ID)) + if path: + paths.append(path) + return paths + + +def _named_path(body: dict, name: str) -> str: + """Return one named link from a Redfish resource. + + Args: + body: A Redfish resource payload. + name: Property that may hold an OData link. + + Returns: + The linked path, or an empty string when the property is absent. + """ + return _reference_path(body.get(name)) + + +class RedfishInventoryCollector( + RedfishDataCollector[RedfishInventoryDataModel, RedfishInventoryCollectorArgs] +): + """Collect standard Redfish inventory resources without product-specific validation.""" + + DATA_MODEL = RedfishInventoryDataModel + + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "GET the Redfish service root and follow standard collection links such as Chassis, " + "Managers, Systems, TelemetryService, and UpdateService.", + "Expand collection members and follow Assembly, PowerSubsystem, FruDevices, Memory, " + "Processors, FirmwareInventory, and metric-report links when the service publishes them.", + "collection_args.collections and collection_args.member_links append extra property " + "names to that profile set. Names already in the set are ignored.", + "Uses the active Redfish connection, including a direct session or the SSH proxy.", + "An SSH-proxy HTTP 404 for a listed collection member is an empty slot and is not a failure.", + ) + + def collect_data( + self, args: Optional[RedfishInventoryCollectorArgs] = None + ) -> tuple[TaskResult, Optional[RedfishInventoryDataModel]]: + """Collect the selected standard Redfish inventory resources. + + Args: + args: Profile and optional collection overrides. + + Returns: + The task result and collected responses, or None when the service root cannot be read. + """ + collect_args = args or RedfishInventoryCollectorArgs() + api_root = getattr(self.connection, "api_root", "redfish/v1") + root_path = "/" + str(api_root).strip("/") + responses: dict[str, dict] = {} + seen: set[str] = set() + found_collections = 0 + missing: list[str] = [] + + root = self._fetch(root_path, responses, seen) + if root is None: + self.result.message = "No Redfish inventory could be read" + self.result.status = ExecutionStatus.ERROR + return self.result, None + + for name in collect_args.selected_collections(): + collection_path = _named_path(root, name) + if not collection_path: + missing.append(name) + continue + found_collections += 1 + for body in self._fetch_pages(collection_path, responses, seen, collect_args.max_pages): + self._follow_links(body, responses, seen, collect_args) + for member_path in _member_paths(body): + member = self._fetch(member_path, responses, seen, allow_missing=True) + if member is not None: + self._follow_links(member, responses, seen, collect_args) + + for name in missing: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Redfish service root has no {name} link", + priority=EventPriority.WARNING, + console_log=False, + ) + + data = RedfishInventoryDataModel(responses=responses) + if found_collections == 0: + self.result.message = "No requested Redfish inventory collections were present" + self.result.status = ExecutionStatus.WARNING + return self.result, data + + self.result.message = f"Collected {len(responses)} Redfish inventory resource(s)" + self.result.status = ExecutionStatus.OK + return self.result, data + + def _fetch( + self, + path: str, + responses: dict[str, dict], + seen: set[str], + allow_missing: bool = False, + ) -> Optional[dict]: + """GET one Redfish path and store a successful object body. + + Args: + path: Absolute Redfish path. + responses: Collected bodies keyed by path. + seen: Paths already requested. + allow_missing: When True, an SSH-proxy HTTP 404 is an empty slot and is not a warning. + + Returns: + The JSON object, or None when the GET did not return one. + """ + if path in seen: + return responses.get(path) + seen.add(path) + res = self._run_redfish_get(path, log_artifact=True) + if res.success and isinstance(res.data, dict): + responses[path] = res.data + self.logger.info("Collected Redfish resource %s", path) + return res.data + if ( + allow_missing + and res.status_code == 404 + and isinstance(self.connection, SshProxyRedfishConnection) + ): + return None + self._log_event( + category=EventCategory.RUNTIME, + description=f"Redfish GET failed for {path}: {res.error or 'unknown'}", + priority=EventPriority.WARNING, + console_log=True, + ) + return None + + def _fetch_pages( + self, + path: str, + responses: dict[str, dict], + seen: set[str], + max_pages: int, + ) -> list[dict]: + """GET a resource and additional collection pages up to max_pages. + + Args: + path: First page path. + responses: Collected bodies keyed by path. + seen: Paths already requested. + max_pages: Maximum pages to keep. + + Returns: + Bodies that were read, in page order. + """ + bodies: list[dict] = [] + current = path + pages = 0 + while current and pages < max_pages: + body = self._fetch(current, responses, seen) + pages += 1 + if body is None: + break + bodies.append(body) + current = _reference_path(body.get(RF_MEMBERS_NEXT_LINK)) + return bodies + + def _follow_links( + self, + body: dict, + responses: dict[str, dict], + seen: set[str], + args: RedfishInventoryCollectorArgs, + ) -> None: + """Follow configured links and expand one level of their members. + + Args: + body: Resource that may contain the configured links. + responses: Collected bodies keyed by path. + seen: Paths already requested. + args: Collector args selecting which links to follow. + + Returns: + None + """ + for name in args.selected_links(): + link = _named_path(body, name) + if not link: + continue + for linked in self._fetch_pages(link, responses, seen, args.max_pages): + for member_path in _member_paths(linked): + self._fetch(member_path, responses, seen, allow_missing=True) diff --git a/nodescraper/plugins/ooband/redfish_inventory/inventory_data.py b/nodescraper/plugins/ooband/redfish_inventory/inventory_data.py new file mode 100644 index 00000000..a9cb930f --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/inventory_data.py @@ -0,0 +1,34 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import DataModel + + +class RedfishInventoryDataModel(DataModel): + """Collected standard Redfish inventory responses, keyed by resource path.""" + + responses: dict[str, dict] = Field(default_factory=dict) diff --git a/nodescraper/plugins/ooband/redfish_inventory/inventory_plugin.py b/nodescraper/plugins/ooband/redfish_inventory/inventory_plugin.py new file mode 100644 index 00000000..3f119ab8 --- /dev/null +++ b/nodescraper/plugins/ooband/redfish_inventory/inventory_plugin.py @@ -0,0 +1,170 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Any, Optional, Union + +from nodescraper.base import OOBandDataPlugin +from nodescraper.connection.redfish import ( + RedfishSshProxyConnectionManager, + collected_multi_target_data, +) +from nodescraper.enums import EventPriority +from nodescraper.models import TaskResult + +from .analyzer_args import RedfishInventoryAnalyzerArgs +from .collector_args import RedfishInventoryCollectorArgs +from .inventory_analyzer import RedfishInventoryAnalyzer +from .inventory_collector import RedfishInventoryCollector +from .inventory_data import RedfishInventoryDataModel + + +class RedfishInventoryPlugin( + OOBandDataPlugin[ + RedfishInventoryDataModel, + RedfishInventoryCollectorArgs, + RedfishInventoryAnalyzerArgs, + ] +): + """Collect portable BMC and SMC Redfish inventory over the native Redfish connection. + + BMC coverage is Chassis, Managers, Assembly, PowerSubsystem, FruDevices, telemetry + metric reports, UpdateService, FirmwareInventory, and Systems. SMC coverage is + ComputerSystem, Memory, Processors, and FirmwareInventory. Every area is enabled + unless collection_args.enabled turns it off. + Collection follows links the service publishes and does not apply product member expectations. + When two or more targets connect, analysis compares FirmwareInventory across those + targets. compare_firmware defaults to true. A different version or a member missing on + a connected target logs an FW warning. AmcRedfishInventoryPlugin runs that comparison + on its own targets. + + Keep these outside node-scraper core: IPMI, SSH commands and file capture, PLDM, + manufacturing-tool access, vendor-only BMC actions, BIOS and diagnostic logs, CPLD, + post codes, SOL, CPER, and product checks for chassis topology, ports, PCIe, sensors, + certificates, and firmware integrity. Non-Redfish fallbacks stay with the product + collector that owns them. + """ + + DATA_MODEL = RedfishInventoryDataModel + COLLECTOR = RedfishInventoryCollector + COLLECTOR_ARGS = RedfishInventoryCollectorArgs + ANALYZER = RedfishInventoryAnalyzer + ANALYZER_ARGS = RedfishInventoryAnalyzerArgs + + def analyze( + self, + max_event_priority_level: Optional[Union[EventPriority, str]] = EventPriority.CRITICAL, + analysis_args: Optional[Union[RedfishInventoryAnalyzerArgs, dict]] = None, + data: Optional[Any] = None, + ) -> TaskResult: + """Compare firmware once across every target this plugin collected. + + Args: + max_event_priority_level: Priority limit for events. + analysis_args: Analyzer arguments. Defaults to comparing firmware. + data: Pre-collected inventory for a single-target run. + + Returns: + TaskResult: WARNING when firmware differs across connected targets. + """ + if data: + self.data = data + args = self._analysis_args(analysis_args) + analyzer = RedfishInventoryAnalyzer( + system_info=self.system_info.model_copy(), + logger=self.logger, + max_event_priority_level=max_event_priority_level or EventPriority.CRITICAL, + parent=self.__class__.__name__, + task_result_hooks=self.task_result_hooks, + event_reporter=self.event_reporter, + session_id=self.session_id, + ) + self.analysis_result = analyzer.compare_firmware(self._firmware_targets(), args) + return self.analysis_result + + def _analysis_args( + self, analysis_args: Optional[Union[RedfishInventoryAnalyzerArgs, dict]] + ) -> RedfishInventoryAnalyzerArgs: + """Return analyzer args, applying the compare_firmware default when omitted. + + Args: + analysis_args: Caller-supplied analyzer arguments. + + Returns: + Validated analyzer arguments. + """ + if analysis_args is None: + return RedfishInventoryAnalyzerArgs() + if isinstance(analysis_args, dict): + return RedfishInventoryAnalyzerArgs.model_validate(analysis_args) + return analysis_args + + def _firmware_targets(self) -> dict[str, RedfishInventoryDataModel]: + """Return inventory models for the connected targets, or the single collected model. + + Args: + None + + Returns: + Target key to inventory model. + """ + models: dict[str, RedfishInventoryDataModel] = {} + for key, value in collected_multi_target_data(self.connection_manager).items(): + model = _as_inventory_model(value) + if model is not None: + models[str(key)] = model + if models: + return models + model = _as_inventory_model(self.data) + if model is None: + return {} + return {"target": model} + + +def _as_inventory_model(value: Any) -> Optional[RedfishInventoryDataModel]: + """Return an inventory model when value is one or a dumped responses dict. + + Args: + value: Collected data for one target. + + Returns: + The inventory model, or None when value is not inventory. + """ + if isinstance(value, RedfishInventoryDataModel): + return value + if isinstance(value, dict) and isinstance(value.get("responses"), dict): + return RedfishInventoryDataModel.model_validate(value) + return None + + +class AmcRedfishInventoryPlugin(RedfishInventoryPlugin): + """Collect the same Redfish inventory through the BMC SSH proxy. + + Pair this with RedfishInventoryPlugin so one run reads the BMC on the direct + Redfish connection and the AMC on RedfishSshProxyConnectionManager. collection_args + and analysis_args are the same for both plugins. Firmware comparison uses only the + targets this plugin connected. + """ + + CONNECTION_TYPE = RedfishSshProxyConnectionManager # type: ignore[assignment] diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/__init__.py b/nodescraper/plugins/ooband/redfish_oem_diag/__init__.py index 1a761c81..6ff220a9 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/__init__.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/__init__.py @@ -1,22 +1,22 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -############################################################################### -from .analyzer_args import RedfishOemDiagAnalyzerArgs -from .collector_args import RedfishOemDiagCollectorArgs -from .oem_diag_analyzer import RedfishOemDiagAnalyzer -from .oem_diag_collector import RedfishOemDiagCollector -from .oem_diag_data import RedfishOemDiagDataModel -from .oem_diag_plugin import RedfishOemDiagPlugin - -__all__ = [ - "RedfishOemDiagAnalyzer", - "RedfishOemDiagAnalyzerArgs", - "RedfishOemDiagCollector", - "RedfishOemDiagCollectorArgs", - "RedfishOemDiagDataModel", - "RedfishOemDiagPlugin", -] +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +############################################################################### +from .analyzer_args import RedfishOemDiagAnalyzerArgs +from .collector_args import RedfishOemDiagCollectorArgs +from .oem_diag_analyzer import RedfishOemDiagAnalyzer +from .oem_diag_collector import RedfishOemDiagCollector +from .oem_diag_data import RedfishOemDiagDataModel +from .oem_diag_plugin import RedfishOemDiagPlugin + +__all__ = [ + "RedfishOemDiagAnalyzer", + "RedfishOemDiagAnalyzerArgs", + "RedfishOemDiagCollector", + "RedfishOemDiagCollectorArgs", + "RedfishOemDiagDataModel", + "RedfishOemDiagPlugin", +] diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/analyzer_args.py b/nodescraper/plugins/ooband/redfish_oem_diag/analyzer_args.py index 60759a94..9f965a33 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/analyzer_args.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/analyzer_args.py @@ -1,37 +1,37 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from pydantic import Field - -from nodescraper.models import AnalyzerArgs - - -class RedfishOemDiagAnalyzerArgs(AnalyzerArgs): - """Analyzer args for Redfish OEM diagnostic log results.""" - - require_all_success: bool = Field( - default=False, - description="If True, analysis fails when any OEM type collection failed.", - ) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pydantic import Field + +from nodescraper.models import AnalyzerArgs + + +class RedfishOemDiagAnalyzerArgs(AnalyzerArgs): + """Analyzer args for Redfish OEM diagnostic log results.""" + + require_all_success: bool = Field( + default=False, + description="If True, analysis fails when any OEM type collection failed.", + ) diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/collector_args.py b/nodescraper/plugins/ooband/redfish_oem_diag/collector_args.py index 7eb6bac7..13737e5e 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/collector_args.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/collector_args.py @@ -1,65 +1,72 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from __future__ import annotations - -from typing import Optional - -from pydantic import Field, model_validator - -from nodescraper.models import CollectorArgs - -DEFAULT_TASK_TIMEOUT_S = 1800 - - -class RedfishOemDiagCollectorArgs(CollectorArgs): - """Collector/analyzer args for Redfish OEM diagnostic log collection.""" - - log_service_path: str = Field( - default="redfish/v1/Systems/UBB/LogServices/DiagLogs", - description="Redfish path to the LogService (e.g. DiagLogs).", - ) - oem_diagnostic_types_allowable: Optional[list[str]] = Field( - default=None, - description="Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagnostic_types when empty.", - ) - oem_diagnostic_types: list[str] = Field( - default_factory=list, - description="OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.", - ) - task_timeout_s: int = Field( - default=DEFAULT_TASK_TIMEOUT_S, - ge=1, - le=3600, - description="Max seconds to wait for each BMC task.", - ) - - @model_validator(mode="after") - def _default_oem_diagnostic_types(self) -> RedfishOemDiagCollectorArgs: - if not self.oem_diagnostic_types and self.oem_diagnostic_types_allowable: - return self.model_copy( - update={"oem_diagnostic_types": list(self.oem_diagnostic_types_allowable)} - ) - return self +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Optional + +from pydantic import Field, model_validator + +from nodescraper.models import CollectorArgs + +DEFAULT_TASK_TIMEOUT_S = 1800 + + +class RedfishOemDiagCollectorArgs(CollectorArgs): + """Collector/analyzer args for Redfish OEM diagnostic log collection.""" + + log_service_path: str = Field( + default="redfish/v1/Systems/UBB/LogServices/DiagLogs", + description="Redfish path to the LogService (e.g. DiagLogs).", + ) + oem_diagnostic_types_allowable: Optional[list[str]] = Field( + default=None, + description="Allowable OEM diagnostic types for this architecture/BMC. When set, used for validation and as default for oem_diagnostic_types when empty.", + ) + oem_diagnostic_types: list[str] = Field( + default_factory=list, + description="OEM diagnostic types to collect. When empty and oem_diagnostic_types_allowable is set, defaults to that list.", + ) + task_timeout_s: int = Field( + default=DEFAULT_TASK_TIMEOUT_S, + ge=1, + le=3600, + description="Max seconds to wait for each BMC task.", + ) + clear_logs_after_collection: bool = Field( + default=False, + description=( + "If True, POST LogService.ClearLog to the configured log_service_path after a " + "successful collection. Requires system_interaction_level >= INTERACTIVE." + ), + ) + + @model_validator(mode="after") + def _default_oem_diagnostic_types(self) -> RedfishOemDiagCollectorArgs: + if not self.oem_diagnostic_types and self.oem_diagnostic_types_allowable: + return self.model_copy( + update={"oem_diagnostic_types": list(self.oem_diagnostic_types_allowable)} + ) + return self diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_analyzer.py b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_analyzer.py index 11aaa1e8..62b8d33d 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_analyzer.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_analyzer.py @@ -1,74 +1,74 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Optional - -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.interfaces import DataAnalyzer -from nodescraper.models import TaskResult - -from .analyzer_args import RedfishOemDiagAnalyzerArgs -from .oem_diag_data import RedfishOemDiagDataModel - - -class RedfishOemDiagAnalyzer(DataAnalyzer[RedfishOemDiagDataModel, RedfishOemDiagAnalyzerArgs]): - """Analyzes Redfish OEM diagnostic log collection results.""" - - DATA_MODEL = RedfishOemDiagDataModel - - DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( - "Summarizes success/failure per OEM diagnostic type from collected results.", - "When analysis_args.require_all_success is true, fails the run if any type failed collection.", - ) - - def analyze_data( - self, - data: RedfishOemDiagDataModel, - args: Optional[RedfishOemDiagAnalyzerArgs] = None, - ) -> TaskResult: - """Check collection results; optionally fail if any type failed.""" - if not data.results: - self.result.message = "No OEM diagnostic results to analyze" - self.result.status = ExecutionStatus.NOT_RAN - return self.result - - failed = [t for t, r in data.results.items() if not r.success] - success_count = len(data.results) - len(failed) - - if args and args.require_all_success and failed: - self._log_event( - category=EventCategory.RUNTIME, - description=f"OEM diag types failed: {failed}", - data={t: data.results[t].error for t in failed}, - priority=EventPriority.ERROR, - console_log=True, - ) - self.result.message = f"OEM diag: {len(failed)} type(s) failed: {failed}" - self.result.status = ExecutionStatus.ERROR - return self.result - - self.result.message = f"OEM diag: {success_count}/{len(data.results)} types collected" - self.result.status = ExecutionStatus.OK - return self.result +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analyzer_args import RedfishOemDiagAnalyzerArgs +from .oem_diag_data import RedfishOemDiagDataModel + + +class RedfishOemDiagAnalyzer(DataAnalyzer[RedfishOemDiagDataModel, RedfishOemDiagAnalyzerArgs]): + """Analyzes Redfish OEM diagnostic log collection results.""" + + DATA_MODEL = RedfishOemDiagDataModel + + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "Summarizes success/failure per OEM diagnostic type from collected results.", + "When analysis_args.require_all_success is true, fails the run if any type failed collection.", + ) + + def analyze_data( + self, + data: RedfishOemDiagDataModel, + args: Optional[RedfishOemDiagAnalyzerArgs] = None, + ) -> TaskResult: + """Check collection results; optionally fail if any type failed.""" + if not data.results: + self.result.message = "No OEM diagnostic results to analyze" + self.result.status = ExecutionStatus.NOT_RAN + return self.result + + failed = [t for t, r in data.results.items() if not r.success] + success_count = len(data.results) - len(failed) + + if args and args.require_all_success and failed: + self._log_event( + category=EventCategory.RUNTIME, + description=f"OEM diag types failed: {failed}", + data={t: data.results[t].error for t in failed}, + priority=EventPriority.ERROR, + console_log=True, + ) + self.result.message = f"OEM diag: {len(failed)} type(s) failed: {failed}" + self.result.status = ExecutionStatus.ERROR + return self.result + + self.result.message = f"OEM diag: {success_count}/{len(data.results)} types collected" + self.result.status = ExecutionStatus.OK + return self.result diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_collector.py b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_collector.py index f2e3d1d2..3f9d5228 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_collector.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_collector.py @@ -1,115 +1,155 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from pathlib import Path -from typing import Any, Optional - -from nodescraper.base import RedfishDataCollector -from nodescraper.connection.redfish import collect_oem_diagnostic_data -from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus -from nodescraper.models import TaskResult -from nodescraper.utils import pascal_to_snake - -from .collector_args import RedfishOemDiagCollectorArgs -from .oem_diag_data import OemDiagTypeResult, RedfishOemDiagDataModel - - -class RedfishOemDiagCollector( - RedfishDataCollector[RedfishOemDiagDataModel, RedfishOemDiagCollectorArgs] -): - """Collects Redfish OEM diagnostic logs (e.g. JournalControl, AllLogs) via LogService.CollectDiagnosticData.""" - - DATA_MODEL = RedfishOemDiagDataModel - - DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( - "Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types " - "(collection_args.log_service_path selects the LogService).", - "Optional binary archives under the plugin log path when log_path is set.", - ) - - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.log_path = kwargs.pop("log_path", None) - super().__init__(*args, **kwargs) - - def collect_data( - self, args: Optional[RedfishOemDiagCollectorArgs] = None - ) -> tuple[TaskResult, Optional[RedfishOemDiagDataModel]]: - """Run OEM diagnostic collection for each type in args.oem_diagnostic_types.""" - if args is None: - args = RedfishOemDiagCollectorArgs() - types_to_collect = list(args.oem_diagnostic_types) if args.oem_diagnostic_types else [] - if not types_to_collect: - self.result.message = "No OEM diagnostic types configured" - self.result.status = ExecutionStatus.NOT_RAN - return self.result, None - - if self.log_path: - output_dir = ( - Path(self.log_path) - / pascal_to_snake(self.parent or "") - / pascal_to_snake(self.__class__.__name__) - / "diag_logs" - ).resolve() - else: - output_dir = None - - if output_dir is not None: - output_dir.mkdir(parents=True, exist_ok=True) - self.logger.info( - "(RedfishOemDiagPlugin) Diagnostic archives (e.g. *.tar.xz) will be written to: %s", - output_dir, - ) - - results: dict[str, OemDiagTypeResult] = {} - validate = bool(args.oem_diagnostic_types_allowable) - gathered = [] - for oem_type in types_to_collect: - log_bytes, metadata, err = collect_oem_diagnostic_data( - self.connection, - log_service_path=args.log_service_path, - oem_diagnostic_type=oem_type, - task_timeout_s=args.task_timeout_s, - output_dir=output_dir, - validate_type=validate, - allowed_types=args.oem_diagnostic_types_allowable, - logger=self.logger, - ) - gathered.append((oem_type, log_bytes, metadata, err)) - for oem_type, _log_bytes, metadata, err in gathered: - if err: - self._log_event( - category=EventCategory.RUNTIME, - description=f"OEM diag {oem_type!r}: {err}", - priority=EventPriority.WARNING, - console_log=True, - ) - results[oem_type] = OemDiagTypeResult(success=False, error=err, metadata=None) - else: - results[oem_type] = OemDiagTypeResult(success=True, error=None, metadata=metadata) - - success_count = sum(1 for r in results.values() if r.success) - self.result.message = f"OEM diag: {success_count}/{len(results)} types collected" - self.result.status = ExecutionStatus.OK if success_count else ExecutionStatus.ERROR - return self.result, RedfishOemDiagDataModel(results=results) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from pathlib import Path +from typing import Optional + +from nodescraper.base import RedfishDataCollector +from nodescraper.connection.redfish import collect_oem_diagnostic_data +from nodescraper.connection.redfish.redfish_clear_log import ( + _parse_clear_endpoint, + clear_redfish_logs, +) +from nodescraper.enums import ( + EventCategory, + EventPriority, + ExecutionStatus, + SystemInteractionLevel, +) +from nodescraper.models import TaskResult + +from .collector_args import RedfishOemDiagCollectorArgs +from .oem_diag_data import OemDiagTypeResult, RedfishOemDiagDataModel + + +class RedfishOemDiagCollector( + RedfishDataCollector[RedfishOemDiagDataModel, RedfishOemDiagCollectorArgs] +): + """Collects Redfish OEM diagnostic logs (e.g. JournalControl, AllLogs) via LogService.CollectDiagnosticData.""" + + DATA_MODEL = RedfishOemDiagDataModel + + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "Redfish LogService.CollectDiagnosticData for each entry in collection_args.oem_diagnostic_types " + "(collection_args.log_service_path selects the LogService).", + "Optional binary archives under the plugin log path when log_path is set.", + ) + + def collect_data( + self, args: Optional[RedfishOemDiagCollectorArgs] = None + ) -> tuple[TaskResult, Optional[RedfishOemDiagDataModel]]: + """Run OEM diagnostic collection for each type in args.oem_diagnostic_types.""" + if args is None: + args = RedfishOemDiagCollectorArgs() + types_to_collect = list(args.oem_diagnostic_types) if args.oem_diagnostic_types else [] + if not types_to_collect: + self.result.message = "No OEM diagnostic types configured" + self.result.status = ExecutionStatus.NOT_RAN + return self.result, None + + if self.log_path: + output_dir = (Path(self.log_path) / "diag_logs").resolve() + self.logger.info( + "(RedfishOemDiagPlugin) Diagnostic archives (e.g. *.tar.xz) will be written to: %s", + output_dir, + ) + else: + output_dir = None + + results: dict[str, OemDiagTypeResult] = {} + validate = bool(args.oem_diagnostic_types_allowable) + gathered = [] + for oem_type in types_to_collect: + log_bytes, metadata, err = collect_oem_diagnostic_data( + self.connection, + log_service_path=args.log_service_path, + oem_diagnostic_type=oem_type, + task_timeout_s=args.task_timeout_s, + output_dir=output_dir, + validate_type=validate, + allowed_types=args.oem_diagnostic_types_allowable, + logger=self.logger, + ) + gathered.append((oem_type, log_bytes, metadata, err)) + for oem_type, _log_bytes, metadata, err in gathered: + if err: + self._log_event( + category=EventCategory.RUNTIME, + description=f"OEM diag {oem_type!r}: error", + priority=EventPriority.WARNING, + data={"response_body": err}, + console_log=True, + ) + results[oem_type] = OemDiagTypeResult(success=False, error=err, metadata=None) + else: + results[oem_type] = OemDiagTypeResult(success=True, error=None, metadata=metadata) + + success_count = sum(1 for r in results.values() if r.success) + self.result.message = f"OEM diag: {success_count}/{len(results)} types collected" + self.result.status = ExecutionStatus.OK if success_count else ExecutionStatus.ERROR + + if ( + args.clear_logs_after_collection + and success_count > 0 + and self.system_interaction_level >= SystemInteractionLevel.INTERACTIVE + ): + self._clear_log_service(args.log_service_path) + elif args.clear_logs_after_collection and success_count == 0: + self.logger.debug("Skipping log clear: no OEM diag types collected successfully") + elif args.clear_logs_after_collection: + self.logger.debug( + "Skipping log clear: system_interaction_level is %s (requires INTERACTIVE)", + self.system_interaction_level.name, + ) + + return self.result, RedfishOemDiagDataModel(results=results) + + def _clear_log_service(self, log_service_path: str) -> None: + """POST LogService.ClearLog to the configured log service if it supports the action. + + Args: + log_service_path: Redfish path to the LogService. + """ + svc_res = self._run_redfish_get(log_service_path, log_artifact=False) + if not svc_res.success or not svc_res.data: + self.logger.debug("Skipping log clear: could not GET log service %s", log_service_path) + return + endpoint = _parse_clear_endpoint(svc_res.data) + if endpoint is None: + self.logger.debug("LogService %s does not advertise ClearLog action", log_service_path) + return + clear_results = clear_redfish_logs(self.connection, [endpoint], self.logger) + if clear_results and clear_results[0].success: + self.result.message += "; cleared log store" + elif clear_results and not clear_results[0].success: + self._log_event( + category=EventCategory.RUNTIME, + description=f"Failed to clear log store {endpoint.path}: {clear_results[0].error}", + priority=EventPriority.WARNING, + console_log=True, + ) + if self.result.status == ExecutionStatus.OK: + self.result.status = ExecutionStatus.WARNING + self.result.message += "; log clear failed" diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_data.py b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_data.py index db3f5b59..8e3affe1 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_data.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_data.py @@ -1,44 +1,44 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from typing import Any, Optional - -from pydantic import Field - -from nodescraper.models import DataModel - - -class OemDiagTypeResult(DataModel): - """Result of collecting one OEM diagnostic log type (no raw bytes; JSON-serializable).""" - - success: bool = False - error: Optional[str] = None - metadata: Optional[dict[str, Any]] = None - - -class RedfishOemDiagDataModel(DataModel): - """Collected Redfish OEM diagnostic log results: OEM type -> result (success, error, metadata).""" - - results: dict[str, OemDiagTypeResult] = Field(default_factory=dict) +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Any, Optional + +from pydantic import Field + +from nodescraper.models import DataModel + + +class OemDiagTypeResult(DataModel): + """Result of collecting one OEM diagnostic log type (no raw bytes; JSON-serializable).""" + + success: bool = False + error: Optional[str] = None + metadata: Optional[dict[str, Any]] = None + + +class RedfishOemDiagDataModel(DataModel): + """Collected Redfish OEM diagnostic log results: OEM type -> result (success, error, metadata).""" + + results: dict[str, OemDiagTypeResult] = Field(default_factory=dict) diff --git a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_plugin.py b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_plugin.py index 6fd2ee5a..2bdf8797 100644 --- a/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_plugin.py +++ b/nodescraper/plugins/ooband/redfish_oem_diag/oem_diag_plugin.py @@ -1,52 +1,52 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from nodescraper.base import OOBandDataPlugin - -from .analyzer_args import RedfishOemDiagAnalyzerArgs -from .collector_args import RedfishOemDiagCollectorArgs -from .oem_diag_analyzer import RedfishOemDiagAnalyzer -from .oem_diag_collector import RedfishOemDiagCollector -from .oem_diag_data import RedfishOemDiagDataModel - - -class RedfishOemDiagPlugin( - OOBandDataPlugin[ - RedfishOemDiagDataModel, - RedfishOemDiagCollectorArgs, - RedfishOemDiagAnalyzerArgs, - ] -): - """Collect Redfish OEM diagnostic logs (e.g. JournalControl, AllLogs, Dmesg) via LogService.CollectDiagnosticData. - - Uses RedfishConnectionManager. Configure log_service_path, oem_diagnostic_types (and optional output_dir) - in collection_args; use analysis_args.require_all_success to fail if any type fails. - """ - - DATA_MODEL = RedfishOemDiagDataModel - COLLECTOR = RedfishOemDiagCollector - ANALYZER = RedfishOemDiagAnalyzer - COLLECTOR_ARGS = RedfishOemDiagCollectorArgs - ANALYZER_ARGS = RedfishOemDiagAnalyzerArgs +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.base import OOBandDataPlugin + +from .analyzer_args import RedfishOemDiagAnalyzerArgs +from .collector_args import RedfishOemDiagCollectorArgs +from .oem_diag_analyzer import RedfishOemDiagAnalyzer +from .oem_diag_collector import RedfishOemDiagCollector +from .oem_diag_data import RedfishOemDiagDataModel + + +class RedfishOemDiagPlugin( + OOBandDataPlugin[ + RedfishOemDiagDataModel, + RedfishOemDiagCollectorArgs, + RedfishOemDiagAnalyzerArgs, + ] +): + """Collect Redfish OEM diagnostic logs (e.g. JournalControl, AllLogs, Dmesg) via LogService.CollectDiagnosticData. + + Uses RedfishConnectionManager. Configure log_service_path, oem_diagnostic_types (and optional output_dir) + in collection_args; use analysis_args.require_all_success to fail if any type fails. + """ + + DATA_MODEL = RedfishOemDiagDataModel + COLLECTOR = RedfishOemDiagCollector + ANALYZER = RedfishOemDiagAnalyzer + COLLECTOR_ARGS = RedfishOemDiagCollectorArgs + ANALYZER_ARGS = RedfishOemDiagAnalyzerArgs diff --git a/nodescraper/plugins/serviceability/afid_events.py b/nodescraper/plugins/serviceability/afid_events.py index 645d9177..6261f262 100644 --- a/nodescraper/plugins/serviceability/afid_events.py +++ b/nodescraper/plugins/serviceability/afid_events.py @@ -98,12 +98,32 @@ def _afid_event_from_cper_slot(unit: str, payload: Any) -> Optional[AfidEvent]: ) +def _oem_mappings(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Return Oem objects on a log entry, including one nested under CPER. + + Args: + payload: Redfish log member or switch event. + + Returns: + Oem mappings that may contain AMDFieldIdentifiers. + """ + found: list[dict[str, Any]] = [] + oem = payload.get("Oem") + if isinstance(oem, dict): + found.append(oem) + cper = payload.get("CPER") + if isinstance(cper, dict): + nested = cper.get("Oem") + if isinstance(nested, dict): + found.append(nested) + return found + + def _extract_afid(payload: dict[str, Any]) -> Optional[int]: for key in _AFID_KEYS: if key in payload and payload[key] is not None: return int(payload[key]) - oem = payload.get("Oem") - if isinstance(oem, dict): + for oem in _oem_mappings(payload): for vendor_payload in oem.values(): found = _extract_afid_from_oem_fragment(vendor_payload) if found is not None: @@ -148,6 +168,148 @@ def _origin_dict_to_unit(value: Any) -> Optional[str]: return None +def _unit_odata_id(unit: Any) -> Optional[str]: + if not isinstance(unit, dict): + return None + for key in ("@odata.id", "odata.id"): + value = unit.get(key) + if isinstance(value, str) and value.strip(): + return value + return None + + +def usable_serviceable_units(units: Any) -> list[Any]: + """Return ServiceableUnits entries that carry an @odata.id or odata.id string. + + Args: + units: ServiceableUnits value from an AFID identifier entry. + + Returns: + Unit objects that name a location. Empty when none qualify. + """ + if not isinstance(units, list): + return [] + return [unit for unit in units if _unit_odata_id(unit) is not None] + + +def promote_serviceable_unit_odata_ids(member: dict[str, Any]) -> dict[str, Any]: + """Copy odata.id onto @odata.id so the hub can read the location. + + Args: + member: Redfish log member. + + Returns: + The same member, or a copy when a ServiceableUnits entry needed @odata.id. + """ + oem = member.get("Oem") + if not isinstance(oem, dict): + return member + amd = oem.get("AMD") + if not isinstance(amd, dict): + return member + identifiers = amd.get("AMDFieldIdentifiers") + if not isinstance(identifiers, list): + return member + new_identifiers: list[Any] = [] + changed = False + for entry in identifiers: + if not isinstance(entry, dict) or not isinstance(entry.get("ServiceableUnits"), list): + new_identifiers.append(entry) + continue + new_units: list[Any] = [] + entry_changed = False + for unit in entry["ServiceableUnits"]: + if ( + isinstance(unit, dict) + and not isinstance(unit.get("@odata.id"), str) + and isinstance(unit.get("odata.id"), str) + and unit["odata.id"].strip() + ): + copied = dict(unit) + copied["@odata.id"] = unit["odata.id"] + new_units.append(copied) + entry_changed = True + else: + new_units.append(unit) + if entry_changed: + changed = True + new_identifiers.append({**entry, "ServiceableUnits": new_units}) + else: + new_identifiers.append(entry) + if not changed: + return member + return { + **member, + "Oem": {**oem, "AMD": {**amd, "AMDFieldIdentifiers": new_identifiers}}, + } + + +def amd_field_identifiers_with_units( + oem: Any, +) -> Optional[tuple[list[dict[str, Any]], list[Any]]]: + """Split Oem.AMD.AMDFieldIdentifiers by whether ServiceableUnits is usable. + + Args: + oem: Redfish Oem object from a log member. + + Returns: + (usable entries, unusable entries) when Oem.AMD.AMDFieldIdentifiers is a list. + None when that shape is absent so another source shape can be tried. + """ + if not isinstance(oem, dict): + return None + amd = oem.get("AMD") + if not isinstance(amd, dict): + return None + identifiers = amd.get("AMDFieldIdentifiers") + if not isinstance(identifiers, list): + return None + usable: list[dict[str, Any]] = [] + unusable: list[Any] = [] + for entry in identifiers: + units = entry.get("ServiceableUnits") if isinstance(entry, dict) else None + if isinstance(entry, dict) and usable_serviceable_units(units): + usable.append(entry) + else: + unusable.append(entry) + return usable, unusable + + +_LOCATION_EXTRACTORS = (amd_field_identifiers_with_units,) + + +def split_afid_location_entries( + oem: Any, +) -> Optional[tuple[list[dict[str, Any]], list[Any]]]: + """Return usable and unusable AFID entries for the first matching source shape. + + Args: + oem: Redfish Oem object from a log member. + + Returns: + (usable, unusable) for a recognized shape, or None when no shape applies. + """ + for extractor in _LOCATION_EXTRACTORS: + split = extractor(oem) + if split is not None: + return split + return None + + +def _first_unit_from_identifiers(identifiers: Any) -> Optional[str]: + if not isinstance(identifiers, list): + return None + for item in identifiers: + if not isinstance(item, dict): + continue + units = item.get("ServiceableUnits") or item.get("serviceable_units") + for unit in usable_serviceable_units(units): + name = _origin_dict_to_unit(unit) + if name: + return name + return None + + def _extract_serviceable_unit(payload: dict[str, Any]) -> Optional[str]: for key in ("serviceable_unit", "ServiceableUnit", "OriginOfCondition", "Device"): value = payload.get(key) @@ -172,45 +334,78 @@ def _extract_serviceable_unit(payload: dict[str, Any]) -> Optional[str]: if unit: return unit - oem = payload.get("Oem") - if isinstance(oem, dict): - for vendor_payload in oem.values(): + for oem in _oem_mappings(payload): + for vendor_name, vendor_payload in oem.items(): if isinstance(vendor_payload, dict): unit = vendor_payload.get("serviceable_unit") or vendor_payload.get( "ServiceableUnit" ) if unit is not None and str(unit).strip(): return str(unit).strip() - identifiers = vendor_payload.get("AMDFieldIdentifiers") - if isinstance(identifiers, list): - for item in identifiers: - if not isinstance(item, dict): - continue - su = item.get("ServiceableUnits") or item.get("serviceable_units") - if isinstance(su, list) and su: - u = _origin_dict_to_unit(su[0]) - if u: - return u - elif isinstance(vendor_payload, list): - for item in vendor_payload: - if not isinstance(item, dict): - continue - su = item.get("ServiceableUnits") or item.get("serviceable_units") - if isinstance(su, list) and su: - u = _origin_dict_to_unit(su[0]) - if u: - return u + if vendor_name != "AMD": + unit = _first_unit_from_identifiers(vendor_payload.get("AMDFieldIdentifiers")) + if unit: + return unit + else: + unit = _first_unit_from_identifiers(vendor_payload) + if unit: + return unit + split = amd_field_identifiers_with_units(oem) + if split is not None: + unit = _first_unit_from_identifiers(split[0]) + if unit: + return unit + return _switch_serviceable_unit(payload) + + +def _switch_serviceable_unit(payload: dict[str, Any]) -> Optional[str]: + """Return the component name carried by one switch event. + + Args: + payload: Switch event object. + + Returns: + object-ref name, unit id, or node name, or None when this is not a switch event. + """ + if payload.get("kind") != "Event": + return None + object_ref = payload.get("object-ref") + if not isinstance(object_ref, dict): + object_ref = payload.get("object_ref") + if isinstance(object_ref, dict): + name = object_ref.get("name") + if isinstance(name, str) and name.strip(): + return name.strip() + source = payload.get("source") + if not isinstance(source, dict): + return None + for key in ("unit-id", "node-name"): + value = source.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() return None def _extract_timestamp(payload: dict[str, Any]) -> Optional[str]: for key in _EVENT_TIMESTAMP_KEYS: - value = payload.get(key) - if value is not None and str(value).strip(): - return str(value).strip() + text = _timestamp_text(payload.get(key)) + if text: + return text + meta = payload.get("meta") + if isinstance(meta, dict): + for key in ("creation-time", "mod-time"): + text = _timestamp_text(meta.get(key)) + if text: + return text return None +def _timestamp_text(value: Any) -> str: + if value is None: + return "" + return str(value).replace("\x00", "").strip() + + def _unit_from_odata_id(odata_id: str) -> str: segment = odata_id.rstrip("/").split("/")[-1] return segment or odata_id diff --git a/nodescraper/plugins/serviceability/afid_sag_lookup.py b/nodescraper/plugins/serviceability/afid_sag_lookup.py index 52957ce5..a13bee20 100644 --- a/nodescraper/plugins/serviceability/afid_sag_lookup.py +++ b/nodescraper/plugins/serviceability/afid_sag_lookup.py @@ -48,19 +48,46 @@ def load_afid_sag_data(path: Optional[str]) -> Optional[dict[str, Any]]: def normalize_fru_name(name: str) -> str: - """Normalize FRU labels for comparison (SAG serviceable_fru vs afid.fru).""" + """Normalize a FRU or action subject for comparison. + + Args: + name: SAG fru or action_subject label. + + Returns: + Uppercase label with hyphens replaced by underscores. + """ return str(name).strip().upper().replace("-", "_") -def known_fru_names_from_sag(sag: Optional[dict[str, Any]]) -> list[str]: - """Return normalized FRU names declared under serviceable_fru in the SAG.""" +def _serviceable_catalog(sag: Optional[dict[str, Any]]) -> list[Any]: + """Return the SAG list of serviceable groups. + + Args: + sag: Loaded AFID SAG object. + + Returns: + serviceable_fru when that list is present, otherwise serviceable_action_subject. + """ if not sag: return [] - raw = sag.get("serviceable_fru") - if not isinstance(raw, list): - return [] + for key in ("serviceable_fru", "serviceable_action_subject"): + raw = sag.get(key) + if isinstance(raw, list) and raw: + return raw + return [] + + +def known_fru_names_from_sag(sag: Optional[dict[str, Any]]) -> list[str]: + """Return normalized serviceable group names from the SAG catalog. + + Args: + sag: Loaded AFID SAG object. + + Returns: + Normalized fru or action_subject names declared in the SAG catalog. + """ names: list[str] = [] - for item in raw: + for item in _serviceable_catalog(sag): if not isinstance(item, dict): continue for key in item: @@ -71,14 +98,16 @@ def known_fru_names_from_sag(sag: Optional[dict[str, Any]]) -> list[str]: def sag_fru_display_names(sag: Optional[dict[str, Any]]) -> list[str]: - """Return sorted FRU labels from SAG serviceable_fru using the original key text.""" - if not sag: - return [] - raw = sag.get("serviceable_fru") - if not isinstance(raw, list): - return [] + """Return serviceable group labels from the SAG catalog. + + Args: + sag: Loaded AFID SAG object. + + Returns: + Original fru or action_subject key text from the SAG catalog. + """ names: list[str] = [] - for item in raw: + for item in _serviceable_catalog(sag): if not isinstance(item, dict): continue for key in item: @@ -100,14 +129,23 @@ def afid_entry_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[di def afid_fru_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[str]: - """Return the FRU string for an AFID from the SAG afid table.""" + """Return the FRU or action subject for an AFID from the SAG afid table. + + Args: + afid: AMD Fault ID. + sag: Loaded AFID SAG object. + + Returns: + afid.fru when set, otherwise afid.action_subject. None when neither is set. + """ entry = afid_entry_from_sag(afid, sag) if not entry: return None - fru = entry.get("fru") - if fru is None or not str(fru).strip(): - return None - return str(fru).strip() + for key in ("fru", "action_subject"): + value = entry.get(key) + if value is not None and str(value).strip(): + return str(value).strip() + return None def afid_summary_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[str]: @@ -182,7 +220,15 @@ def group_afid_events_by_fru( events: list[AfidEvent], sag: Optional[dict[str, Any]], ) -> dict[str, list[AfidEvent]]: - """Group parsed AFID events by SAG FRU (unknown FRU bucket when lookup fails).""" + """Group parsed AFID events by SAG fru or action subject. + + Args: + events: Parsed AFID events. + sag: Loaded AFID SAG object. + + Returns: + Events keyed by normalized fru or action subject. Missing labels use UNKNOWN_FRU. + """ grouped: dict[str, list[AfidEvent]] = defaultdict(list) for event in events: fru = afid_fru_from_sag(event.afid, sag) or "UNKNOWN_FRU" @@ -215,13 +261,6 @@ def format_collected_afid_fru_summary_lines( unit_bits = ", ".join(f"{unit} x{count}" for unit, count in sorted(unit_counts.items())) lines.append(f" {fru}: AFIDs [{afid_bits}]") lines.append(f" units: {unit_bits}") - known = known_fru_names_from_sag(sag) - if known: - missing = sorted(set(known) - set(grouped.keys())) - if missing: - lines.append(f" FRUs in SAG with no collected AFID events: {', '.join(missing)}") - else: - lines.append(" All SAG serviceable FRUs have at least one collected AFID event.") return lines diff --git a/nodescraper/plugins/serviceability/analysis_window.py b/nodescraper/plugins/serviceability/analysis_window.py index 911246e1..d0721f1f 100644 --- a/nodescraper/plugins/serviceability/analysis_window.py +++ b/nodescraper/plugins/serviceability/analysis_window.py @@ -35,7 +35,12 @@ from .analyzer_args import ServiceabilityAnalyzerArgs from .cper_decode import CperDecodeError, decode_cper_raw_attachments from .se_models import AfidEvent, ServiceabilityBlock -from .se_runner import HubRunError, run_entry_point_hub, run_service_hub +from .se_runner import ( + HubRunError, + resolve_runtime_afid_sag_path, + run_entry_point_hub, + run_service_hub, +) from .serviceability_data import ServiceabilityDataModel @@ -161,7 +166,11 @@ def analyze_serviceability_window( error="empty afid_events", ) - sag_path = args.resolved_afid_sag_path() + sag_path = ( + resolve_runtime_afid_sag_path(args, log) + if args.uses_entry_point_hub() + else args.resolved_afid_sag_path() + ) data.afid_sag_path = sag_path log.info( "(%s) Using AFID_SAG file: %s", @@ -191,6 +200,7 @@ def analyze_serviceability_window( rf_event_count=len(data.rf_events), raise_on_error=args.hub_raise_on_error, prefer_rf_events=args.hub_prefer_rf_events and not bool(cper_data), + logger=log, ) except (HubRunError, ValueError) as exc: return ServiceabilityWindowResult( diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py index 6676a370..91613bcb 100644 --- a/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py @@ -25,10 +25,18 @@ ############################################################################### from __future__ import annotations +import json +from datetime import datetime, timezone + +from nodescraper.connection.inband import TextFileArtifact +from nodescraper.plugins.serviceability.se_models import ServiceabilityBlock +from nodescraper.plugins.serviceability.serviceability_api import top_action_summary from nodescraper.plugins.serviceability.serviceability_hub_analyzer import ( ServiceabilityHubAnalyzer, ) +SUMMARY_FILENAME = "summary.json" + class MI4XXAnalyzer(ServiceabilityHubAnalyzer): """Build AFID events from collected data and run the configured entry-point hub.""" @@ -38,4 +46,21 @@ class MI4XXAnalyzer(ServiceabilityHubAnalyzer): "Runs the configured entry-point service hub (hub_entry_point in analysis_args) to produce service recommendations.", "When analysis_args.skip_hub is true, only builds AFID events without running the hub.", "Supports offline analysis from a prior collection via --data with --collection False.", + f"Writes the top service action to {SUMMARY_FILENAME} in the analyzer log directory.", ) + + def _append_hub_artifacts(self, block: ServiceabilityBlock) -> None: + super()._append_hub_artifacts(block) + self.result.artifacts.append( + TextFileArtifact( + filename=SUMMARY_FILENAME, + contents=json.dumps( + top_action_summary( + block, + system_name=self.system_info.name, + analyzed_at=datetime.now(timezone.utc).isoformat(), + ), + indent=2, + ), + ) + ) diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py index a87cdf7c..2ac11a1b 100644 --- a/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py @@ -33,19 +33,19 @@ class Mi4xxServiceabilityAnalyzerArgs(ServiceabilityAnalyzerArgs): - """Analysis args for Mi4xxServiceabilityPlugin (AFSE entry point).""" + """Analysis args for Mi4xxServiceabilityPlugin (Hub entry point afse).""" hub_entry_point: str = Field( default="afse", - description="Registered AFSE entry point name (MI4XX service hub).", + description="Registered Hub entry point name (default afse).", ) hub_display_name: Optional[str] = Field( - default="AFSE", + default="Hub", description="Label for analyzer status messages.", ) hub_python_module: Optional[str] = Field( default=None, - description="Not used for MI4XX; AFSE is selected via hub_entry_point afse.", + description="Not used for MI4XX; Hub is selected via hub_entry_point afse.", ) rf_event_log_uri: str = Field( default="/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries", @@ -65,10 +65,10 @@ def resolved_rf_event_log_uri(self) -> str: return str(self.rf_event_log_uri).strip() @model_validator(mode="after") - def _mi4xx_uses_afse(self) -> "Mi4xxServiceabilityAnalyzerArgs": + def _mi4xx_uses_afse_entry_point(self) -> "Mi4xxServiceabilityAnalyzerArgs": if self.hub_python_module: raise ValueError( - "Mi4xxServiceabilityPlugin uses AFSE via hub_entry_point; " + "Mi4xxServiceabilityPlugin uses Hub via hub_entry_point afse; " "hub_python_module is not supported" ) if str(self.hub_entry_point).strip().lower() != "afse": diff --git a/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py b/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py index 4341f5d4..ab542dd3 100644 --- a/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py +++ b/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py @@ -25,12 +25,12 @@ ############################################################################### from __future__ import annotations -from typing import Annotated, Any, Optional, Union +from typing import Annotated, Any, Optional, Union, cast from pydantic import Field -from nodescraper.enums import EventPriority, ExecutionStatus, SystemInteractionLevel -from nodescraper.models import DataPluginResult, PluginResult, TaskResult +from nodescraper.enums import EventPriority, SystemInteractionLevel +from nodescraper.models import PluginResult from nodescraper.plugins.serviceability.afid_sag_lookup import log_afid_fru_summary from nodescraper.plugins.serviceability.serviceability_data import ( ServiceabilityDataLoadError, @@ -140,25 +140,22 @@ def _merge_mi4xx_collection_args( return collection_args.model_copy(update={"rf_event_log_uri": uri_text}) return collection_args - def _plugin_error_result(self, message: str) -> PluginResult: - self.logger.error("(%s) %s", self.__class__.__name__, message) - return PluginResult( - status=ExecutionStatus.ERROR, - source=self.__class__.__name__, - message=message, - result_data=DataPluginResult( - collection_result=TaskResult( - status=ExecutionStatus.NOT_RAN, - parent=self.__class__.__name__, - message="Data collection skipped", - ), - analysis_result=TaskResult( - status=ExecutionStatus.ERROR, - parent=self.__class__.__name__, - message=message, - ), - ), - ) + def _apply_afid_sag_path_to_collection_args( + self, + afid_sag_path: str, + collection_args: Any, + ) -> Any: + """Copy a resolved AFID_SAG path onto MI4xx collector args. + + Args: + afid_sag_path: Absolute path returned by preflight. + collection_args: MI4xx collector args for this run, or None. + + Returns: + Collector args with afid_sag_path set. + """ + merged_collection, _ = self._merge_afid_sag_path(afid_sag_path, collection_args, None) + return merged_collection def run( # type: ignore[override] self, @@ -175,7 +172,7 @@ def run( # type: ignore[override] Union[SystemInteractionLevel, str], "System interaction level (e.g. PASSIVE, INTERACTIVE, DISRUPTIVE).", ] = SystemInteractionLevel.INTERACTIVE, - preserve_connection: bool = False, + preserve_connection: bool = True, data: Annotated[ Optional[Union[str, dict, ServiceabilityDataModel]], Field( @@ -198,6 +195,16 @@ def run( # type: ignore[override] analysis_args, ) + if analysis: + preflight = self._preflight_hub_sag(collection_args, analysis_args) + if isinstance(preflight, PluginResult): + return preflight + collection_args, updated_analysis = preflight + analysis_args = cast( + Optional[Union[Mi4xxServiceabilityAnalyzerArgs, dict[str, Any]]], + updated_analysis, + ) + if analysis and not collection and data is not None: try: loaded = ( diff --git a/nodescraper/plugins/serviceability/se_adapter.py b/nodescraper/plugins/serviceability/se_adapter.py index 56b0f3b1..a4c8e170 100644 --- a/nodescraper/plugins/serviceability/se_adapter.py +++ b/nodescraper/plugins/serviceability/se_adapter.py @@ -279,11 +279,53 @@ def _optional_int(value: Any) -> Optional[int]: def _entry_point_result_rows(hub_result: dict[str, Any]) -> list[Any]: - """Extract triage rows from the entry-point hub analyze response.""" + """Extract triage.results rows from the entry-point hub analyze response.""" + triage = hub_result.get("triage") + if isinstance(triage, dict): + nested = triage.get("results") + if isinstance(nested, list): + return nested results = hub_result.get("results") return results if isinstance(results, list) else [] +def _entry_point_top_rows(hub_result: dict[str, Any]) -> list[Any]: + """Extract triage.top rows from the entry-point hub analyze response.""" + top = hub_result.get("top") + if isinstance(top, list): + return top + triage = hub_result.get("triage") + if isinstance(triage, dict): + nested = triage.get("top") + if isinstance(nested, list): + return nested + return [] + + +def _service_action_steps_from_row( + row: dict[str, Any], + san: int, + sag: Optional[dict[str, Any]], +) -> list[str]: + sa = row.get("service_action") + if isinstance(sa, dict): + steps = sa.get("steps") + if isinstance(steps, list): + descriptions: list[str] = [] + for step in steps: + if not isinstance(step, dict): + continue + description = step.get("description") + if description is None: + continue + text = str(description).strip() + if text: + descriptions.append(text) + if descriptions: + return descriptions + return service_action_step_descriptions_from_sag(san, sag) + + def _service_action_title_from_row(row: dict[str, Any]) -> Optional[str]: title = row.get("service_action_title") if title is not None: @@ -358,7 +400,7 @@ def _hub_triage_result_from_row( service_action_title=title, service_action_category=category, service_action_severity=sa_severity, - service_action_steps=service_action_step_descriptions_from_sag(san, sag), + service_action_steps=_service_action_steps_from_row(row, san, sag), afid_summary=_afid_summary_from_sag(afid, sag), ) @@ -424,9 +466,10 @@ def format_serviceability_solution_lines(block: ServiceabilityBlock) -> list[str if block.afid_sag_file_version: lines.append(f"AFID_SAG file: {block.afid_sag_file_version}") if block.hub_triage_results: - lines.append("Hub triage results:") - for index, row in enumerate(block.hub_triage_results, start=1): - lines.extend(_format_hub_triage_result_lines(index, row)) + lines.append( + f"{len(block.hub_triage_results)} prioritized recommendation(s); " + "see recommendation tables below" + ) return lines if block.short_service_info: lines.append("short_service_info:") @@ -530,6 +573,7 @@ def serviceability_block_from_entry_point_hub( hub_label: str = "Service hub", rf_event_count: int = 0, afid_sag_path: Optional[str] = None, + hub_analyze_response: Optional[dict[str, Any]] = None, ) -> ServiceabilityBlock: """Build a ServiceabilityBlock from a registered entry-point hub analyze() response.""" hub_name = str(hub_result.get("engine") or hub_label) @@ -606,6 +650,14 @@ def serviceability_block_from_entry_point_hub( parsed = _hub_triage_result_from_row(row, sag) if parsed is not None: triage_results.append(parsed) + + top_results: list[HubTriageResult] = [] + for row in _entry_point_top_rows(hub_result): + if not isinstance(row, dict): + continue + parsed = _hub_triage_result_from_row(row, sag) + if parsed is not None: + top_results.append(parsed) sag_metadata = None afid_sag_file_version = None if sag: @@ -629,5 +681,7 @@ def serviceability_block_from_entry_point_hub( hub_version=hub_version, afid_sag_file_version=afid_sag_file_version, afid_sag_metadata=sag_metadata, + hub_analyze_response=hub_analyze_response, + hub_top_results=top_results, hub_triage_results=triage_results, ) diff --git a/nodescraper/plugins/serviceability/se_models.py b/nodescraper/plugins/serviceability/se_models.py index 94baa835..8969ff66 100644 --- a/nodescraper/plugins/serviceability/se_models.py +++ b/nodescraper/plugins/serviceability/se_models.py @@ -74,6 +74,29 @@ class ServiceabilitySolution(BaseModel): ) +class PrioritizedServiceAction(BaseModel): + """One hub-ranked service action row for tables and API-style consumers.""" + + rank: int + afid: int + location: str + count: int = 1 + service_action_num: int + service_action_title: Optional[str] = None + service_action_category: Optional[str] = None + priority: Optional[int] = None + sa_severity: Optional[int] = None + tier: Optional[int] = None + tier_label: Optional[str] = None + fru: Optional[str] = None + fru_rank: Optional[int] = None + hub_sort_priority: Optional[int] = None + multi_mask: Optional[int] = None + afid_summary: Optional[str] = None + service_action_steps: List[str] = Field(default_factory=list) + serviceable_units: Optional[List[str]] = None + + class HubTriageResult(BaseModel): """One service hub triage row with SAG-enriched action details.""" @@ -130,6 +153,14 @@ class ServiceabilityBlock(BaseModel): "per-unit dict payloads are collapsed, identical messages merged with unit lists)." ), ) + hub_analyze_response: Optional[dict[str, Any]] = Field( + default=None, + description="Unmodified Hub analyze() JSON (triage.top, triage.results, status, pid, revision).", + ) + hub_top_results: List[HubTriageResult] = Field( + default_factory=list, + description="Hub triage.top rows (highest se_sort_priority).", + ) hub_triage_results: List[HubTriageResult] = Field( default_factory=list, description="Full service hub triage rows with SAG-enriched action details.", diff --git a/nodescraper/plugins/serviceability/se_runner.py b/nodescraper/plugins/serviceability/se_runner.py index c30858b5..1d1682cb 100644 --- a/nodescraper/plugins/serviceability/se_runner.py +++ b/nodescraper/plugins/serviceability/se_runner.py @@ -31,17 +31,30 @@ import importlib import importlib.metadata import inspect +import logging +import re from pathlib import Path -from typing import Any, Callable, Optional, Protocol, Type, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, Type, Union, cast -from .afid_sag_paths import validate_afid_sag_path +from nodescraper.constants import DEFAULT_LOGGER + +from .afid_events import promote_serviceable_unit_odata_ids, split_afid_location_entries +from .afid_sag_paths import default_afid_sag_path, validate_afid_sag_path from .se_adapter import ( serviceability_block_from_entry_point_hub, serviceability_block_from_service_result, ) from .se_models import AfidEvent, ServiceabilityBlock +if TYPE_CHECKING: + from .analyzer_args import ServiceabilityAnalyzerArgs + HUB_ENTRY_POINT_GROUP = "amd.serviceability_engines" +_MIN_AFSE_VERSION = (0, 2) +# Leading major.minor. A patch, pre-release, or local .dev suffix is ignored. +_AFSE_VERSION_RE = re.compile(r"^v?(?P\d+)\.(?P\d+)") +# setuptools_scm no-guess-dev with no ancestor tag: 0.0.post1.devN+gHASH +_UNTAGGED_AFSE_RE = re.compile(r"^v?0\.0(?:\.post\d+)?\.dev\d+(?:\+[0-9A-Za-z.]+)?$") def normalize_hub_entry_point(hub_name: str) -> str: @@ -292,15 +305,78 @@ def events_payload_for_hub_analyze( raise HubRunError("rf_events or afid_events is required") +def _redfish_severity_is_ok(member: dict[str, Any]) -> bool: + """Return whether a Redfish log member is an OK event. + + Args: + member: Redfish log member. + + Returns: + True when Severity is OK. + """ + severity = member.get("Severity") + return isinstance(severity, str) and severity.strip().upper() == "OK" + + +def drop_afids_without_serviceable_units( + rf_events: list[Any], + logger: Optional[logging.Logger] = None, +) -> list[Any]: + """Waive AMDFieldIdentifiers entries the hub would reject for lacking ServiceableUnits. + + The entry-point hub rejects the whole event list when any Oem.AMD.AMDFieldIdentifiers + entry has no ServiceableUnits, so drop those entries before analysis. OK events are + dropped quietly. Any other severity is logged. + + Args: + rf_events: Raw Redfish log members. + logger: Logger for the per-entry warning. Defaults to the nodescraper logger. + + Returns: + Members with unusable AFID entries removed. Members left with no AFID entries are dropped. + """ + log = logger or logging.getLogger(DEFAULT_LOGGER) + kept: list[Any] = [] + for member in rf_events: + oem = member.get("Oem") if isinstance(member, dict) else None + split = split_afid_location_entries(oem) + if split is None: + kept.append(member) + continue + usable, unusable = split + if not _redfish_severity_is_ok(member): + for entry in unusable: + afid = entry.get("AFID") if isinstance(entry, dict) else None + log.warning( + "Waiving AFID %s from %s: AMDFieldIdentifiers entry has no ServiceableUnits", + afid, + member.get("@odata.id") or member.get("Id") or "", + ) + if not unusable: + kept.append(promote_serviceable_unit_odata_ids(member)) + elif usable: + amd = cast(dict, oem).get("AMD") + amd_copy = {**cast(dict, amd), "AMDFieldIdentifiers": usable} + kept.append( + promote_serviceable_unit_odata_ids( + {**member, "Oem": {**cast(dict, oem), "AMD": amd_copy}} + ) + ) + return kept + + def resolve_hub_events_payload( *, afid_events: list[AfidEvent], rf_events: Optional[list[Any]] = None, prefer_rf_events: bool = True, + logger: Optional[logging.Logger] = None, ) -> list[Any]: """Choose rf_events or aggregated afid_events for hub analyze().""" if prefer_rf_events and rf_events: - return events_payload_for_hub_analyze(rf_events=rf_events) + usable_rf_events = drop_afids_without_serviceable_units(rf_events, logger) + if usable_rf_events: + return events_payload_for_hub_analyze(rf_events=usable_rf_events) return events_payload_for_hub_analyze( afid_events_payload=afid_events_to_entry_point_payload(afid_events) ) @@ -371,6 +447,7 @@ def entry_point_hub_result_from_triage( "status": "error", "error": {"message": error_message or "Service hub analyze failed"}, "results": [], + "top": [], "tier_grouped": {}, } @@ -378,16 +455,23 @@ def entry_point_hub_result_from_triage( result_entries = ( getattr(triage_section, "results", None) if triage_section is not None else None ) + top_entries = getattr(triage_section, "top", None) if triage_section is not None else None results = ( [_resolved_entry_to_hub_row(entry) for entry in result_entries] if isinstance(result_entries, list) else [] ) + top = ( + [_resolved_entry_to_hub_row(entry) for entry in top_entries] + if isinstance(top_entries, list) + else [] + ) return { **base, "status": "ok", "error": None, "results": results, + "top": top, "tier_grouped": _tier_grouped_from_rows(results), } @@ -405,6 +489,7 @@ def _synthetic_error_hub_result( "engine": engine_name, "engine_version": engine_version, "results": [], + "top": [], "tier_grouped": {}, } @@ -428,6 +513,259 @@ def _entry_point_analyze_error(hub_result: dict[str, Any]) -> Optional[str]: return "Service hub analyze failed" +def _hub_analyze_response_from_raw(raw_result: Any) -> dict[str, Any]: + """Return the unmodified Hub analyze JSON payload.""" + if isinstance(raw_result, dict): + return dict(raw_result) + return dataclasses.asdict(raw_result) + + +def _response_field(result: Any, name: str) -> Any: + """Read a field from an AFSE TriageResponse dataclass or a dict payload.""" + if isinstance(result, dict): + return result.get(name) + return getattr(result, name, None) + + +def validate_afid_sag_with_hub(hub: Any, sag_path: str) -> bool: + """Ask the loaded hub to validate the AFID SAG file via its validate() API. + + Args: + hub: Loaded entry-point hub (AFSE). + sag_path: AFID SAG file to validate. + + Returns: + True when the hub validated the file, False when the hub has no validate(). + + Raises: + HubRunError: When validate() raises or reports a status other than ok. + """ + validate = getattr(hub, "validate", None) + if not callable(validate): + return False + try: + result = validate(sag_path) + except Exception as exc: # noqa: BLE001 + raise HubRunError(f"AFSE SAG validation failed: {exc}") from exc + status = str(_response_field(result, "status") or "").strip().lower() + if status == "ok": + return True + error = _response_field(result, "error") + code = _response_field(error, "code") if error is not None else None + message = _response_field(error, "message") if error is not None else None + detail = str(message or code or status or "unknown error") + raise HubRunError(f"AFSE SAG validation failed: {detail}") + + +def resolve_runtime_afid_sag_path( + args: ServiceabilityAnalyzerArgs, + logger: Optional[logging.Logger] = None, +) -> str: + """Return the AFID SAG file analysis should open. + + A configured afid_sag_path wins. Otherwise the loaded entry-point hub is asked for its + default_sag_path(), falling back to the built-in /opt/amd/afid/AFID_SAG.json. + + Args: + args: Analyzer args for this run. + logger: Logger for the "path omitted" line. Defaults to the nodescraper logger. + + Returns: + SAG path to use for analysis. + """ + if args.afid_sag_path and str(args.afid_sag_path).strip(): + return str(args.afid_sag_path).strip() + hub: Any = None + if args.uses_entry_point_hub(): + try: + hub = load_hub_from_entry_point(args.resolved_hub_entry_point()) + except (HubRunError, ValueError): + hub = None + path = default_afid_sag_path() + lookup = getattr(hub, "default_sag_path", None) + if callable(lookup): + found = lookup() + if found is not None and str(found).strip(): + path = str(found).strip() + (logger or logging.getLogger(DEFAULT_LOGGER)).info( + "AFID_SAG path omitted; using default: %s", path + ) + return path + + +def _untagged_afse_has_redfish_and_switch(version: str) -> bool: + """Return whether an untagged AFSE build can parse Redfish and switch events. + + Args: + version: Version reported by the AFSE package or module hub. + + Returns: + True when version has no release tag and both parsers import. + """ + if _UNTAGGED_AFSE_RE.match(version.strip()) is None: + return False + try: + redfish = importlib.import_module("serviceability_engine.transformers.redfish") + switch = importlib.import_module("serviceability_engine.transformers.switch") + except ImportError: + return False + return callable(getattr(redfish, "parse", None)) and callable(getattr(switch, "parse", None)) + + +def _afse_version_error( + version: str, + logger: Optional[logging.Logger] = None, +) -> Optional[str]: + """Return an error when AFSE is older than 0.2. + + Args: + version: Version reported by the AFSE package or module hub. + logger: Logger used when an untagged build is accepted. + + Returns: + Error text when the major.minor is below 0.2 or missing, otherwise None. + """ + if _untagged_afse_has_redfish_and_switch(version): + (logger or logging.getLogger(DEFAULT_LOGGER)).info( + "AFSE %s is an untagged build with Redfish and switch event support", + version, + ) + return None + match = _AFSE_VERSION_RE.match(version.strip()) + found = (int(match.group("major")), int(match.group("minor"))) if match else None + if found is None or found < _MIN_AFSE_VERSION: + return ( + f"AFSE {_MIN_AFSE_VERSION[0]}.{_MIN_AFSE_VERSION[1]} or newer is required " + f"(Redfish log entries and switch events); found {version}" + ) + return None + + +def _entry_point_distribution_version(hub_name: str) -> Optional[str]: + """Return the installed package version for a registered AFSE entry point. + + Args: + hub_name: Entry point name such as afse. + + Returns: + Distribution version string, or None when it cannot be resolved. + """ + wanted = normalize_hub_entry_point(hub_name) + for ep in _entry_points_for_group(HUB_ENTRY_POINT_GROUP): + if getattr(ep, "name", None) != wanted: + continue + dist = getattr(ep, "dist", None) + version = getattr(dist, "version", None) if dist is not None else None + if version: + return str(version) + value = str(getattr(ep, "value", "") or getattr(ep, "module", "") or "") + top = value.split(":")[0].split(".")[0].strip() + if not top: + continue + try: + return importlib.metadata.version(top) + except importlib.metadata.PackageNotFoundError: + continue + return None + + +def _probe_entry_point_hub( + args: ServiceabilityAnalyzerArgs, +) -> tuple[Any, str, str]: + """Load the configured AFSE entry point and return the hub, label, and version. + + Args: + args: Analyzer args for this run. + + Returns: + Loaded hub instance, display label, and version string. + """ + name = args.resolved_hub_entry_point() + hub = load_hub_from_entry_point(name) + label = args.hub_display_name or getattr(hub, "name", None) or name + version = _entry_point_distribution_version(name) or _hub_engine_version(hub) + return hub, str(label), version + + +def preflight_module_hub( + args: ServiceabilityAnalyzerArgs, + logger: Optional[logging.Logger] = None, +) -> None: + """Check that a Python-module hub is installed and new enough. + + Args: + args: Analyzer args that select hub_python_module. + logger: Logger for the install line. Defaults to the nodescraper logger. + + Returns: + None. + + Raises: + HubRunError: When the module cannot be imported or AFSE is older than 0.2. + """ + log = logger or logging.getLogger(DEFAULT_LOGGER) + module_name = str(args.hub_python_module) + try: + mod = importlib.import_module(module_name) + except ImportError as exc: + raise HubRunError(f"AFSE hub module {module_name!r} is not installed: {exc}") from exc + hub_cls = _resolve_hub_class(mod, args.hub_analyze_method) + label = args.hub_display_name or getattr(hub_cls, "name", None) or module_name + version = getattr(mod, "__version__", None) or _hub_engine_version(hub_cls) + if callable(version): + version = version() + version_text = version.strip() if isinstance(version, str) and version.strip() else "unknown" + log.info("AFSE install found: %s version %s", label, version_text) + version_error = _afse_version_error(version_text, log) + if version_error: + raise HubRunError(version_error) + + +def preflight_afid_sag( + args: ServiceabilityAnalyzerArgs, + logger: Optional[logging.Logger] = None, +) -> str: + """Resolve the AFID SAG file and check it with AFSE before any collection runs. + + Args: + args: Analyzer args for this run (entry-point hub). + logger: Logger for the OK lines. Defaults to the nodescraper logger. + + Returns: + Resolved absolute SAG path. + + Raises: + HubRunError: When the file is missing, the hub cannot load, AFSE is too old, or AFSE rejects the file. + """ + log = logger or logging.getLogger(DEFAULT_LOGGER) + errors: list[str] = [] + hub: Any = None + hub_label = "" + try: + hub, hub_label, hub_version = _probe_entry_point_hub(args) + log.info("AFSE install found: %s version %s", hub_label, hub_version) + version_error = _afse_version_error(hub_version, log) + if version_error: + errors.append(version_error) + except (HubRunError, ValueError) as exc: + errors.append(str(exc)) + + resolved = "" + try: + sag_path = resolve_runtime_afid_sag_path(args, log) + validate_afid_sag_path(sag_path) + resolved = str(Path(sag_path).expanduser().resolve()) + log.info("AFID_SAG file OK: %s", resolved) + if hub is not None and validate_afid_sag_with_hub(hub, resolved): + log.info("AFID_SAG validated by %s: %s", getattr(hub, "name", hub_label), resolved) + except HubRunError as exc: + errors.append(str(exc)) + + if errors: + raise HubRunError("; ".join(errors)) + return resolved + + def run_entry_point_hub( *, hub_entry_point: str, @@ -438,20 +776,27 @@ def run_entry_point_hub( rf_event_count: int = 0, raise_on_error: bool = False, prefer_rf_events: bool = True, + logger: Optional[logging.Logger] = None, ) -> ServiceabilityBlock: """Run a registered entry-point service hub and return a :class:`ServiceabilityBlock`.""" if not afid_events and not (prefer_rf_events and rf_events): raise HubRunError("No AFID events to analyze") + log = logger or logging.getLogger(DEFAULT_LOGGER) validate_afid_sag_path(afid_sag_path) + resolved_sag = Path(afid_sag_path).expanduser().resolve() + log.debug("AFID_SAG file OK: %s", resolved_sag) label = hub_display_name or normalize_hub_entry_point(hub_entry_point) hub = load_hub_from_entry_point(hub_entry_point) + if validate_afid_sag_with_hub(hub, str(resolved_sag)): + log.debug("AFID_SAG validated by %s: %s", getattr(hub, "name", label), resolved_sag) hub_label = getattr(hub, "name", label) engine_version = _hub_engine_version(hub) events = resolve_hub_events_payload( afid_events=afid_events, rf_events=rf_events, prefer_rf_events=prefer_rf_events, + logger=logger, ) try: raw_result = hub.analyze(events, afid_sag_path) @@ -466,7 +811,9 @@ def run_entry_point_hub( if isinstance(raw_result, dict): hub_result = raw_result + hub_analyze_response = _hub_analyze_response_from_raw(raw_result) else: + hub_analyze_response = _hub_analyze_response_from_raw(raw_result) hub_result = entry_point_hub_result_from_triage( raw_result, engine_name=hub_label, @@ -484,6 +831,7 @@ def run_entry_point_hub( hub_label=label, rf_event_count=event_count, afid_sag_path=afid_sag_path, + hub_analyze_response=hub_analyze_response, ) diff --git a/nodescraper/plugins/serviceability/serviceability_api.py b/nodescraper/plugins/serviceability/serviceability_api.py new file mode 100644 index 00000000..5cfc8ea1 --- /dev/null +++ b/nodescraper/plugins/serviceability/serviceability_api.py @@ -0,0 +1,396 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Any, Optional + +from .se_models import ( + HubTriageResult, + PrioritizedServiceAction, + ServiceabilityBlock, + ServiceabilitySolution, +) + + +def service_action_identity(action: PrioritizedServiceAction) -> tuple[int, str, int]: + """Return the Hub row identity used to match top vs results entries.""" + return (action.afid, action.location, action.service_action_num) + + +def _hub_triage_identity(row: HubTriageResult) -> tuple[int, str, int]: + return (row.afid, row.location, row.service_action_num) + + +def _triage_row_to_action(rank: int, row: HubTriageResult) -> PrioritizedServiceAction: + return PrioritizedServiceAction( + rank=rank, + afid=row.afid, + location=row.location, + count=row.count, + service_action_num=row.service_action_num, + service_action_title=row.service_action_title, + service_action_category=row.service_action_category, + priority=row.priority, + sa_severity=row.sa_severity, + tier=row.tier, + tier_label=row.tier_label, + fru=row.fru, + fru_rank=row.fru_rank, + hub_sort_priority=row.hub_sort_priority, + multi_mask=row.multi_mask, + afid_summary=row.afid_summary, + service_action_steps=list(row.service_action_steps), + ) + + +def _solution_to_action(rank: int, solution: ServiceabilitySolution) -> PrioritizedServiceAction: + return PrioritizedServiceAction( + rank=rank, + afid=solution.afid, + location=solution.serviceable_unit[0] if solution.serviceable_unit else "", + count=1, + service_action_num=solution.service_action_num, + service_action_title=solution.service_action_title, + service_action_category=None, + priority=None, + sa_severity=None, + tier=None, + tier_label=solution.service_action_tier, + fru=None, + fru_rank=None, + hub_sort_priority=None, + multi_mask=None, + afid_summary=solution.afid_summary, + service_action_steps=[], + serviceable_units=list(solution.serviceable_unit), + ) + + +def _optional_int(value: Any) -> Optional[int]: + """Return value as an int when it is numeric. + + Args: + value: Field from an AFSE triage row. + + Returns: + The integer value, or None when value is missing or not numeric. + """ + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _afse_service_action(row: dict[str, Any]) -> dict[str, Any]: + """Return the service_action object on an AFSE triage row. + + Args: + row: One AFSE triage.top or triage.results entry. + + Returns: + The service_action mapping, or an empty mapping when it is absent. + """ + service_action = row.get("service_action") + if isinstance(service_action, dict): + return service_action + return {} + + +def _afse_action_title(row: dict[str, Any]) -> Optional[str]: + """Return the service action title from an AFSE triage row. + + Args: + row: One AFSE triage.top or triage.results entry. + + Returns: + The title text, or None when the row has no title. + """ + service_action = _afse_service_action(row) + for candidate in (service_action.get("title"), row.get("service_action_title")): + if candidate is None: + continue + text = str(candidate).strip() + if text: + return text + return None + + +def _afse_action_steps(row: dict[str, Any]) -> list[str]: + """Return service action step descriptions from an AFSE triage row. + + Args: + row: One AFSE triage.top or triage.results entry. + + Returns: + Step description strings in AFSE order. + """ + steps = _afse_service_action(row).get("steps") + if not isinstance(steps, list): + return [] + descriptions: list[str] = [] + for step in steps: + if not isinstance(step, dict): + continue + description = step.get("description") + if description is None: + continue + text = str(description).strip() + if text: + descriptions.append(text) + return descriptions + + +def _action_from_afse_row(rank: int, row: dict[str, Any]) -> Optional[PrioritizedServiceAction]: + """Map one AFSE triage row to a table action. + + Args: + rank: 1-based position of this row inside triage.top. + row: One AFSE triage.top entry. + + Returns: + The table action, or None when the row has no AFID, action number, or location. + """ + afid_raw = row.get("afid", row.get("afid_num")) + san_raw = row.get("service_action_num") + location = row.get("location") or row.get("serviceable_unit") + if afid_raw is None or san_raw is None or location is None: + return None + try: + afid = int(afid_raw) + service_action_num = int(san_raw) + except (TypeError, ValueError): + return None + location_text = str(location).strip() + if not location_text: + return None + service_action = _afse_service_action(row) + tier_label = row.get("tier_label") + if tier_label is not None: + tier_label = str(tier_label).strip() or None + category = service_action.get("category") + category_text = ( + str(category).strip() if category is not None and str(category).strip() else None + ) + return PrioritizedServiceAction( + rank=rank, + afid=afid, + location=location_text, + count=max(1, _optional_int(row.get("count")) or 1), + service_action_num=service_action_num, + service_action_title=_afse_action_title(row), + service_action_category=category_text, + priority=_optional_int(row.get("priority")), + sa_severity=_optional_int(row.get("sa_severity")) + or _optional_int(service_action.get("severity")), + tier=_optional_int(row.get("tier")), + tier_label=tier_label, + hub_sort_priority=_optional_int( + row.get("se_sort_priority") or row.get("hub_sort_priority") + ), + multi_mask=_optional_int(row.get("multi_mask")), + service_action_steps=_afse_action_steps(row), + ) + + +def actions_from_afse_triage_top(block: ServiceabilityBlock) -> list[PrioritizedServiceAction]: + """Return table rows for AFSE triage.top. + + Args: + block: Serviceability block that may hold the raw hub analyze response. + + Returns: + One action per AFSE triage.top entry. Empty when that list is absent. + """ + response = block.hub_analyze_response + if not isinstance(response, dict): + return [] + triage = response.get("triage") + if not isinstance(triage, dict): + return [] + top = triage.get("top") + if not isinstance(top, list): + return [] + actions: list[PrioritizedServiceAction] = [] + for index, row in enumerate(top, start=1): + if not isinstance(row, dict): + continue + action = _action_from_afse_row(index, row) + if action is not None: + actions.append(action) + return actions + + +def _afse_triage_section(block: ServiceabilityBlock) -> Optional[dict[str, Any]]: + """Return the AFSE triage object when the hub response includes one. + + Args: + block: Serviceability block populated by hub analysis. + + Returns: + The triage mapping, or None when the response has no triage section. + """ + response = block.hub_analyze_response + if not isinstance(response, dict): + return None + triage = response.get("triage") + if not isinstance(triage, dict): + return None + return triage + + +def afse_top_action_missing(block: ServiceabilityBlock) -> bool: + """Return whether an AFSE triage section has no top action. + + Args: + block: Serviceability block populated by hub analysis. + + Returns: + True when triage is present and triage.top has no usable action. + """ + if _afse_triage_section(block) is None: + return False + return not actions_from_afse_triage_top(block) + + +def build_top_service_actions(block: ServiceabilityBlock) -> list[PrioritizedServiceAction]: + """Return the AFSE triage.top service actions for display. + + Args: + block: Serviceability block populated by hub analysis. + + Returns: + Actions from AFSE triage.top when that list has entries. Empty when triage + is present but triage.top is empty. Parsed hub rows are used when the + response has no triage section. + """ + afse_top = actions_from_afse_triage_top(block) + if afse_top: + return afse_top + if _afse_triage_section(block) is not None: + return [] + if block.hub_top_results: + rank_by_identity = { + _hub_triage_identity(row): rank + for rank, row in enumerate(block.hub_triage_results, start=1) + } + top_actions: list[PrioritizedServiceAction] = [] + for index, row in enumerate(block.hub_top_results, start=1): + rank = rank_by_identity.get(_hub_triage_identity(row), index) + top_actions.append(_triage_row_to_action(rank, row)) + return top_actions + if block.hub_triage_results: + return [_triage_row_to_action(1, block.hub_triage_results[0])] + if block.solution: + return [_solution_to_action(1, block.solution[0])] + return [] + + +def build_prioritized_service_actions( + block: ServiceabilityBlock, +) -> list[PrioritizedServiceAction]: + """Return hub-ranked service actions from triage.results order.""" + if block.hub_triage_results: + return [ + _triage_row_to_action(rank, row) + for rank, row in enumerate(block.hub_triage_results, start=1) + ] + return [ + _solution_to_action(rank, solution) for rank, solution in enumerate(block.solution, start=1) + ] + + +def split_recommendation_actions( + block: ServiceabilityBlock, +) -> tuple[list[PrioritizedServiceAction], list[PrioritizedServiceAction]]: + """Split Hub triage.top service actions from lower-priority triage.results rows.""" + prioritized = build_prioritized_service_actions(block) + top = build_top_service_actions(block) + if not prioritized: + return top, [] + if not top: + if _afse_triage_section(block) is not None: + return [], prioritized + return [prioritized[0]], prioritized[1:] + + top_sort_priorities = { + action.hub_sort_priority for action in top if action.hub_sort_priority is not None + } + if top_sort_priorities: + additional = [ + action for action in prioritized if action.hub_sort_priority not in top_sort_priorities + ] + return top, additional + + top_identities = {service_action_identity(action) for action in top} + additional = [ + action for action in prioritized if service_action_identity(action) not in top_identities + ] + return top, additional + + +def top_action_summary( + block: ServiceabilityBlock, + system_name: str, + analyzed_at: str, +) -> dict[str, Any]: + """Build a top-service-action summary payload from a hub result. + + Args: + block: Serviceability block returned by the hub. + system_name: Name of the analyzed system. + analyzed_at: ISO timestamp for when analysis ran. + + Returns: + System name, analysis time, hub and SAG versions, AFID event count, and the + top service action(s). Ties for the top rank are all included. + """ + top, _ = split_recommendation_actions(block) + return { + "system_name": system_name, + "analyzed_at": analyzed_at, + "hub_version": block.hub_version, + "afid_sag_file_version": block.afid_sag_file_version, + "afid_event_count": len(block.afid_events), + "top_service_actions": [ + action.model_dump(mode="json", exclude_none=True) for action in top + ], + } + + +def export_serviceability_json(block: ServiceabilityBlock) -> Optional[dict[str, Any]]: + """Return the AFSE analyze payload written to serviceability.json. + + Args: + block: Serviceability block that may hold the raw hub analyze response. + + Returns: + The hub analyze response, or None when AFSE did not return one. + """ + if block.hub_analyze_response is None: + return None + return dict(block.hub_analyze_response) diff --git a/nodescraper/plugins/serviceability/serviceability_data.py b/nodescraper/plugins/serviceability/serviceability_data.py index 69f81fab..eeb2f6b2 100644 --- a/nodescraper/plugins/serviceability/serviceability_data.py +++ b/nodescraper/plugins/serviceability/serviceability_data.py @@ -37,6 +37,7 @@ from .event_log_utils import rf_events_from_json_payload from .se_models import AfidEvent, ServiceabilityBlock +from .serviceability_api import export_serviceability_json class ServiceabilityDataLoadError(ValueError): @@ -151,13 +152,11 @@ def log_model(self, log_path: str) -> None: with open(cper_path, "w", encoding="utf-8") as f: json.dump(self.cper_data, f, indent=2) if self.serviceability is not None: - serviceability_path = os.path.join(log_path, "serviceability.json") - with open(serviceability_path, "w", encoding="utf-8") as f: - json.dump( - self.serviceability.model_dump(mode="json"), - f, - indent=2, - ) + payload = export_serviceability_json(self.serviceability) + if payload is not None: + serviceability_path = os.path.join(log_path, "serviceability.json") + with open(serviceability_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) from .afid_fru_csv import write_afid_fru_summary_csv write_afid_fru_summary_csv( diff --git a/nodescraper/plugins/serviceability/serviceability_plugin_base.py b/nodescraper/plugins/serviceability/serviceability_plugin_base.py index 67ff45ca..edb76ac8 100644 --- a/nodescraper/plugins/serviceability/serviceability_plugin_base.py +++ b/nodescraper/plugins/serviceability/serviceability_plugin_base.py @@ -23,10 +23,14 @@ # SOFTWARE. # ############################################################################### +from typing import Any, Optional, Union + from nodescraper.base import OOBandDataPlugin -from nodescraper.models import CollectorArgs +from nodescraper.enums import ExecutionStatus +from nodescraper.models import CollectorArgs, DataPluginResult, PluginResult, TaskResult from .analyzer_args import ServiceabilityAnalyzerArgs +from .se_runner import HubRunError, preflight_afid_sag, preflight_module_hub from .serviceability_collector import ServiceabilityCollectorBase from .serviceability_data import ServiceabilityDataModel @@ -44,3 +48,92 @@ class ServiceabilityPluginBase( COLLECTOR = ServiceabilityCollectorBase COLLECTOR_ARGS = CollectorArgs ANALYZER_ARGS = ServiceabilityAnalyzerArgs + + def _apply_afid_sag_path_to_collection_args( + self, + afid_sag_path: str, + collection_args: Any, + ) -> Any: + """Copy a resolved AFID_SAG path onto collector args when the collector stores one. + + Args: + afid_sag_path: Absolute path returned by preflight. + collection_args: Collector args for this run, or None. + + Returns: + Collector args unchanged. Subclasses that store the path override this. + """ + return collection_args + + def _plugin_error_result(self, message: str) -> PluginResult: + """Build a plugin result that skips collection and fails analysis. + + Args: + message: Error recorded on the plugin and analysis results. + + Returns: + Plugin result with collection NOT_RAN and analysis ERROR. + """ + self.logger.error("(%s) %s", self.__class__.__name__, message) + return PluginResult( + status=ExecutionStatus.ERROR, + source=self.__class__.__name__, + message=message, + result_data=DataPluginResult( + collection_result=TaskResult( + status=ExecutionStatus.NOT_RAN, + parent=self.__class__.__name__, + message="Data collection skipped", + ), + analysis_result=TaskResult( + status=ExecutionStatus.ERROR, + parent=self.__class__.__name__, + message=message, + ), + ), + ) + + def _preflight_hub_sag( + self, + collection_args: Any, + analysis_args: Optional[Union[ServiceabilityAnalyzerArgs, dict[str, Any]]], + ) -> Union[ + tuple[Any, Optional[Union[ServiceabilityAnalyzerArgs, dict[str, Any]]]], + PluginResult, + ]: + """Resolve and validate AFID_SAG before collection when an entry-point hub will run. + + Args: + collection_args: Collector args for this run, or None. + analysis_args: Analyzer args for this run, or None. + + Returns: + Updated collector and analyzer args, or a plugin error when preflight fails. + """ + try: + parsed = ( + analysis_args + if isinstance(analysis_args, self.ANALYZER_ARGS) + else self.ANALYZER_ARGS(**(analysis_args or {})) + ) + if parsed.skip_hub: + return collection_args, analysis_args + if not parsed.uses_entry_point_hub(): + preflight_module_hub(parsed, self.logger) + return collection_args, analysis_args + sag_path = preflight_afid_sag(parsed, self.logger) + except (HubRunError, ValueError) as exc: + return self._plugin_error_result(f"AFID_SAG preflight failed: {exc}") + if isinstance(analysis_args, dict): + updated_analysis: Optional[Union[ServiceabilityAnalyzerArgs, dict[str, Any]]] = { + **analysis_args, + "afid_sag_path": sag_path, + } + elif analysis_args is None: + updated_analysis = {"afid_sag_path": sag_path} + else: + updated_analysis = parsed.model_copy(update={"afid_sag_path": sag_path}) + return ( + self._apply_afid_sag_path_to_collection_args(sag_path, collection_args), + updated_analysis, + ) diff --git a/nodescraper/plugins/serviceability/serviceability_recommendations_table.py b/nodescraper/plugins/serviceability/serviceability_recommendations_table.py new file mode 100644 index 00000000..0383cf0a --- /dev/null +++ b/nodescraper/plugins/serviceability/serviceability_recommendations_table.py @@ -0,0 +1,306 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import logging +import sys +from textwrap import wrap +from typing import Any, Optional + +from nodescraper.constants import DEFAULT_LOGGER +from nodescraper.models import PluginResult + +from .se_models import PrioritizedServiceAction, ServiceabilityBlock +from .serviceability_api import afse_top_action_missing, split_recommendation_actions + +RECOMMENDATION_TABLE_HEADERS: tuple[str, ...] = ( + "Rank", + "Priority", + "SA Sev", + "Tier", + "Units", + "Service Action", + "Steps", +) + +PRIMARY_RECOMMENDATION_TITLE = "Top Hub service action" + +RECOMMENDATION_TABLE_MAX_WIDTHS: dict[str, int] = { + "Rank": 4, + "Priority": 8, + "SA Sev": 6, + "Tier": 10, + "Units": 28, + "Service Action": 40, + "Steps": 96, +} + + +def _recommendation_section_title( + title: str, + actions: list[PrioritizedServiceAction], +) -> str: + if not actions: + return title + ranks = [action.rank for action in actions if action.rank] + if len(actions) == 1 and ranks: + return f"{title} (rank {ranks[0]})" + if ranks: + return ( + f"{title} (ranks {min(ranks)}-{max(ranks)}, " + f"{len(actions)} item{'s' if len(actions) != 1 else ''})" + ) + return f"{title} ({len(actions)} item{'s' if len(actions) != 1 else ''})" + + +def _short_serviceable_unit(location: str) -> str: + text = str(location).strip().rstrip("/") + if not text: + return "" + return text.rsplit("/", 1)[-1] + + +def _recommendation_units_cell(action: PrioritizedServiceAction) -> str: + units = list(action.serviceable_units or []) + if not units and action.location: + units = [action.location] + return ", ".join(_short_serviceable_unit(unit) for unit in units if unit) + + +def _recommendation_service_action_cell(action: PrioritizedServiceAction) -> str: + title = (action.service_action_title or "").strip() + if title: + return f"{action.afid}: {title}" + return f"{action.afid}: SA {action.service_action_num}" + + +def _recommendation_steps_cell(action: PrioritizedServiceAction) -> str: + steps = [str(step).strip() for step in action.service_action_steps if str(step).strip()] + if not steps: + return "" + return "; ".join(f"{index}: {step}" for index, step in enumerate(steps)) + + +def _recommendation_table_row(action: PrioritizedServiceAction) -> list[str]: + priority = "" if action.priority is None else str(action.priority) + sa_severity = "" if action.sa_severity is None else str(action.sa_severity) + tier = (action.tier_label or "").strip() + return [ + str(action.rank), + priority, + sa_severity, + tier, + _recommendation_units_cell(action), + _recommendation_service_action_cell(action), + _recommendation_steps_cell(action), + ] + + +def render_recommendations_table( + actions: list[PrioritizedServiceAction], + *, + headers: tuple[str, ...] = RECOMMENDATION_TABLE_HEADERS, + max_widths: Optional[dict[str, int]] = None, +) -> str: + """Render prioritized Hub recommendations as a bordered ASCII table.""" + if max_widths is None: + max_widths = dict(RECOMMENDATION_TABLE_MAX_WIDTHS) + rows = [_recommendation_table_row(action) for action in actions] + return _gen_str_table(list(headers), rows, max_widths=max_widths) + + +def render_recommendations_section( + title: str, + actions: list[PrioritizedServiceAction], + *, + note: Optional[str] = None, +) -> str: + """Render a titled recommendations block with an optional explanatory note.""" + if not actions: + return "" + lines = [title] + if note: + lines.append(note) + lines.append(render_recommendations_table(actions)) + return "\n".join(lines) + + +def _serviceability_block_from_plugin_result( + plugin_result: PluginResult, +) -> Optional[ServiceabilityBlock]: + """Return a ServiceabilityBlock when a plugin result includes hub recommendation data.""" + result_data = plugin_result.result_data + if result_data is None: + return None + + system_data: Any + if isinstance(result_data, dict): + system_data = result_data.get("system_data") + else: + system_data = getattr(result_data, "system_data", None) + if system_data is None: + return None + + serviceability: Any + if isinstance(system_data, dict): + serviceability = system_data.get("serviceability") + else: + serviceability = getattr(system_data, "serviceability", None) + if serviceability is None: + return None + + if isinstance(serviceability, dict): + serviceability = ServiceabilityBlock.model_validate(serviceability) + if not ( + serviceability.hub_triage_results + or serviceability.hub_top_results + or isinstance(serviceability.hub_analyze_response, dict) + ): + return None + return serviceability + + +def render_serviceability_recommendation_tables_for_plugin_results( + plugin_results: list[PluginResult], + logger: Optional[logging.Logger] = None, +) -> str: + """Render Hub recommendation tables for plugins that produced serviceability output. + + Args: + plugin_results: Plugin results from the run. + logger: Logger for the missing-top warning. Defaults to the nodescraper logger. + + Returns: + Joined top-action tables. Empty when AFSE returned no triage.top action. + """ + log = logger or logging.getLogger(DEFAULT_LOGGER) + sections: list[str] = [] + for plugin_result in plugin_results: + block = _serviceability_block_from_plugin_result(plugin_result) + if block is None: + continue + if afse_top_action_missing(block): + log.warning("AFSE did not return a top action item") + continue + rendered = render_serviceability_recommendation_tables(block) + if rendered: + sections.append(rendered) + return "\n\n".join(sections) + + +def render_serviceability_recommendation_tables(block: ServiceabilityBlock) -> str: + """Render the AFSE triage.top service action table. + + Args: + block: Serviceability block populated by hub analysis. + + Returns: + The top service action table. Lower-ranked triage.results rows are omitted. + """ + primary, _ = split_recommendation_actions(block) + return render_recommendations_section( + _recommendation_section_title(PRIMARY_RECOMMENDATION_TITLE, primary), + primary, + ) + + +def emit_serviceability_recommendation_tables(block: ServiceabilityBlock) -> None: + """Print the AFSE top service action table to stdout. + + Args: + block: Serviceability block populated by hub analysis. + + Returns: + None. Logs a warning and prints nothing when AFSE returned no top action. + """ + if afse_top_action_missing(block): + logging.getLogger(DEFAULT_LOGGER).warning("AFSE did not return a top action item") + return + output = render_serviceability_recommendation_tables(block) + if not output: + return + sys.stdout.write(f"\n{output}\n") + + +def _gen_str_table( + headers: list[str], + rows: list[list[str]], + max_widths: Optional[dict[str, int]] = None, +) -> str: + max_widths = max_widths or {} + norm_rows: list[list[str]] = [[str(cell) for cell in row] for row in rows] + ncols = len(headers) + + raw_widths: list[int] = [len(header) for header in headers] + for norm_row in norm_rows: + for index, cell in enumerate(norm_row): + for part in cell.splitlines() or [""]: + if len(part) > raw_widths[index]: + raw_widths[index] = len(part) + + target_widths: list[int] = [] + for index, header in enumerate(headers): + cap = max_widths.get(header) + if cap is None: + target_widths.append(raw_widths[index]) + else: + target_widths.append(max(len(header), min(raw_widths[index], cap))) + + wrapped_rows: list[list[list[str]]] = [] + for norm_row in norm_rows: + wrapped_cells: list[list[str]] = [] + for index, cell in enumerate(norm_row): + cell_lines: list[str] = [] + for paragraph in cell.splitlines() or [""]: + cell_lines.extend(wrap(paragraph, width=target_widths[index]) or [""]) + wrapped_cells.append(cell_lines) + wrapped_rows.append(wrapped_cells) + + col_widths: list[int] = [] + for index in range(ncols): + widest_line = len(headers[index]) + for wrapped_row in wrapped_rows: + for line in wrapped_row[index]: + if len(line) > widest_line: + widest_line = len(line) + col_widths.append(widest_line) + + border = "+" + "+".join("-" * (width + 2) for width in col_widths) + "+" + + def render_physical_row(parts: list[str]) -> str: + return "| " + " | ".join(part.ljust(width) for part, width in zip(parts, col_widths)) + " |" + + table_lines = [border, render_physical_row(headers), border] + for wrapped_row in wrapped_rows: + height = max(len(cell_lines) for cell_lines in wrapped_row) + for line_index in range(height): + parts = [ + wrapped_row[column][line_index] if line_index < len(wrapped_row[column]) else "" + for column in range(ncols) + ] + table_lines.append(render_physical_row(parts)) + table_lines.append(border) + return "\n".join(table_lines) diff --git a/nodescraper/resultcollators/tablesummary.py b/nodescraper/resultcollators/tablesummary.py index 49f816b2..e8b5b2f3 100644 --- a/nodescraper/resultcollators/tablesummary.py +++ b/nodescraper/resultcollators/tablesummary.py @@ -28,6 +28,9 @@ from nodescraper.interfaces import PluginResultCollator from nodescraper.models import PluginResult, TaskResult +from nodescraper.plugins.serviceability.serviceability_recommendations_table import ( + render_serviceability_recommendation_tables_for_plugin_results, +) class TableSummary(PluginResultCollator): @@ -119,7 +122,8 @@ def render_physical_row(parts: list[str]) -> str: table_lines.append(border) return "\n".join(table_lines) - tables = "" + table_sections: list[str] = [] + if connection_results: conn_rows: list[list[Optional[str]]] = [] for connection_result in connection_results: @@ -136,7 +140,14 @@ def render_physical_row(parts: list[str]) -> str: conn_rows, max_widths={"Connection": 32, "Status": 20, "Message": 80}, ) - tables += f"\n\n{table}" + table_sections.append(table) + + serviceability_tables = render_serviceability_recommendation_tables_for_plugin_results( + plugin_results, + logger=self.logger, + ) + if serviceability_tables: + table_sections.append(serviceability_tables) if plugin_results: plug_rows: list[list[Optional[str]]] = [] @@ -153,7 +164,8 @@ def render_physical_row(parts: list[str]) -> str: plug_rows, max_widths={"Plugin": 32, "Status": 20, "Message": 80}, ) - tables += f"\n\n{table}" + table_sections.append(table) - if tables: + if table_sections: + tables = "\n\n".join(f"\n{section}" for section in table_sections) self.logger.info("%s\n", tables) diff --git a/nodescraper/taskresulthooks/filesystemloghook.py b/nodescraper/taskresulthooks/filesystemloghook.py index 50184b4e..40656f70 100644 --- a/nodescraper/taskresulthooks/filesystemloghook.py +++ b/nodescraper/taskresulthooks/filesystemloghook.py @@ -31,23 +31,68 @@ from nodescraper.utils import resolve_log_dir_name +def _next_log_suffix(log_path: str) -> str: + """First suffix ("", "(1)", "(2)", ...) for which neither result nor events file exists in log_path.""" + count = 0 + while True: + suffix = f"({count})" if count else "" + if not any( + os.path.exists(os.path.join(log_path, f"{name}{suffix}.json")) + for name in ("result", "events") + ): + return suffix + count += 1 + + class FileSystemLogHook(TaskResultHook): + def __init__(self, log_base_path=None, include_task_path: bool = True, **kwargs) -> None: + """Create a FileSystemLogHook Instance - def __init__(self, log_base_path=None, **kwargs) -> None: + Args: + log_base_path (Optional[str], optional): The base path where logs will be stored. Defaults to the current working directory. + include_task_path: Append parent and task directory names when True. + **kwargs: Additional keyword arguments, which are not used. + """ if log_base_path is None: log_base_path = os.getcwd() self.log_base_path = log_base_path + self.include_task_path = include_task_path def process_result(self, task_result: TaskResult, data: Optional[DataModel] = None, **kwargs): - """Log task result to the filesystem (single events.json per directory).""" + """Log task result to the filesystem. + + The first result written to a directory uses ``result.json`` / ``events.json``. Later results + sent to the same directory get an integer appended (``result(1).json`` / ``events(1).json``) + so earlier logs are never overwritten. + """ log_path = self.log_base_path - if task_result.parent: - log_path = os.path.join(log_path, resolve_log_dir_name(task_result.parent)) - if task_result.task: - log_path = os.path.join(log_path, resolve_log_dir_name(task_result.task)) + if self.include_task_path: + if task_result.parent: + log_path = os.path.join(log_path, resolve_log_dir_name(task_result.parent)) + if task_result.task: + log_path = os.path.join(log_path, resolve_log_dir_name(task_result.task)) - task_result.log_result(log_path) + task_result.log_result(log_path, _next_log_suffix(log_path)) if data: data.log_model(log_path) + + +def hooks_for_fixed_directory(hooks: list, directory: str) -> list: + """Point filesystem log hooks at one directory. + + Args: + hooks: Hooks copied from the parent task. + directory: Directory that should receive this task's logs. + + Returns: + list: Hooks with filesystem logs written directly into directory. + """ + rewritten = [] + for hook in hooks: + if isinstance(hook, FileSystemLogHook): + rewritten.append(FileSystemLogHook(log_base_path=directory, include_task_path=False)) + else: + rewritten.append(hook) + return rewritten diff --git a/nodescraper/typeutils.py b/nodescraper/typeutils.py index bc4ce244..440a8a2b 100644 --- a/nodescraper/typeutils.py +++ b/nodescraper/typeutils.py @@ -25,7 +25,17 @@ ############################################################################### import inspect import types -from typing import Annotated, Any, Callable, Optional, Type, Union, get_args, get_origin +from typing import ( + Annotated, + Any, + Callable, + Literal, + Optional, + Type, + Union, + get_args, + get_origin, +) from pydantic import BaseModel, Field @@ -34,6 +44,7 @@ class TypeClass(BaseModel): """Class to hold type class information""" type_class: Any + # for a Literal type class this holds the full list of allowed values inner_type: Optional[Any] = None @@ -110,6 +121,26 @@ def get_func_arg_types( return type_map + @classmethod + def build_type_class(cls, input_type: Any, origin: Any) -> TypeClass: + """Build a TypeClass for a parameterized type + + Args: + input_type (Any): parameterized type, e.g. list[str] or Literal["a", "b"] + origin (Any): origin of the type + + Returns: + TypeClass: type class with inner type details + """ + if origin is Literal: + # args of a Literal are values rather than types, so keep all of them + return TypeClass(type_class=Literal, inner_type=list(get_args(input_type))) + + return TypeClass( + type_class=origin, + inner_type=next((arg for arg in get_args(input_type) if arg is not type(None)), None), + ) + @classmethod def process_type(cls, input_type: type[Any]) -> list[TypeClass]: """Process a type to extract its class and any inner types @@ -131,28 +162,17 @@ def process_type(cls, input_type: type[Any]) -> list[TypeClass]: input_types = [arg for arg in input_type.__args__ if arg is not type(None)] for type_item in input_types: origin = get_origin(type_item) + if origin is Annotated: + type_item = get_args(type_item)[0] + origin = get_origin(type_item) if origin is None: type_classes.append(TypeClass(type_class=type_item)) else: - type_classes.append( - TypeClass( - type_class=origin, - inner_type=next( - (arg for arg in get_args(type_item) if arg is not type(None)), None - ), - ) - ) + type_classes.append(cls.build_type_class(type_item, origin)) return type_classes else: - return [ - TypeClass( - type_class=origin, - inner_type=next( - (arg for arg in get_args(input_type) if arg is not type(None)), None - ), - ) - ] + return [cls.build_type_class(input_type, origin)] @classmethod def get_model_types(cls, model: type[BaseModel]) -> dict[str, TypeData]: @@ -175,3 +195,22 @@ def get_model_types(cls, model: type[BaseModel]) -> dict[str, TypeData]: ) return type_map + + @staticmethod + def filter_to_model_fields( + model: Union[BaseModel, Type[BaseModel]], data: dict[str, Any] + ) -> dict[str, Any]: + """Keep only the entries of ``data`` whose keys are fields of a pydantic model + + Use this to apply shared (global) arguments to a model that forbids extra keys, so keys + meant for other models are dropped instead of raising a validation error. + + Args: + model (Union[BaseModel, Type[BaseModel]]): model class or instance whose fields are allowed + data (dict[str, Any]): arguments to filter; not modified + + Returns: + dict[str, Any]: new dict with only the keys that are fields of the model + """ + model_cls = model if isinstance(model, type) else type(model) + return {key: value for key, value in data.items() if key in model_cls.model_fields} diff --git a/nodescraper/utils.py b/nodescraper/utils.py index 11c3ab57..389f27ce 100644 --- a/nodescraper/utils.py +++ b/nodescraper/utils.py @@ -44,6 +44,69 @@ T = TypeVar("T") +def _validate_firmware_policy(records: list[Dict[str, Any]], policies: Any) -> list[Dict[str, Any]]: + """Validate firmware records against analyzer policies.""" + policy_list = [policies] if isinstance(policies, dict) else policies or [] + issues = [] + for record in records: + policy = next( + ( + item + for item in policy_list + if isinstance(item, dict) and _policy_matches(record, item) + ), + None, + ) + if policy is None: + continue + actual = _normalize_firmware_version(record.get("version")) + expected = _normalize_firmware_version(policy.get("expected_nic_firmware")) + if not actual: + reason = "firmware version is unavailable" + elif expected and not _firmware_matches(actual, expected): + reason = ( + f"actual {record.get('version')} != expected_nic_firmware " + f"{policy.get('expected_nic_firmware')}" + ) + else: + continue + issues.append({"reason": reason, "record": record, "policy": dict(policy)}) + return issues + + +def _policy_matches(record: Dict[str, Any], policy: Dict[str, Any]) -> bool: + match = policy.get("match", {}) + if not isinstance(match, dict): + return False + if not match: + match = {field: policy[field] for field in record if field in policy} + for field, expected in match.items(): + actual = record.get(str(field)) + if actual is None: + return False + expected_values = expected if isinstance(expected, list) else [expected] + if not any( + str(actual).strip().lower() == str(value).strip().lower() for value in expected_values + ): + return False + return True + + +def _normalize_firmware_version(value: Any) -> Optional[str]: + if value is None: + return None + normalized = str(value).strip().strip("'\"").lower() + return normalized or None + + +def _firmware_matches(actual: str, expected_pattern: str) -> bool: + try: + firmware_regex = re.compile(expected_pattern, re.IGNORECASE) + except re.error: + return False + return firmware_regex.match(actual) is not None + + class AutoNameStrEnum(Enum): """For enums where the value is the same as the name of the attribute""" @@ -125,11 +188,12 @@ def command_result_event_data( return data -def convert_to_bytes(value: str, si=False) -> int: +def convert_to_bytes(value: str, si=True) -> int: """ Convert human-readable memory sizes (like GB, MB) to bytes. - Default to use IEC units. - Factor of powers of 2, not 10. (e.g. 1KB is interpeted as 1KiB=1024 bytes) + If si=False (default), IEC units are used (1KiB=1024 bytes). + If si=True, SI units are used (1KB=1000 bytes). + Factor of powers of 2, not 10. (e.g. 1KB=1000 bytes and 1KiB=1024 bytes) This can be changed with si=True (1KB=1000 bytes) """ value = value.strip().upper() @@ -220,6 +284,25 @@ def pascal_to_snake(input_str: str) -> str: return normalized.lower() +# Byte multiplier for every size unit that a firmware source may report. +SIZE_UNITS = { + "B": 1, + "KB": 1024, + "KIB": 1024, + "MB": 1024**2, + "MIB": 1024**2, + "GB": 1024**3, + "GIB": 1024**3, + "TB": 1024**4, + "TIB": 1024**4, + "PB": 1024**5, + "PIB": 1024**5, +} +HUMAN_SIZES = { + k: v for k, v in SIZE_UNITS.items() if k.upper() in ["B", "KB", "MB", "GB", "TB", "PB"] +} + + def bytes_to_human_readable(input_bytes: int) -> str: """Converts a bytes int to a human-readable string in B, KB, MB, GB, TB, or PB (decimal). @@ -233,13 +316,33 @@ def bytes_to_human_readable(input_bytes: int) -> str: return "0B" if input_bytes == 0: return "0B" - units = [(10**12, "TB"), (10**9, "GB"), (10**6, "MB"), (10**3, "KB"), (1, "B")] - for scale, label in units: + sizes_reverse_size = sorted(HUMAN_SIZES.items(), key=lambda x: x[1], reverse=True) + for name, scale in sizes_reverse_size: if input_bytes >= scale: - return f"{round(float(input_bytes) / scale, 2)}{label}" + return f"{round(float(input_bytes) / scale, 2)} {name}" return "0B" +def format_bytes(size_bytes: int) -> str: + """Render a byte count using the largest binary unit that divides it exactly. + + Unlike :func:`bytes_to_human_readable`, this never rounds: a count that no + unit divides exactly is rendered in bytes. + + Args: + size_bytes (int): size in bytes. + + Returns: + str: human readable size, e.g. "64GB". + """ + sizes_reverse_size = sorted(SIZE_UNITS.items(), key=lambda x: x[1], reverse=True) + for unit, factor in sizes_reverse_size: + if unit == "B": + return f"{size_bytes}B" + if size_bytes >= factor and not size_bytes % factor: + return f"{size_bytes // factor}{unit}" + + def find_annotation_in_container( annotation, target_type ) -> Union[tuple[Any, list[Any]], tuple[None, list[Any]]]: @@ -278,6 +381,9 @@ def find_annotation_in_container( if result: containers.append(origin) return result, containers + # Check if it is not a type cause if you put an origin in its anything + if not isinstance(item, type): + item = type(item) if len(get_args(item)) == 0 and issubclass(item, target_type): containers.append(origin) return item, containers diff --git a/plugin_config_mi4xx_example.json b/plugin_config_mi4xx_example.json deleted file mode 100644 index fa150b45..00000000 --- a/plugin_config_mi4xx_example.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "Mi4xxServiceability", - "desc": "Helios MI4xx — Instinct accelerator event log + configured service hub entry point", - "global_args": {}, - "plugins": { - "Mi4xxServiceabilityPlugin": { - "collection": true, - "analysis": true, - "collection_args": { - "rf_event_log_uri": "/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries", - "follow_next_link": true, - "max_pages": 200 - }, - "analysis_args": { - "afid_sag_path": "/opt/amd/afid/AFID_SAG.json", - "hub_entry_point": "hub", - "rf_event_log_uri": "/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries" - } - } - }, - "result_collators": {} -} diff --git a/pyproject.toml b/pyproject.toml index d2f1bdef..f7a887ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,23 +29,6 @@ dependencies = [ "urllib3>=2.6.0" ] -[project.optional-dependencies] -dev = [ - "build", - "black", - "pylint", - "coverage", - "twine", - "ruff", - "pre-commit", - "pytest", - "pytest-cov", - "mypy", - "types-paramiko", - "types-requests", - "types-setuptools", -] - [project.urls] homepage = "https://github.com/amd/node-scraper" documentation = "https://github.com/amd/node-scraper/blob/main/docs/PLUGIN_DOC.md" @@ -69,9 +52,56 @@ nodescraper = ["py.typed"] [project.scripts] node-scraper = "nodescraper.cli:cli_entry" +[project.entry-points."nodescraper.plugins"] +AmcRedfishDiagPlugin = "nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_plugin:AmcRedfishDiagPlugin" +AmcRedfishEndpointPlugin = "nodescraper.plugins.ooband.redfish_endpoint.endpoint_plugin:AmcRedfishEndpointPlugin" +AmcRedfishEventLogPlugin = "nodescraper.plugins.ooband.redfish_event_log.event_log_plugin:AmcRedfishEventLogPlugin" +AmcRedfishInventoryPlugin = "nodescraper.plugins.ooband.redfish_inventory.inventory_plugin:AmcRedfishInventoryPlugin" +AmdSmiFlavorPlugin = "nodescraper.plugins.inband.amdsmi.amdsmi_flavor_plugin:AmdSmiFlavorPlugin" +AmdSmiPlugin = "nodescraper.plugins.inband.amdsmi.amdsmi_plugin:AmdSmiPlugin" +BiosPlugin = "nodescraper.plugins.inband.bios.bios_plugin:BiosPlugin" +CmdlinePlugin = "nodescraper.plugins.inband.cmdline.cmdline_plugin:CmdlinePlugin" +DeviceEnumerationPlugin = "nodescraper.plugins.inband.device_enumeration.device_enumeration_plugin:DeviceEnumerationPlugin" +DimmPlugin = "nodescraper.plugins.inband.dimm.dimm_plugin:DimmPlugin" +DkmsPlugin = "nodescraper.plugins.inband.dkms.dkms_plugin:DkmsPlugin" +DmesgPlugin = "nodescraper.plugins.inband.dmesg.dmesg_plugin:DmesgPlugin" +FabricsPlugin = "nodescraper.plugins.inband.fabrics.fabrics_plugin:FabricsPlugin" +GenericCollectionPlugin = "nodescraper.plugins.generic_collection.inband_plugin:GenericCollectionPlugin" +JournalPlugin = "nodescraper.plugins.inband.journal.journal_plugin:JournalPlugin" +KernelModulePlugin = "nodescraper.plugins.inband.kernel_module.kernel_module_plugin:KernelModulePlugin" +KernelPlugin = "nodescraper.plugins.inband.kernel.kernel_plugin:KernelPlugin" +MemoryPlugin = "nodescraper.plugins.inband.memory.memory_plugin:MemoryPlugin" +Mi4xxServiceabilityPlugin = "nodescraper.plugins.serviceability.mi4xx.serviceability_plugin_mi4xx:Mi4xxServiceabilityPlugin" +NetworkPlugin = "nodescraper.plugins.inband.network.network_plugin:NetworkPlugin" +NicPlugin = "nodescraper.plugins.inband.nic.nic_plugin:NicPlugin" +NvmePlugin = "nodescraper.plugins.inband.nvme.nvme_plugin:NvmePlugin" +OobBmcArchivePlugin = "nodescraper.plugins.ooband.bmc_archive.bmc_archive_plugin:OobBmcArchivePlugin" +OobGenericCollectionPlugin = "nodescraper.plugins.generic_collection.oob_plugin:OobGenericCollectionPlugin" +OsPlugin = "nodescraper.plugins.inband.os.os_plugin:OsPlugin" +PackagePlugin = "nodescraper.plugins.inband.package.package_plugin:PackagePlugin" +PciePlugin = "nodescraper.plugins.inband.pcie.pcie_plugin:PciePlugin" +ProcessPlugin = "nodescraper.plugins.inband.process.process_plugin:ProcessPlugin" +RdmaPlugin = "nodescraper.plugins.inband.rdma.rdma_plugin:RdmaPlugin" +RedfishEndpointPlugin = "nodescraper.plugins.ooband.redfish_endpoint.endpoint_plugin:RedfishEndpointPlugin" +RedfishEventLogPlugin = "nodescraper.plugins.ooband.redfish_event_log.event_log_plugin:RedfishEventLogPlugin" +RedfishInventoryPlugin = "nodescraper.plugins.ooband.redfish_inventory.inventory_plugin:RedfishInventoryPlugin" +RedfishOemDiagPlugin = "nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_plugin:RedfishOemDiagPlugin" +RegexSearchPlugin = "nodescraper.plugins.inband.regex_search.regex_search_plugin:RegexSearchPlugin" +RocmPlugin = "nodescraper.plugins.inband.rocm.rocm_plugin:RocmPlugin" +ScaleOutAristaPlugin = "nodescraper.plugins.inband.switch.scale_out_arista.scale_out_arista_plugin:ScaleOutAristaPlugin" +ScaleOutDellPlugin = "nodescraper.plugins.inband.switch.scale_out_dell.scale_out_dell_plugin:ScaleOutDellPlugin" +ScaleUpSonicPlugin = "nodescraper.plugins.inband.switch.scale_up_sonic.scale_up_sonic_plugin:ScaleUpSonicPlugin" +ServiceabilityPluginBase = "nodescraper.plugins.serviceability.serviceability_plugin_base:ServiceabilityPluginBase" +ServiceabilityPluginMI3XX = "nodescraper.plugins.serviceability.mi3xx.serviceability_plugin_mi3xx:ServiceabilityPluginMI3XX" +StoragePlugin = "nodescraper.plugins.inband.storage.storage_plugin:StoragePlugin" +SysSettingsPlugin = "nodescraper.plugins.inband.sys_settings.sys_settings_plugin:SysSettingsPlugin" +SysctlPlugin = "nodescraper.plugins.inband.sysctl.sysctl_plugin:SysctlPlugin" +SyslogPlugin = "nodescraper.plugins.inband.syslog.syslog_plugin:SyslogPlugin" +UptimePlugin = "nodescraper.plugins.inband.uptime.uptime_plugin:UptimePlugin" + [project.entry-points."nodescraper.plugin_configs"] NodeStatus = "nodescraper.pluginrecipe.node_status:NodeStatus" -AllPlugins = "nodescraper.pluginrecipe.all_plugins:AllPlugins" +AllIbPlugins = "nodescraper.pluginrecipe.all_ib_plugins:AllIbPlugins" [tool.black] line-length = 100 @@ -91,3 +121,20 @@ explicit_package_bases = true [tool.setuptools_scm] version_scheme = "post-release" + +[dependency-groups] +dev = [ + "build", + "black", + "pylint", + "coverage", + "twine", + "ruff", + "pre-commit", + "pytest", + "pytest-cov", + "mypy", + "types-paramiko", + "types-requests", + "types-setuptools", +] diff --git a/test/functional/fixtures/dmesg_sample.log b/test/functional/fixtures/dmesg_sample.log index e0844e42..98518d45 100644 --- a/test/functional/fixtures/dmesg_sample.log +++ b/test/functional/fixtures/dmesg_sample.log @@ -1,71 +1,71 @@ -kern :info : 2026-01-07T10:00:00,123456-06:00 Linux version 5.15.0-91-generic (buildd@amd64-builder) (gcc version 11.4.0) #101-Ubuntu SMP -kern :info : 2026-01-07T10:00:00,234567-06:00 Command line: BOOT_IMAGE=/boot/vmlinuz-5.15.0-91-generic root=UUID=a1b2c3d4 ro quiet splash vt.handoff=7 -kern :info : 2026-01-07T10:00:01,345678-06:00 KERNEL supported cpus: -kern :info : 2026-01-07T10:00:01,456789-06:00 Intel GenuineIntel -kern :info : 2026-01-07T10:00:01,567890-06:00 AMD AuthenticAMD -kern :info : 2026-01-07T10:00:02,678901-06:00 x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers' -kern :info : 2026-01-07T10:00:02,789012-06:00 x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers' -kern :info : 2026-01-07T10:00:03,890123-06:00 Memory: 32823616K/33554432K available -kern :warn : 2026-01-07T10:00:05,123456-06:00 pci 0000:00:01.0: BAR 0: failed to assign [mem size 0x01000000] -kern :info : 2026-01-07T10:00:06,234567-06:00 PCI: Using ACPI for IRQ routing -kern :info : 2026-01-07T10:00:07,345678-06:00 NetLabel: Initializing -kern :info : 2026-01-07T10:00:08,456789-06:00 DMA: preallocated 4096 KiB GFP_KERNEL pool for atomic allocations -kern :err : 2026-01-07T10:00:10,567890-06:00 WARNING: CPU: 0 PID: 1 at drivers/gpu/drm/amd/amdgpu/amdgpu_device.c:123 amdgpu_device_init+0x456/0x789 -kern :info : 2026-01-07T10:00:11,678901-06:00 Modules linked in: amdgpu drm_ttm_helper ttm drm_kms_helper -kern :info : 2026-01-07T10:00:12,789012-06:00 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.15.0-91-generic #101-Ubuntu -kern :err : 2026-01-07T10:00:15,890123-06:00 AMD-Vi: Event logged [IO_PAGE_FAULT device=00:14.0 domain=0x0000 address=0xfffffffffffffef0 flags=0x0010] -kern :info : 2026-01-07T10:00:16,123456-06:00 SCSI subsystem initialized -kern :info : 2026-01-07T10:00:17,234567-06:00 libata version 3.00 loaded -kern :info : 2026-01-07T10:00:18,345678-06:00 ACPI: Added _OSI(Module Device) -kern :info : 2026-01-07T10:00:19,456789-06:00 ACPI: Added _OSI(Processor Device) -kern :err : 2026-01-07T10:00:20,567890-06:00 ACPI Error: Method parse/execution failed \_SB.PCI0.GPP0.SWUS.SWDS.VGA.LCD._BCM, AE_NOT_FOUND -kern :info : 2026-01-07T10:00:22,678901-06:00 [drm] amdgpu kernel modesetting enabled -kern :info : 2026-01-07T10:00:23,789012-06:00 [drm] initializing kernel modesetting (NAVI21 0x1002:0x73BF) -kern :info : 2026-01-07T10:00:25,890123-06:00 amdgpu 0000:03:00.0: amdgpu: Fetched VBIOS from VFCT -kern :info : 2026-01-07T10:00:26,123456-06:00 amdgpu 0000:03:00.0: amdgpu: ATOM BIOS: 113-D4120100-O04 -kern :info : 2026-01-07T10:00:28,234567-06:00 [drm] GPU posting now... -kern :warn : 2026-01-07T10:00:30,345678-06:00 [drm] *ERROR* Timeout waiting for DMCUB auto-load -kern :info : 2026-01-07T10:00:32,456789-06:00 [drm] Display Core initialized with v3.2.149! -kern :info : 2026-01-07T10:00:35,567890-06:00 [drm] VCN decode and encode initialized successfully -kern :info : 2026-01-07T10:00:38,678901-06:00 [drm] fb0: amdgpudrmfb frame buffer device -kern :info : 2026-01-07T10:00:40,789012-06:00 amdgpu 0000:03:00.0: amdgpu: ring gfx_0.0.0 uses VM inv eng 0 on hub 0 -kern :info : 2026-01-07T10:00:42,890123-06:00 [drm] Initialized amdgpu 3.42.0 20150101 for 0000:03:00.0 on minor 0 -kern :info : 2026-01-07T10:00:45,123456-06:00 EXT4-fs (nvme0n1p2): mounted filesystem with ordered data mode -kern :info : 2026-01-07T10:00:48,234567-06:00 systemd[1]: systemd 249.11-0ubuntu3.6 running in system mode -kern :info : 2026-01-07T10:00:50,345678-06:00 systemd[1]: Detected architecture x86-64 -kern :info : 2026-01-07T10:00:55,456789-06:00 audit: type=1400 audit(1704636055.456:2): apparmor="STATUS" operation="profile_load" -kern :info : 2026-01-07T10:01:00,567890-06:00 Adding 33554428k swap on /swapfile -kern :info : 2026-01-07T10:01:05,678901-06:00 IPv6: ADDRCONF(NETDEV_CHANGE): enp5s0: link becomes ready -kern :info : 2026-01-07T10:01:10,789012-06:00 NFSD: Using UMH upcall client tracking operations -kern :info : 2026-01-07T10:01:15,890123-06:00 NFSD: starting 90-second grace period (net f0000098) -kern :info : 2026-01-07T10:01:20,123456-06:00 Bluetooth: BNEP (Ethernet Emulation) ver 1.3 -kern :info : 2026-01-07T10:01:25,234567-06:00 Bluetooth: BNEP filters: protocol multicast -kern :info : 2026-01-07T10:01:30,345678-06:00 System operational - all services started successfully -kern :err : 2026-01-07T10:02:00,123456-06:00 amdgpu 0000:01:00.0: {14}poison is consumed by client 12, kick off gpu reset flow -kern :err : 2026-01-07T10:02:00,234567-06:00 amdgpu 0000:01:00.0: {15}Poison is created -kern :err : 2026-01-07T10:02:01,345678-06:00 amdgpu 0000:02:00.0: amdgpu: Saved bad pages 150 reaches threshold value 128 -kern :err : 2026-01-07T10:03:00,456789-06:00 {1}[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 1 -kern :err : 2026-01-07T10:03:00,567890-06:00 {1}[Hardware Error]: event severity: recoverable -kern :err : 2026-01-07T10:03:00,678901-06:00 {1}[Hardware Error]: Error 0, type: recoverable -kern :err : 2026-01-07T10:03:00,789012-06:00 {1}[Hardware Error]: section_type: PCIe error -kern :err : 2026-01-07T10:03:00,890123-06:00 {1}[Hardware Error]: port_type: 4, root port -kern :err : 2026-01-07T10:03:01,123456-06:00 {1}[Hardware Error]: version: 3.0 -kern :err : 2026-01-07T10:03:01,234567-06:00 {1}[Hardware Error]: command: 0x0547, status: 0x4010 -kern :err : 2026-01-07T10:03:01,345678-06:00 {1}[Hardware Error]: device_id: 0000:11:01.0 -kern :err : 2026-01-07T10:03:01,456789-06:00 {1}[Hardware Error]: slot: 5 -kern :err : 2026-01-07T10:03:01,567890-06:00 {1}[Hardware Error]: secondary_bus: 0x12 -kern :err : 2026-01-07T10:03:01,678901-06:00 {1}[Hardware Error]: vendor_id: 0x1002, device_id: 0x74a1 -kern :err : 2026-01-07T10:03:01,789012-06:00 {1}[Hardware Error]: class_code: 060400 -kern :err : 2026-01-07T10:03:01,890123-06:00 {1}[Hardware Error]: bridge: secondary_status: 0x2000, control: 0x0003 -kern :err : 2026-01-07T10:04:00,123456-06:00 amdgpu 0000:02:00.0: amdgpu: Error Address(PA):0x12345000 Row:0x1000 Col:0x0 Bank:0x5 Channel:0x10 -kern :err : 2026-01-07T10:05:00,123456-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: Accelerator Check Architecture events logged -kern :err : 2026-01-07T10:05:00,234567-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].CONTROL=0x00000000000003ff -kern :err : 2026-01-07T10:05:00,345678-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].STATUS=0x98000800003e0000 -kern :err : 2026-01-07T10:05:00,456789-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].ADDR=0x0000000000000000 -kern :err : 2026-01-07T10:05:00,567890-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].MISC=0xd008000000000000 -kern :err : 2026-01-07T10:05:00,678901-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].CONFIG=0x00000021000001fb -kern :err : 2026-01-07T10:05:00,789012-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].IPID=0x0001100138430401 -kern :err : 2026-01-07T10:05:00,890123-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].SYND=0x0000000000000000 -kern :err : 2026-01-07T10:05:01,123456-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].DESTAT=0x0000000000000000 -kern :err : 2026-01-07T10:05:01,234567-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].DEADDR=0x0000000000000000 -kern :err : 2026-01-07T10:05:01,345678-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].CONTROL_MASK=0x0000000000000000 +kern :info : 2026-01-07T10:00:00,123456-06:00 Linux version 5.15.0-91-generic (buildd@amd64-builder) (gcc version 11.4.0) #101-Ubuntu SMP +kern :info : 2026-01-07T10:00:00,234567-06:00 Command line: BOOT_IMAGE=/boot/vmlinuz-5.15.0-91-generic root=UUID=a1b2c3d4 ro quiet splash vt.handoff=7 +kern :info : 2026-01-07T10:00:01,345678-06:00 KERNEL supported cpus: +kern :info : 2026-01-07T10:00:01,456789-06:00 Intel GenuineIntel +kern :info : 2026-01-07T10:00:01,567890-06:00 AMD AuthenticAMD +kern :info : 2026-01-07T10:00:02,678901-06:00 x86/fpu: Supporting XSAVE feature 0x001: 'x87 floating point registers' +kern :info : 2026-01-07T10:00:02,789012-06:00 x86/fpu: Supporting XSAVE feature 0x002: 'SSE registers' +kern :info : 2026-01-07T10:00:03,890123-06:00 Memory: 32823616K/33554432K available +kern :warn : 2026-01-07T10:00:05,123456-06:00 pci 0000:00:01.0: BAR 0: failed to assign [mem size 0x01000000] +kern :info : 2026-01-07T10:00:06,234567-06:00 PCI: Using ACPI for IRQ routing +kern :info : 2026-01-07T10:00:07,345678-06:00 NetLabel: Initializing +kern :info : 2026-01-07T10:00:08,456789-06:00 DMA: preallocated 4096 KiB GFP_KERNEL pool for atomic allocations +kern :err : 2026-01-07T10:00:10,567890-06:00 WARNING: CPU: 0 PID: 1 at drivers/gpu/drm/amd/amdgpu/amdgpu_device.c:123 amdgpu_device_init+0x456/0x789 +kern :info : 2026-01-07T10:00:11,678901-06:00 Modules linked in: amdgpu drm_ttm_helper ttm drm_kms_helper +kern :info : 2026-01-07T10:00:12,789012-06:00 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.15.0-91-generic #101-Ubuntu +kern :err : 2026-01-07T10:00:15,890123-06:00 AMD-Vi: Event logged [IO_PAGE_FAULT device=00:14.0 domain=0x0000 address=0xfffffffffffffef0 flags=0x0010] +kern :info : 2026-01-07T10:00:16,123456-06:00 SCSI subsystem initialized +kern :info : 2026-01-07T10:00:17,234567-06:00 libata version 3.00 loaded +kern :info : 2026-01-07T10:00:18,345678-06:00 ACPI: Added _OSI(Module Device) +kern :info : 2026-01-07T10:00:19,456789-06:00 ACPI: Added _OSI(Processor Device) +kern :err : 2026-01-07T10:00:20,567890-06:00 ACPI Error: Method parse/execution failed \_SB.PCI0.GPP0.SWUS.SWDS.VGA.LCD._BCM, AE_NOT_FOUND +kern :info : 2026-01-07T10:00:22,678901-06:00 [drm] amdgpu kernel modesetting enabled +kern :info : 2026-01-07T10:00:23,789012-06:00 [drm] initializing kernel modesetting (NAVI21 0x1002:0x73BF) +kern :info : 2026-01-07T10:00:25,890123-06:00 amdgpu 0000:03:00.0: amdgpu: Fetched VBIOS from VFCT +kern :info : 2026-01-07T10:00:26,123456-06:00 amdgpu 0000:03:00.0: amdgpu: ATOM BIOS: 113-D4120100-O04 +kern :info : 2026-01-07T10:00:28,234567-06:00 [drm] GPU posting now... +kern :warn : 2026-01-07T10:00:30,345678-06:00 [drm] *ERROR* Timeout waiting for DMCUB auto-load +kern :info : 2026-01-07T10:00:32,456789-06:00 [drm] Display Core initialized with v3.2.149! +kern :info : 2026-01-07T10:00:35,567890-06:00 [drm] VCN decode and encode initialized successfully +kern :info : 2026-01-07T10:00:38,678901-06:00 [drm] fb0: amdgpudrmfb frame buffer device +kern :info : 2026-01-07T10:00:40,789012-06:00 amdgpu 0000:03:00.0: amdgpu: ring gfx_0.0.0 uses VM inv eng 0 on hub 0 +kern :info : 2026-01-07T10:00:42,890123-06:00 [drm] Initialized amdgpu 3.42.0 20150101 for 0000:03:00.0 on minor 0 +kern :info : 2026-01-07T10:00:45,123456-06:00 EXT4-fs (nvme0n1p2): mounted filesystem with ordered data mode +kern :info : 2026-01-07T10:00:48,234567-06:00 systemd[1]: systemd 249.11-0ubuntu3.6 running in system mode +kern :info : 2026-01-07T10:00:50,345678-06:00 systemd[1]: Detected architecture x86-64 +kern :info : 2026-01-07T10:00:55,456789-06:00 audit: type=1400 audit(1704636055.456:2): apparmor="STATUS" operation="profile_load" +kern :info : 2026-01-07T10:01:00,567890-06:00 Adding 33554428k swap on /swapfile +kern :info : 2026-01-07T10:01:05,678901-06:00 IPv6: ADDRCONF(NETDEV_CHANGE): enp5s0: link becomes ready +kern :info : 2026-01-07T10:01:10,789012-06:00 NFSD: Using UMH upcall client tracking operations +kern :info : 2026-01-07T10:01:15,890123-06:00 NFSD: starting 90-second grace period (net f0000098) +kern :info : 2026-01-07T10:01:20,123456-06:00 Bluetooth: BNEP (Ethernet Emulation) ver 1.3 +kern :info : 2026-01-07T10:01:25,234567-06:00 Bluetooth: BNEP filters: protocol multicast +kern :info : 2026-01-07T10:01:30,345678-06:00 System operational - all services started successfully +kern :err : 2026-01-07T10:02:00,123456-06:00 amdgpu 0000:01:00.0: {14}poison is consumed by client 12, kick off gpu reset flow +kern :err : 2026-01-07T10:02:00,234567-06:00 amdgpu 0000:01:00.0: {15}Poison is created +kern :err : 2026-01-07T10:02:01,345678-06:00 amdgpu 0000:02:00.0: amdgpu: Saved bad pages 150 reaches threshold value 128 +kern :err : 2026-01-07T10:03:00,456789-06:00 {1}[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 1 +kern :err : 2026-01-07T10:03:00,567890-06:00 {1}[Hardware Error]: event severity: recoverable +kern :err : 2026-01-07T10:03:00,678901-06:00 {1}[Hardware Error]: Error 0, type: recoverable +kern :err : 2026-01-07T10:03:00,789012-06:00 {1}[Hardware Error]: section_type: PCIe error +kern :err : 2026-01-07T10:03:00,890123-06:00 {1}[Hardware Error]: port_type: 4, root port +kern :err : 2026-01-07T10:03:01,123456-06:00 {1}[Hardware Error]: version: 3.0 +kern :err : 2026-01-07T10:03:01,234567-06:00 {1}[Hardware Error]: command: 0x0547, status: 0x4010 +kern :err : 2026-01-07T10:03:01,345678-06:00 {1}[Hardware Error]: device_id: 0000:11:01.0 +kern :err : 2026-01-07T10:03:01,456789-06:00 {1}[Hardware Error]: slot: 5 +kern :err : 2026-01-07T10:03:01,567890-06:00 {1}[Hardware Error]: secondary_bus: 0x12 +kern :err : 2026-01-07T10:03:01,678901-06:00 {1}[Hardware Error]: vendor_id: 0x1002, device_id: 0x74a1 +kern :err : 2026-01-07T10:03:01,789012-06:00 {1}[Hardware Error]: class_code: 060400 +kern :err : 2026-01-07T10:03:01,890123-06:00 {1}[Hardware Error]: bridge: secondary_status: 0x2000, control: 0x0003 +kern :err : 2026-01-07T10:04:00,123456-06:00 amdgpu 0000:02:00.0: amdgpu: Error Address(PA):0x12345000 Row:0x1000 Col:0x0 Bank:0x5 Channel:0x10 +kern :err : 2026-01-07T10:05:00,123456-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: Accelerator Check Architecture events logged +kern :err : 2026-01-07T10:05:00,234567-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].CONTROL=0x00000000000003ff +kern :err : 2026-01-07T10:05:00,345678-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].STATUS=0x98000800003e0000 +kern :err : 2026-01-07T10:05:00,456789-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].ADDR=0x0000000000000000 +kern :err : 2026-01-07T10:05:00,567890-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].MISC=0xd008000000000000 +kern :err : 2026-01-07T10:05:00,678901-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].CONFIG=0x00000021000001fb +kern :err : 2026-01-07T10:05:00,789012-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].IPID=0x0001100138430401 +kern :err : 2026-01-07T10:05:00,890123-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].SYND=0x0000000000000000 +kern :err : 2026-01-07T10:05:01,123456-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].DESTAT=0x0000000000000000 +kern :err : 2026-01-07T10:05:01,234567-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].DEADDR=0x0000000000000000 +kern :err : 2026-01-07T10:05:01,345678-06:00 amdgpu 0000:03:00.0: {1}[Hardware Error]: ACA[01/01].CONTROL_MASK=0x0000000000000000 diff --git a/test/functional/fixtures/nic_plugin_config_full_analyzer_args.json b/test/functional/fixtures/nic_plugin_config_full_analyzer_args.json index f1cc1668..5d0ab0a1 100644 --- a/test/functional/fixtures/nic_plugin_config_full_analyzer_args.json +++ b/test/functional/fixtures/nic_plugin_config_full_analyzer_args.json @@ -1,37 +1,37 @@ -{ - "name": "NicPlugin config with all analyzer_args", - "desc": "NicPlugin check.", - "global_args": {}, - "plugins": { - "NicPlugin": { - "collection_args": {}, - "analysis_args": { - "expected_values": { - "niccli_list": {"require_success": true}, - "niccli_list_devices": {"require_success": true} - }, - "performance_profile_expected": "RoCE", - "support_rdma_disabled_values": ["0", "false", "disabled", "no", "off"], - "pcie_relaxed_ordering_expected": "enabled", - "expected_qos_prio_map": { - "0": 0, - "1": 1, - "2": 0, - "3": 1, - "4": 0, - "5": 1, - "6": 0, - "7": 1 - }, - "expected_qos_pfc_enabled": 255, - "expected_qos_tsa_map": { - "0": "ets", - "1": "ets" - }, - "expected_qos_tc_bandwidth": [50, 50], - "require_qos_consistent_across_adapters": true - } - } - }, - "result_collators": {} -} +{ + "name": "NicPlugin config with all analyzer_args", + "desc": "NicPlugin check.", + "global_args": {}, + "plugins": { + "NicPlugin": { + "collection_args": {}, + "analysis_args": { + "expected_values": { + "niccli_list": {"require_success": true}, + "niccli_list_devices": {"require_success": true} + }, + "performance_profile_expected": "RoCE", + "support_rdma_disabled_values": ["0", "false", "disabled", "no", "off"], + "pcie_relaxed_ordering_expected": "enabled", + "expected_qos_prio_map": { + "0": 0, + "1": 1, + "2": 0, + "3": 1, + "4": 0, + "5": 1, + "6": 0, + "7": 1 + }, + "expected_qos_pfc_enabled": 255, + "expected_qos_tsa_map": { + "0": "ets", + "1": "ets" + }, + "expected_qos_tc_bandwidth": [50, 50], + "require_qos_consistent_across_adapters": true + } + } + }, + "result_collators": {} +} diff --git a/test/functional/fixtures/plugin_config.json b/test/functional/fixtures/plugin_config.json index 000b7b2a..12f8b5df 100644 --- a/test/functional/fixtures/plugin_config.json +++ b/test/functional/fixtures/plugin_config.json @@ -1 +1 @@ -{"name":"plugin_config","desc":"Full config: all collector_args + analyzer_args, values that pass on Ubuntu","global_args":{},"plugins":{"PackagePlugin":{"collection_args":{},"analysis_args":{"exp_package_ver":{"libc|python|apt":null},"regex_match":true,"rocm_regex":null,"enable_rocm_regex":false}},"SysSettingsPlugin":{"collection_args":{"paths":["/sys/kernel/mm/transparent_hugepage/enabled","/sys/kernel/mm/transparent_hugepage/defrag"],"directory_paths":["/sys/class/net"]},"analysis_args":{"checks":[{"path":"/sys/kernel/mm/transparent_hugepage/enabled","expected":["always","madvise","never"],"name":"thp_enabled","pattern":null},{"path":"/sys/kernel/mm/transparent_hugepage/defrag","expected":["always","madvise","never","defer"],"name":"thp_defrag","pattern":null},{"path":"/sys/class/net","expected":[],"name":"net_interfaces","pattern":"^(lo|eth|enp|wl|br-|docker|ens)"}]}}},"result_collators":{}} +{"name":"plugin_config","desc":"Full config: all collector_args + analyzer_args, values that pass on Ubuntu","global_args":{},"plugins":{"PackagePlugin":{"collection_args":{},"analysis_args":{"exp_package_ver":{"libc|python|apt":null},"regex_match":true,"rocm_regex":null,"enable_rocm_regex":false}},"SysSettingsPlugin":{"collection_args":{"paths":["/sys/kernel/mm/transparent_hugepage/enabled","/sys/kernel/mm/transparent_hugepage/defrag"],"directory_paths":["/sys/class/net"]},"analysis_args":{"checks":[{"path":"/sys/kernel/mm/transparent_hugepage/enabled","expected":["always","madvise","never"],"name":"thp_enabled","pattern":null},{"path":"/sys/kernel/mm/transparent_hugepage/defrag","expected":["always","madvise","never","defer"],"name":"thp_defrag","pattern":null},{"path":"/sys/class/net","expected":[],"name":"net_interfaces","pattern":"^(lo|eth|enp|wl|br-|docker|ens)"}]}}},"result_collators":{}} diff --git a/test/functional/fixtures/process_plugin_config.json b/test/functional/fixtures/process_plugin_config.json index 752da786..ed7009b3 100644 --- a/test/functional/fixtures/process_plugin_config.json +++ b/test/functional/fixtures/process_plugin_config.json @@ -3,7 +3,6 @@ "plugins": { "ProcessPlugin": { "analysis_args": { - "max_kfd_processes": 0, "max_cpu_usage": 20.0 } } diff --git a/test/functional/fixtures/redfish_connection_config.json b/test/functional/fixtures/redfish_connection_config.json index 6a0475b9..c3c98fe8 100644 --- a/test/functional/fixtures/redfish_connection_config.json +++ b/test/functional/fixtures/redfish_connection_config.json @@ -1,9 +1,9 @@ -{ - "RedfishConnectionManager": { - "host": "https://bmc.example.com", - "username": "ADMIN", - "password": "placeholder", - "verify_ssl": false, - "timeout_seconds": 30 - } -} +{ + "RedfishConnectionManager": { + "host": "https://bmc.example.com", + "username": "ADMIN", + "password": "placeholder", + "verify_ssl": false, + "timeout_seconds": 30 + } +} diff --git a/test/functional/fixtures/redfish_endpoint_plugin_config.json b/test/functional/fixtures/redfish_endpoint_plugin_config.json index f79cf256..e777a7e4 100644 --- a/test/functional/fixtures/redfish_endpoint_plugin_config.json +++ b/test/functional/fixtures/redfish_endpoint_plugin_config.json @@ -1,12 +1,12 @@ -{ - "plugins": { - "RedfishEndpointPlugin": { - "collection_args": { - "uris": ["/redfish/v1", "/redfish/v1/Systems"] - }, - "analysis_args": { - "checks": {} - } - } - } -} +{ + "plugins": { + "RedfishEndpointPlugin": { + "collection_args": { + "uris": ["/redfish/v1", "/redfish/v1/Systems"] + }, + "analysis_args": { + "checks": {} + } + } + } +} diff --git a/test/functional/fixtures/redfish_endpoint_plugin_config_full_args.json b/test/functional/fixtures/redfish_endpoint_plugin_config_full_args.json index 099057c5..c0cf2e5a 100644 --- a/test/functional/fixtures/redfish_endpoint_plugin_config_full_args.json +++ b/test/functional/fixtures/redfish_endpoint_plugin_config_full_args.json @@ -1,26 +1,26 @@ -{ - "plugins": { - "RedfishEndpointPlugin": { - "collection_args": { - "uris": ["/redfish/v1", "/redfish/v1/Systems"], - "discover_tree": false, - "tree_max_depth": 3, - "tree_max_endpoints": 100, - "max_workers": 2 - }, - "analysis_args": { - "checks": { - "/redfish/v1/Systems/1": { - "Status/Health": { "anyOf": ["OK", "Warning"] }, - "PowerState": { "eq": "On" }, - "PowerConsumedWatts": { "min": 0, "max": 500 } - }, - "*": { - "Status/State": { "eq": "Enabled" }, - "Status/Health": { "anyOf": ["OK", "Warning"] } - } - } - } - } - } -} +{ + "plugins": { + "RedfishEndpointPlugin": { + "collection_args": { + "uris": ["/redfish/v1", "/redfish/v1/Systems"], + "discover_tree": false, + "tree_max_depth": 3, + "tree_max_endpoints": 100, + "max_workers": 2 + }, + "analysis_args": { + "checks": { + "/redfish/v1/Systems/1": { + "Status/Health": { "anyOf": ["OK", "Warning"] }, + "PowerState": { "eq": "On" }, + "PowerConsumedWatts": { "min": 0, "max": 500 } + }, + "*": { + "Status/State": { "eq": "Enabled" }, + "Status/Health": { "anyOf": ["OK", "Warning"] } + } + } + } + } + } +} diff --git a/test/functional/fixtures/regex_search_multi_logs/app_error.log b/test/functional/fixtures/regex_search_multi_logs/app_error.log index 391fb32c..25ba7a23 100644 --- a/test/functional/fixtures/regex_search_multi_logs/app_error.log +++ b/test/functional/fixtures/regex_search_multi_logs/app_error.log @@ -1,3 +1,3 @@ -startup complete -ERROR: dependency timeout connecting to backend -shutdown clean +startup complete +ERROR: dependency timeout connecting to backend +shutdown clean diff --git a/test/functional/fixtures/regex_search_multi_logs/no_errors.log b/test/functional/fixtures/regex_search_multi_logs/no_errors.log index 854d9322..e433bfc4 100644 --- a/test/functional/fixtures/regex_search_multi_logs/no_errors.log +++ b/test/functional/fixtures/regex_search_multi_logs/no_errors.log @@ -1,2 +1,2 @@ -2026-04-09 service healthy -all checks passed +2026-04-09 service healthy +all checks passed diff --git a/test/functional/fixtures/regex_search_multi_logs/storage_warn.log b/test/functional/fixtures/regex_search_multi_logs/storage_warn.log index bc11a75a..432bf314 100644 --- a/test/functional/fixtures/regex_search_multi_logs/storage_warn.log +++ b/test/functional/fixtures/regex_search_multi_logs/storage_warn.log @@ -1,3 +1,3 @@ -mount ok -WARNING: disk full on /data within 5% -idle +mount ok +WARNING: disk full on /data within 5% +idle diff --git a/test/functional/fixtures/regex_search_sample.log b/test/functional/fixtures/regex_search_sample.log index 7564228b..0490f4ec 100644 --- a/test/functional/fixtures/regex_search_sample.log +++ b/test/functional/fixtures/regex_search_sample.log @@ -1,4 +1,4 @@ -2026-04-09T10:00:00Z service started OK -2026-04-09T10:01:00Z ERROR: connection reset by peer -2026-04-09T10:02:00Z routine check passed -2026-04-09T10:03:00Z WARNING: disk full on /var +2026-04-09T10:00:00Z service started OK +2026-04-09T10:01:00Z ERROR: connection reset by peer +2026-04-09T10:02:00Z routine check passed +2026-04-09T10:03:00Z WARNING: disk full on /var diff --git a/test/functional/fixtures/sys_settings_plugin_config.json b/test/functional/fixtures/sys_settings_plugin_config.json index 2a2013c7..d912520b 100644 --- a/test/functional/fixtures/sys_settings_plugin_config.json +++ b/test/functional/fixtures/sys_settings_plugin_config.json @@ -1,36 +1,36 @@ -{ - "name": "SysSettingsPlugin config", - "desc": "Config for testing SysSettingsPlugin (sysfs settings)", - "global_args": {}, - "plugins": { - "SysSettingsPlugin": { - "collection_args": { - "paths": [ - "/sys/kernel/mm/transparent_hugepage/enabled", - "/sys/kernel/mm/transparent_hugepage/defrag", - "/sys/kernel/mm/transparent_hugepage/shmem_enabled" - ] - }, - "analysis_args": { - "checks": [ - { - "path": "/sys/kernel/mm/transparent_hugepage/enabled", - "expected": ["always", "madvise", "never"], - "name": "thp_enabled" - }, - { - "path": "/sys/kernel/mm/transparent_hugepage/defrag", - "expected": ["always", "madvise", "never", "defer"], - "name": "thp_defrag" - }, - { - "path": "/sys/kernel/mm/transparent_hugepage/shmem_enabled", - "expected": [], - "name": "thp_shmem" - } - ] - } - } - }, - "result_collators": {} -} +{ + "name": "SysSettingsPlugin config", + "desc": "Config for testing SysSettingsPlugin (sysfs settings)", + "global_args": {}, + "plugins": { + "SysSettingsPlugin": { + "collection_args": { + "paths": [ + "/sys/kernel/mm/transparent_hugepage/enabled", + "/sys/kernel/mm/transparent_hugepage/defrag", + "/sys/kernel/mm/transparent_hugepage/shmem_enabled" + ] + }, + "analysis_args": { + "checks": [ + { + "path": "/sys/kernel/mm/transparent_hugepage/enabled", + "expected": ["always", "madvise", "never"], + "name": "thp_enabled" + }, + { + "path": "/sys/kernel/mm/transparent_hugepage/defrag", + "expected": ["always", "madvise", "never", "defer"], + "name": "thp_defrag" + }, + { + "path": "/sys/kernel/mm/transparent_hugepage/shmem_enabled", + "expected": [], + "name": "thp_shmem" + } + ] + } + } + }, + "result_collators": {} +} diff --git a/test/functional/test_cli_no_console_log.py b/test/functional/test_cli_no_console_log.py index ed9fd8fa..9a5ee964 100644 --- a/test/functional/test_cli_no_console_log.py +++ b/test/functional/test_cli_no_console_log.py @@ -1,50 +1,50 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2025 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -import subprocess -import sys - - -def test_no_console_log_with_log_path_none_still_parses(): - """--log-path None + --no-console-log defaults to ./scraper_logs_* (no argparse error).""" - result = subprocess.run( - [ - sys.executable, - "-m", - "nodescraper.cli.cli", - "--log-path", - "None", - "--no-console-log", - "run-plugins", - "OsPlugin", - "-h", - ], - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - combined = (result.stdout or "") + (result.stderr or "") - assert "no-console-log requires" not in combined.lower() +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import subprocess +import sys + + +def test_no_console_log_with_log_path_none_still_parses(): + """--log-path None + --no-console-log defaults to ./scraper_logs_* (no argparse error).""" + result = subprocess.run( + [ + sys.executable, + "-m", + "nodescraper.cli.cli", + "--log-path", + "None", + "--no-console-log", + "run-plugins", + "OsPlugin", + "-h", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + combined = (result.stdout or "") + (result.stderr or "") + assert "no-console-log requires" not in combined.lower() diff --git a/test/functional/test_nic_plugin.py b/test/functional/test_nic_plugin.py index b862845e..c9fa0e64 100644 --- a/test/functional/test_nic_plugin.py +++ b/test/functional/test_nic_plugin.py @@ -1,135 +1,135 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -from pathlib import Path - -import pytest - - -@pytest.fixture -def fixtures_dir(): - """Return path to fixtures directory.""" - return Path(__file__).parent / "fixtures" - - -@pytest.fixture -def nic_plugin_config_full_analyzer_args(fixtures_dir): - """Return path to NicPlugin config with all analyzer_args populated.""" - return fixtures_dir / "nic_plugin_config_full_analyzer_args.json" - - -@pytest.fixture -def nic_plugin_config_minimal(fixtures_dir): - """Return path to minimal NicPlugin config (niccli_plugin_config.json).""" - return fixtures_dir / "niccli_plugin_config.json" - - -def test_nic_plugin_with_full_analyzer_args_config( - run_cli_command, nic_plugin_config_full_analyzer_args, tmp_path -): - """Test NicPlugin using config with all analyzer_args (performance_profile, getqos, etc.).""" - assert ( - nic_plugin_config_full_analyzer_args.exists() - ), f"Config file not found: {nic_plugin_config_full_analyzer_args}" - - log_path = str(tmp_path / "logs_nic_full_args") - result = run_cli_command( - [ - "--log-path", - log_path, - f"--plugin-configs={nic_plugin_config_full_analyzer_args}", - ], - check=False, - ) - - assert result.returncode in [ - 0, - 1, - 2, - ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" - output = result.stdout + result.stderr - assert len(output) > 0 - assert "NicPlugin" in output or "nic" in output.lower() - - -def test_nic_plugin_with_minimal_config(run_cli_command, nic_plugin_config_minimal, tmp_path): - """Test NicPlugin using minimal config (default collection_args, no analysis_args).""" - assert nic_plugin_config_minimal.exists(), f"Config file not found: {nic_plugin_config_minimal}" - - log_path = str(tmp_path / "logs_nic_minimal") - result = run_cli_command( - ["--log-path", log_path, f"--plugin-configs={nic_plugin_config_minimal}"], - check=False, - ) - - assert result.returncode in [ - 0, - 1, - 2, - ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" - output = result.stdout + result.stderr - assert len(output) > 0 - assert "NicPlugin" in output or "nic" in output.lower() - - -def test_nic_plugin_with_run_plugins_subcommand(run_cli_command, tmp_path): - """Test NicPlugin via run-plugins subcommand (no config).""" - log_path = str(tmp_path / "logs_nic_subcommand") - result = run_cli_command(["--log-path", log_path, "run-plugins", "NicPlugin"], check=False) - - assert result.returncode in [ - 0, - 1, - 2, - ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" - output = result.stdout + result.stderr - assert len(output) > 0 - assert "NicPlugin" in output or "nic" in output.lower() - - -def test_nic_plugin_full_config_validates_analysis_args( - run_cli_command, nic_plugin_config_full_analyzer_args, tmp_path -): - """Config with all analyzer_args loads and runs without validation error.""" - assert nic_plugin_config_full_analyzer_args.exists() - - log_path = str(tmp_path / "logs_nic_validate") - result = run_cli_command( - [ - "--log-path", - log_path, - f"--plugin-configs={nic_plugin_config_full_analyzer_args}", - ], - check=False, - ) - - assert result.returncode in [ - 0, - 1, - 2, - ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" - output = result.stdout + result.stderr - assert "NicPlugin" in output +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +from pathlib import Path + +import pytest + + +@pytest.fixture +def fixtures_dir(): + """Return path to fixtures directory.""" + return Path(__file__).parent / "fixtures" + + +@pytest.fixture +def nic_plugin_config_full_analyzer_args(fixtures_dir): + """Return path to NicPlugin config with all analyzer_args populated.""" + return fixtures_dir / "nic_plugin_config_full_analyzer_args.json" + + +@pytest.fixture +def nic_plugin_config_minimal(fixtures_dir): + """Return path to minimal NicPlugin config (niccli_plugin_config.json).""" + return fixtures_dir / "niccli_plugin_config.json" + + +def test_nic_plugin_with_full_analyzer_args_config( + run_cli_command, nic_plugin_config_full_analyzer_args, tmp_path +): + """Test NicPlugin using config with all analyzer_args (performance_profile, getqos, etc.).""" + assert ( + nic_plugin_config_full_analyzer_args.exists() + ), f"Config file not found: {nic_plugin_config_full_analyzer_args}" + + log_path = str(tmp_path / "logs_nic_full_args") + result = run_cli_command( + [ + "--log-path", + log_path, + f"--plugin-configs={nic_plugin_config_full_analyzer_args}", + ], + check=False, + ) + + assert result.returncode in [ + 0, + 1, + 2, + ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" + output = result.stdout + result.stderr + assert len(output) > 0 + assert "NicPlugin" in output or "nic" in output.lower() + + +def test_nic_plugin_with_minimal_config(run_cli_command, nic_plugin_config_minimal, tmp_path): + """Test NicPlugin using minimal config (default collection_args, no analysis_args).""" + assert nic_plugin_config_minimal.exists(), f"Config file not found: {nic_plugin_config_minimal}" + + log_path = str(tmp_path / "logs_nic_minimal") + result = run_cli_command( + ["--log-path", log_path, f"--plugin-configs={nic_plugin_config_minimal}"], + check=False, + ) + + assert result.returncode in [ + 0, + 1, + 2, + ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" + output = result.stdout + result.stderr + assert len(output) > 0 + assert "NicPlugin" in output or "nic" in output.lower() + + +def test_nic_plugin_with_run_plugins_subcommand(run_cli_command, tmp_path): + """Test NicPlugin via run-plugins subcommand (no config).""" + log_path = str(tmp_path / "logs_nic_subcommand") + result = run_cli_command(["--log-path", log_path, "run-plugins", "NicPlugin"], check=False) + + assert result.returncode in [ + 0, + 1, + 2, + ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" + output = result.stdout + result.stderr + assert len(output) > 0 + assert "NicPlugin" in output or "nic" in output.lower() + + +def test_nic_plugin_full_config_validates_analysis_args( + run_cli_command, nic_plugin_config_full_analyzer_args, tmp_path +): + """Config with all analyzer_args loads and runs without validation error.""" + assert nic_plugin_config_full_analyzer_args.exists() + + log_path = str(tmp_path / "logs_nic_validate") + result = run_cli_command( + [ + "--log-path", + log_path, + f"--plugin-configs={nic_plugin_config_full_analyzer_args}", + ], + check=False, + ) + + assert result.returncode in [ + 0, + 1, + 2, + ], f"Unexpected return code: {result.returncode}. stdout: {result.stdout[:500]!r}" + output = result.stdout + result.stderr + assert "NicPlugin" in output diff --git a/test/functional/test_os_plugin.py b/test/functional/test_os_plugin.py new file mode 100644 index 00000000..12d6103a --- /dev/null +++ b/test/functional/test_os_plugin.py @@ -0,0 +1,261 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Functional tests for OsPlugin with --plugin-configs.""" + +from pathlib import Path +from typing import Any, Union + +import pytest + + +@pytest.fixture +def fixtures_dir(): + """Return path to fixtures directory.""" + return Path(__file__).parent / "fixtures" + + +@pytest.fixture +def os_config_file(fixtures_dir): + """Return path to OsPlugin config file.""" + return fixtures_dir / "os_plugin_config.json" + + +@pytest.fixture +def plugin_executor(plugin_reg: Union[Any, None] = None): + """Fixture that creates a PluginExecutor instance for programmatic testing.""" + from nodescraper.models.pluginconfig import PluginConfig + from nodescraper.pluginexecutor import PluginExecutor + from nodescraper.pluginregistry import PluginRegistry + + if plugin_reg is None: + plugin_reg = PluginRegistry() + + def _create_executor( + plugin_configs: list[PluginConfig], + connections=None, + log_path=None, + system_info=None, + ): + """Create a PluginExecutor with the given configuration. + + Args: + plugin_configs: List of PluginConfig objects + connections: Optional dict of connection configs + log_path: Optional path for logs + + Returns: + PluginExecutor instance + """ + return PluginExecutor( + plugin_configs=plugin_configs, + connections=connections, + system_info=system_info, + log_path=log_path, + plugin_registry=plugin_reg, + ) + + return _create_executor + + +def test_os_plugin_with_basic_config(run_cli_command, os_config_file, tmp_path): + """Test OsPlugin using basic config file.""" + assert os_config_file.exists(), f"Config file not found: {os_config_file}" + + log_path = str(tmp_path / "logs_os_basic") + result = run_cli_command( + ["--log-path", log_path, f"--plugin-configs={os_config_file}"], check=False + ) + + assert ( + result.returncode == 1 + ), f"Expected success (1), got {result.returncode}. Output: {result.stdout + result.stderr}" + + output = result.stdout + result.stderr + assert len(output) > 0 + assert "osplugin" + + +def test_os_plugin_with_os_family_discovery(run_cli_command, tmp_path): + """Test OsPlugin with OS family auto-discovery (not explicitly set).""" + log_path = str(tmp_path / "logs_os_family_discovery") + # Don't specify OS family - let it be discovered + result = run_cli_command( + [ + "--log-path", + log_path, + "run-plugins", + "OsPlugin", + ], + check=False, + ) + + # Should succeed - OS family discovery is required and should always work locally + assert result.returncode in [0, 1, 2, 3, 4] + output = result.stdout + result.stderr + assert len(output) > 0 + # Verify that collection happened (OS family was discovered) + assert "osplugin" in output.lower() or "os" in output.lower() + + +def test_os_plugin_with_executor_programmatic(plugin_executor, tmp_path): + """Test OsPlugin using PluginExecutor The executor no CLI then we will inspect the result.""" + import string + + from nodescraper.models.pluginconfig import PluginConfig + + # Exp os's are every letter and number combo + exp_os = [] + for letter in string.ascii_letters + string.digits: + exp_os.append(letter) + + config = PluginConfig( + global_args={}, + plugins={ + "OsPlugin": { + "analysis_args": {"exp_os": exp_os, "exact_match": False}, + "analysis": True, + "collection": True, + } + }, + name="Test Run", + desc="Test description", + ) + + # Create executor + executor = plugin_executor( + plugin_configs=[config], + log_path=str(tmp_path / "logs_executor"), + ) + + # Run plugins (correct method is run_queue) + results = executor.run_queue() + + # Verify execution succeeded + assert results is not None + assert len(results) > 0 + + # Check that OsPlugin ran + os_results = [r for r in results if r.source == "OsPlugin"] + assert len(os_results) > 0, "OsPlugin did not run" + + # Check the OsPlugin result + from nodescraper.enums import ExecutionStatus + + os_result = os_results[0] + assert os_result.result_data is not None, "OsPlugin result_data is None" + assert ( + os_result.status == ExecutionStatus.OK + ), f"OsPlugin status: {os_result.status}, message: {os_result.message}" + + # Verify collection and analysis results + assert os_result.result_data.collection_result is not None, "Collection result is None" + assert ( + os_result.result_data.collection_result.status == ExecutionStatus.OK + ), f"Collection status: {os_result.result_data.collection_result.status}" + assert os_result.result_data.analysis_result is not None, "Analysis result is None" + assert ( + os_result.result_data.analysis_result.status == ExecutionStatus.OK + ), f"Analysis status: {os_result.result_data.analysis_result.status}" + + # Verify data was collected (OS family was auto-discovered) + assert hasattr(executor, "system_info") + # OS family should be discovered and set + assert executor.system_info.os_family is not None + + +def _run_os_plugin(plugin_executor, tmp_path, system_info=None, connections=None): + from nodescraper.models.pluginconfig import PluginConfig + + config = PluginConfig( + global_args={}, + plugins={"OsPlugin": {"analysis": False, "collection": True}}, + name="Connection result run", + desc="Check connection result is saved with the plugin result", + ) + executor = plugin_executor( + plugin_configs=[config], + log_path=str(tmp_path / "logs"), + system_info=system_info, + connections=connections, + ) + results = executor.run_queue() + os_results = [r for r in results if r.source == "OsPlugin"] + assert len(os_results) == 1, "OsPlugin did not produce a result" + return executor, os_results[0] + + +def test_connection_result_saved_on_successful_connection(plugin_executor, tmp_path): + """A successful connection is recorded as OK in result_data and survives disconnect.""" + from nodescraper.enums import ExecutionStatus + + executor, os_result = _run_os_plugin(plugin_executor, tmp_path) + + connection_result = os_result.result_data.connection_result + assert connection_result is not None, "connection_result was not saved" + assert connection_result.status == ExecutionStatus.OK + assert connection_result.task == "InBandConnectionManager" + + # run_queue disconnects every manager on exit; the result must not be reset to UNSET. + for manager in executor.connection_library.values(): + assert manager.connection is None + assert manager.result.status == ExecutionStatus.OK + + +def test_connection_result_saved_on_failed_connection(plugin_executor, tmp_path): + """A remote connection without credentials fails, reports an event and is saved.""" + from nodescraper.enums import ExecutionStatus, SystemLocation + from nodescraper.models.systeminfo import SystemInfo + + system_info = SystemInfo(name="unreachable_host", location=SystemLocation.REMOTE) + executor, os_result = _run_os_plugin(plugin_executor, tmp_path, system_info=system_info) + + connection_result = os_result.result_data.connection_result + assert connection_result is not None, "connection_result was not saved" + assert connection_result.status in ( + ExecutionStatus.ERROR, + ExecutionStatus.EXECUTION_FAILURE, + ) + assert len(connection_result.events) >= 1 + assert os_result.result_data.collection_result.status == ExecutionStatus.NOT_RAN + assert os_result.status != ExecutionStatus.OK + + for manager in executor.connection_library.values(): + assert manager.result.status in (ExecutionStatus.ERROR, ExecutionStatus.EXECUTION_FAILURE) + + +def test_connection_result_written_to_log_path(plugin_executor, tmp_path): + """The connection result is logged to disk by the connection result hook.""" + import json + + _run_os_plugin(plugin_executor, tmp_path) + + result_files = [ + p + for p in (tmp_path / "logs").rglob("result.json") + if json.loads(p.read_text()).get("task") == "InBandConnectionManager" + ] + assert result_files, "No connection result.json found under the log path" + assert json.loads(result_files[0].read_text())["status"] == "OK" diff --git a/test/functional/test_redfish_endpoint_plugin.py b/test/functional/test_redfish_endpoint_plugin.py index f2513b62..b9f439fd 100644 --- a/test/functional/test_redfish_endpoint_plugin.py +++ b/test/functional/test_redfish_endpoint_plugin.py @@ -1,390 +1,390 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from unittest.mock import MagicMock - -import pytest - -from nodescraper.connection.redfish import RedfishGetResult -from nodescraper.enums import EventCategory, ExecutionStatus -from nodescraper.plugins.ooband.redfish_endpoint import ( - RedfishEndpointCollector, - RedfishEndpointCollectorArgs, -) -from nodescraper.plugins.ooband.redfish_endpoint import endpoint_collector as ec - - -@pytest.fixture -def redfish_endpoint_collector(system_info, redfish_conn_mock): - return RedfishEndpointCollector( - system_info=system_info, - connection=redfish_conn_mock, - ) - - -def test_redfish_endpoint_collector_no_uris(redfish_endpoint_collector): - result, data = redfish_endpoint_collector.collect_data() - assert result.status == ExecutionStatus.NOT_RAN - assert ( - result.message - == "No collection mode configured: set collection_args.discover_tree to true or provide collection_args.uris" - ) - assert data is None - - -def test_redfish_endpoint_collector_no_uris_with_args(redfish_endpoint_collector): - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=[]) - ) - assert result.status == ExecutionStatus.NOT_RAN - assert data is None - - -def test_redfish_endpoint_collector_one_uri_success(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"Name": "Root"}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) - ) - assert result.status == ExecutionStatus.OK - assert result.message == "Collected 1 Redfish endpoint(s)" - assert data is not None - assert data.responses["/redfish/v1"]["Name"] == "Root" - redfish_conn_mock.run_get.assert_called_once() - call_path = redfish_conn_mock.run_get.call_args[0][0] - assert call_path == "/redfish/v1" or call_path.strip("/") == "redfish/v1" - - -def test_redfish_endpoint_collector_uri_normalized_with_leading_slash( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1/Systems", - success=True, - data={"Members": []}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["redfish/v1/Systems"]) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert "/redfish/v1/Systems" in data.responses or "redfish/v1/Systems" in data.responses - - -def test_redfish_endpoint_collector_one_fail_no_success( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=False, - error="Connection refused", - status_code=None, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) - ) - assert result.status == ExecutionStatus.ERROR - assert result.message.startswith("No Redfish endpoints could be read") - assert data is None - assert len(result.events) >= 1 - assert any( - e.category == EventCategory.RUNTIME.value or "Redfish GET failed" in (e.description or "") - for e in result.events - ) - - -def test_redfish_endpoint_collector_mixed_success_fail( - redfish_endpoint_collector, redfish_conn_mock -): - def run_get_side_effect(path): - path_str = str(path) - if "Systems" in path_str: - return RedfishGetResult( - path=path_str if path_str.startswith("/") else "/" + path_str, - success=True, - data={"Id": "1"}, - status_code=200, - ) - return RedfishGetResult( - path=path_str if path_str.startswith("/") else "/" + path_str, - success=False, - error="Not Found", - status_code=404, - ) - - redfish_conn_mock.run_get.side_effect = run_get_side_effect - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["/redfish/v1/Systems", "/redfish/v1/Bad"]) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert len(data.responses) == 1 - keys = list(data.responses.keys()) - assert any("Systems" in k for k in keys) - assert list(data.responses.values())[0].get("Id") == "1" - - -def test_normalize_path_empty_or_invalid(): - api = "redfish/v1" - assert ec._normalize_path("", api) == "" - assert ec._normalize_path(None, api) == "" # type: ignore[arg-type] - assert ec._normalize_path(" ", api) == "" - - -def test_normalize_path_relative_path(): - api = "redfish/v1" - assert ec._normalize_path("/redfish/v1", api) == "/redfish/v1" - assert ec._normalize_path("/redfish/v1/", api) == "/redfish/v1" - assert ec._normalize_path("redfish/v1/Systems", api) == "/redfish/v1/Systems" - assert ec._normalize_path(" /redfish/v1/Chassis ", api) == "/redfish/v1/Chassis" - - -def test_normalize_path_full_url(): - api = "redfish/v1" - assert ec._normalize_path("https://host/redfish/v1/Systems", api) == "/redfish/v1/Systems" - assert ec._normalize_path("http://bmc/redfish/v1", api) == "/redfish/v1" - - -def test_normalize_path_outside_api_root(): - api = "redfish/v1" - assert ec._normalize_path("/other/root", api) == "" - assert ec._normalize_path("https://host/other/path", api) == "" - - -def test_extract_odata_ids_empty(): - assert ec._extract_odata_ids({}) == [] - assert ec._extract_odata_ids([]) == [] - assert ec._extract_odata_ids("x") == [] - - -def test_extract_odata_ids_single(): - assert ec._extract_odata_ids({"@odata.id": "/redfish/v1"}) == ["/redfish/v1"] - assert ec._extract_odata_ids({"@odata.id": "https://host/redfish/v1"}) == [ - "https://host/redfish/v1" - ] - - -def test_extract_odata_ids_members(): - body = { - "Members": [ - {"@odata.id": "/redfish/v1/Systems/1"}, - {"@odata.id": "/redfish/v1/Systems/2"}, - ] - } - assert set(ec._extract_odata_ids(body)) == {"/redfish/v1/Systems/1", "/redfish/v1/Systems/2"} - - -def test_extract_odata_ids_nested_and_members(): - body = { - "@odata.id": "/redfish/v1", - "Systems": {"@odata.id": "/redfish/v1/Systems", "Members": []}, - "Chassis": { - "Members": [{"@odata.id": "/redfish/v1/Chassis/1"}], - }, - } - ids = ec._extract_odata_ids(body) - assert "/redfish/v1" in ids - assert "/redfish/v1/Systems" in ids - assert "/redfish/v1/Chassis/1" in ids - - -def test_uris_from_args_none(): - assert ec._uris_from_args(None) == [] - - -def test_uris_from_args_empty(): - assert ec._uris_from_args(RedfishEndpointCollectorArgs(uris=[])) == [] - - -def test_uris_from_args_with_uris(): - args = RedfishEndpointCollectorArgs(uris=["/redfish/v1", "/redfish/v1/Systems"]) - assert ec._uris_from_args(args) == ["/redfish/v1", "/redfish/v1/Systems"] - - -def test_fetch_one_calls_run_get(): - conn = MagicMock() - conn.run_get.return_value = RedfishGetResult( - path="/redfish/v1", success=True, data={}, status_code=200 - ) - out = ec._fetch_one(conn, "/redfish/v1") - conn.run_get.assert_called_once_with("/redfish/v1") - assert out.success is True - assert out.path == "/redfish/v1" - - -def test_discover_tree_single_root(): - conn = MagicMock() - conn.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"@odata.id": "/redfish/v1", "Name": "Root"}, - status_code=200, - ) - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=2, max_endpoints=0 - ) - assert paths == ["/redfish/v1"] - assert list(responses.keys()) == ["/redfish/v1"] - assert responses["/redfish/v1"]["Name"] == "Root" - assert len(results) == 1 - conn.run_get.assert_called_once_with("/redfish/v1") - - -def test_discover_tree_follows_links(): - conn = MagicMock() - root_data = { - "@odata.id": "/redfish/v1", - "Systems": {"@odata.id": "/redfish/v1/Systems"}, - } - systems_data = { - "@odata.id": "/redfish/v1/Systems", - "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], - } - system1_data = {"@odata.id": "/redfish/v1/Systems/1", "Id": "1"} - - def run_get(path): - if path == "/redfish/v1": - return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) - if path == "/redfish/v1/Systems": - return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) - if path == "/redfish/v1/Systems/1": - return RedfishGetResult(path=path, success=True, data=system1_data, status_code=200) - return RedfishGetResult(path=path, success=False, error="Not Found", status_code=404) - - conn.run_get.side_effect = run_get - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=3, max_endpoints=0 - ) - assert "/redfish/v1" in paths - assert "/redfish/v1/Systems" in paths - assert "/redfish/v1/Systems/1" in paths - assert responses["/redfish/v1"]["@odata.id"] == "/redfish/v1" - assert responses["/redfish/v1/Systems"]["@odata.id"] == "/redfish/v1/Systems" - assert responses["/redfish/v1/Systems/1"]["Id"] == "1" - assert len(results) >= 3 - - -def test_discover_tree_respects_max_depth(): - conn = MagicMock() - root_data = {"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}} - systems_data = { - "@odata.id": "/redfish/v1/Systems", - "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], - } - - def run_get(path): - if path == "/redfish/v1": - return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) - if path == "/redfish/v1/Systems": - return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) - return RedfishGetResult(path=path, success=True, data={}, status_code=200) - - conn.run_get.side_effect = run_get - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=1, max_endpoints=0 - ) - assert "/redfish/v1" in paths - assert "/redfish/v1/Systems" not in paths - assert len(responses) == 1 - - -def test_discover_tree_respects_max_endpoints(): - conn = MagicMock() - conn.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}}, - status_code=200, - ) - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=5, max_endpoints=1 - ) - assert len(paths) == 1 - assert len(responses) == 1 - conn.run_get.assert_called_once() - - -def test_collect_data_discover_tree_success(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.api_root = "redfish/v1" - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"@odata.id": "/redfish/v1", "Name": "Root"}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(discover_tree=True) - ) - assert result.status == ExecutionStatus.OK - assert result.message == "Collected 1 Redfish endpoint(s) from tree" - assert data is not None - assert "/redfish/v1" in data.responses - assert data.responses["/redfish/v1"]["Name"] == "Root" - - -def test_collect_data_discover_tree_no_responses(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.api_root = "redfish/v1" - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", success=False, error="Connection refused", status_code=None - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(discover_tree=True) - ) - assert result.status == ExecutionStatus.ERROR - assert "No Redfish endpoints discovered from tree" in result.message - assert data is None - - -def test_collect_data_concurrent_two_uris(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.copy.return_value = redfish_conn_mock - call_count = 0 - - def run_get(path): - nonlocal call_count - call_count += 1 - return RedfishGetResult( - path=path if path.startswith("/") else "/" + path, - success=True, - data={"path": path}, - status_code=200, - ) - - redfish_conn_mock.run_get.side_effect = run_get - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs( - uris=["/redfish/v1", "/redfish/v1/Systems"], max_workers=2 - ) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert len(data.responses) == 2 - assert "/redfish/v1" in data.responses or "redfish/v1" in data.responses - assert any("Systems" in k for k in data.responses) - assert redfish_conn_mock.copy.called +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock + +import pytest + +from nodescraper.connection.redfish import RedfishGetResult +from nodescraper.enums import EventCategory, ExecutionStatus +from nodescraper.plugins.ooband.redfish_endpoint import ( + RedfishEndpointCollector, + RedfishEndpointCollectorArgs, +) +from nodescraper.plugins.ooband.redfish_endpoint import endpoint_collector as ec + + +@pytest.fixture +def redfish_endpoint_collector(system_info, redfish_conn_mock): + return RedfishEndpointCollector( + system_info=system_info, + connection=redfish_conn_mock, + ) + + +def test_redfish_endpoint_collector_no_uris(redfish_endpoint_collector): + result, data = redfish_endpoint_collector.collect_data() + assert result.status == ExecutionStatus.NOT_RAN + assert ( + result.message + == "No collection mode configured: set collection_args.discover_tree to true or provide collection_args.uris" + ) + assert data is None + + +def test_redfish_endpoint_collector_no_uris_with_args(redfish_endpoint_collector): + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=[]) + ) + assert result.status == ExecutionStatus.NOT_RAN + assert data is None + + +def test_redfish_endpoint_collector_one_uri_success(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"Name": "Root"}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) + ) + assert result.status == ExecutionStatus.OK + assert result.message == "Collected 1 Redfish endpoint(s)" + assert data is not None + assert data.responses["/redfish/v1"]["Name"] == "Root" + redfish_conn_mock.run_get.assert_called_once() + call_path = redfish_conn_mock.run_get.call_args[0][0] + assert call_path == "/redfish/v1" or call_path.strip("/") == "redfish/v1" + + +def test_redfish_endpoint_collector_uri_normalized_with_leading_slash( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1/Systems", + success=True, + data={"Members": []}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["redfish/v1/Systems"]) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert "/redfish/v1/Systems" in data.responses or "redfish/v1/Systems" in data.responses + + +def test_redfish_endpoint_collector_one_fail_no_success( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=False, + error="Connection refused", + status_code=None, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) + ) + assert result.status == ExecutionStatus.ERROR + assert result.message.startswith("No Redfish endpoints could be read") + assert data is None + assert len(result.events) >= 1 + assert any( + e.category == EventCategory.RUNTIME.value or "Redfish GET failed" in (e.description or "") + for e in result.events + ) + + +def test_redfish_endpoint_collector_mixed_success_fail( + redfish_endpoint_collector, redfish_conn_mock +): + def run_get_side_effect(path): + path_str = str(path) + if "Systems" in path_str: + return RedfishGetResult( + path=path_str if path_str.startswith("/") else "/" + path_str, + success=True, + data={"Id": "1"}, + status_code=200, + ) + return RedfishGetResult( + path=path_str if path_str.startswith("/") else "/" + path_str, + success=False, + error="Not Found", + status_code=404, + ) + + redfish_conn_mock.run_get.side_effect = run_get_side_effect + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1/Systems", "/redfish/v1/Bad"]) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert len(data.responses) == 1 + keys = list(data.responses.keys()) + assert any("Systems" in k for k in keys) + assert list(data.responses.values())[0].get("Id") == "1" + + +def test_normalize_path_empty_or_invalid(): + api = "redfish/v1" + assert ec._normalize_path("", api) == "" + assert ec._normalize_path(None, api) == "" # type: ignore[arg-type] + assert ec._normalize_path(" ", api) == "" + + +def test_normalize_path_relative_path(): + api = "redfish/v1" + assert ec._normalize_path("/redfish/v1", api) == "/redfish/v1" + assert ec._normalize_path("/redfish/v1/", api) == "/redfish/v1" + assert ec._normalize_path("redfish/v1/Systems", api) == "/redfish/v1/Systems" + assert ec._normalize_path(" /redfish/v1/Chassis ", api) == "/redfish/v1/Chassis" + + +def test_normalize_path_full_url(): + api = "redfish/v1" + assert ec._normalize_path("https://host/redfish/v1/Systems", api) == "/redfish/v1/Systems" + assert ec._normalize_path("http://bmc/redfish/v1", api) == "/redfish/v1" + + +def test_normalize_path_outside_api_root(): + api = "redfish/v1" + assert ec._normalize_path("/other/root", api) == "" + assert ec._normalize_path("https://host/other/path", api) == "" + + +def test_extract_odata_ids_empty(): + assert ec._extract_odata_ids({}) == [] + assert ec._extract_odata_ids([]) == [] + assert ec._extract_odata_ids("x") == [] + + +def test_extract_odata_ids_single(): + assert ec._extract_odata_ids({"@odata.id": "/redfish/v1"}) == ["/redfish/v1"] + assert ec._extract_odata_ids({"@odata.id": "https://host/redfish/v1"}) == [ + "https://host/redfish/v1" + ] + + +def test_extract_odata_ids_members(): + body = { + "Members": [ + {"@odata.id": "/redfish/v1/Systems/1"}, + {"@odata.id": "/redfish/v1/Systems/2"}, + ] + } + assert set(ec._extract_odata_ids(body)) == {"/redfish/v1/Systems/1", "/redfish/v1/Systems/2"} + + +def test_extract_odata_ids_nested_and_members(): + body = { + "@odata.id": "/redfish/v1", + "Systems": {"@odata.id": "/redfish/v1/Systems", "Members": []}, + "Chassis": { + "Members": [{"@odata.id": "/redfish/v1/Chassis/1"}], + }, + } + ids = ec._extract_odata_ids(body) + assert "/redfish/v1" in ids + assert "/redfish/v1/Systems" in ids + assert "/redfish/v1/Chassis/1" in ids + + +def test_uris_from_args_none(): + assert ec._uris_from_args(None) == [] + + +def test_uris_from_args_empty(): + assert ec._uris_from_args(RedfishEndpointCollectorArgs(uris=[])) == [] + + +def test_uris_from_args_with_uris(): + args = RedfishEndpointCollectorArgs(uris=["/redfish/v1", "/redfish/v1/Systems"]) + assert ec._uris_from_args(args) == ["/redfish/v1", "/redfish/v1/Systems"] + + +def test_fetch_one_calls_run_get(): + conn = MagicMock() + conn.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={}, status_code=200 + ) + out = ec._fetch_one(conn, "/redfish/v1") + conn.run_get.assert_called_once_with("/redfish/v1") + assert out.success is True + assert out.path == "/redfish/v1" + + +def test_discover_tree_single_root(): + conn = MagicMock() + conn.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"@odata.id": "/redfish/v1", "Name": "Root"}, + status_code=200, + ) + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=2, max_endpoints=0 + ) + assert paths == ["/redfish/v1"] + assert list(responses.keys()) == ["/redfish/v1"] + assert responses["/redfish/v1"]["Name"] == "Root" + assert len(results) == 1 + conn.run_get.assert_called_once_with("/redfish/v1") + + +def test_discover_tree_follows_links(): + conn = MagicMock() + root_data = { + "@odata.id": "/redfish/v1", + "Systems": {"@odata.id": "/redfish/v1/Systems"}, + } + systems_data = { + "@odata.id": "/redfish/v1/Systems", + "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], + } + system1_data = {"@odata.id": "/redfish/v1/Systems/1", "Id": "1"} + + def run_get(path): + if path == "/redfish/v1": + return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) + if path == "/redfish/v1/Systems": + return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) + if path == "/redfish/v1/Systems/1": + return RedfishGetResult(path=path, success=True, data=system1_data, status_code=200) + return RedfishGetResult(path=path, success=False, error="Not Found", status_code=404) + + conn.run_get.side_effect = run_get + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=3, max_endpoints=0 + ) + assert "/redfish/v1" in paths + assert "/redfish/v1/Systems" in paths + assert "/redfish/v1/Systems/1" in paths + assert responses["/redfish/v1"]["@odata.id"] == "/redfish/v1" + assert responses["/redfish/v1/Systems"]["@odata.id"] == "/redfish/v1/Systems" + assert responses["/redfish/v1/Systems/1"]["Id"] == "1" + assert len(results) >= 3 + + +def test_discover_tree_respects_max_depth(): + conn = MagicMock() + root_data = {"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}} + systems_data = { + "@odata.id": "/redfish/v1/Systems", + "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], + } + + def run_get(path): + if path == "/redfish/v1": + return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) + if path == "/redfish/v1/Systems": + return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) + return RedfishGetResult(path=path, success=True, data={}, status_code=200) + + conn.run_get.side_effect = run_get + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=1, max_endpoints=0 + ) + assert "/redfish/v1" in paths + assert "/redfish/v1/Systems" not in paths + assert len(responses) == 1 + + +def test_discover_tree_respects_max_endpoints(): + conn = MagicMock() + conn.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}}, + status_code=200, + ) + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=5, max_endpoints=1 + ) + assert len(paths) == 1 + assert len(responses) == 1 + conn.run_get.assert_called_once() + + +def test_collect_data_discover_tree_success(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"@odata.id": "/redfish/v1", "Name": "Root"}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(discover_tree=True) + ) + assert result.status == ExecutionStatus.OK + assert result.message == "Collected 1 Redfish endpoint(s) from tree" + assert data is not None + assert "/redfish/v1" in data.responses + assert data.responses["/redfish/v1"]["Name"] == "Root" + + +def test_collect_data_discover_tree_no_responses(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=False, error="Connection refused", status_code=None + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(discover_tree=True) + ) + assert result.status == ExecutionStatus.ERROR + assert "No Redfish endpoints discovered from tree" in result.message + assert data is None + + +def test_collect_data_concurrent_two_uris(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.copy.return_value = redfish_conn_mock + call_count = 0 + + def run_get(path): + nonlocal call_count + call_count += 1 + return RedfishGetResult( + path=path if path.startswith("/") else "/" + path, + success=True, + data={"path": path}, + status_code=200, + ) + + redfish_conn_mock.run_get.side_effect = run_get + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1", "/redfish/v1/Systems"], max_workers=2 + ) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert len(data.responses) == 2 + assert "/redfish/v1" in data.responses or "redfish/v1" in data.responses + assert any("Systems" in k for k in data.responses) + assert redfish_conn_mock.copy.called diff --git a/test/functional/test_sys_settings_plugin.py b/test/functional/test_sys_settings_plugin.py index 03869184..18fdf59e 100644 --- a/test/functional/test_sys_settings_plugin.py +++ b/test/functional/test_sys_settings_plugin.py @@ -1,110 +1,110 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -"""Functional tests for SysSettingsPlugin with --plugin-configs.""" - -from pathlib import Path - -import pytest - - -@pytest.fixture -def fixtures_dir(): - """Return path to fixtures directory.""" - return Path(__file__).parent / "fixtures" - - -@pytest.fixture -def sys_settings_config_file(fixtures_dir): - """Return path to SysSettingsPlugin config file.""" - return fixtures_dir / "sys_settings_plugin_config.json" - - -@pytest.fixture -def plugin_config_json(fixtures_dir): - """Return path to fixture plugin_config.json.""" - return fixtures_dir / "plugin_config.json" - - -def test_sys_settings_plugin_with_config_file(run_cli_command, sys_settings_config_file, tmp_path): - """Test SysSettingsPlugin using config file with collection_args and analysis_args.""" - assert sys_settings_config_file.exists(), f"Config file not found: {sys_settings_config_file}" - - log_path = str(tmp_path / "logs_sys_settings") - result = run_cli_command( - ["--log-path", log_path, f"--plugin-configs={sys_settings_config_file}"], check=False - ) - - assert result.returncode in [0, 1, 2] - output = result.stdout + result.stderr - assert len(output) > 0 - assert "SysSettingsPlugin" in output or "syssettings" in output.lower() - - -def test_sys_settings_plugin_with_run_plugins_subcommand(run_cli_command, tmp_path): - """Test SysSettingsPlugin via run-plugins subcommand (no config; collector gets no paths).""" - log_path = str(tmp_path / "logs_sys_settings_subcommand") - result = run_cli_command( - ["--log-path", log_path, "run-plugins", "SysSettingsPlugin"], check=False - ) - - assert result.returncode in [0, 1, 2] - output = result.stdout + result.stderr - assert len(output) > 0 - # Without config, plugin runs with no paths -> NOT_RAN or similar - assert "SysSettings" in output or "sys" in output.lower() - - -def test_sys_settings_plugin_output_contains_plugin_result( - run_cli_command, sys_settings_config_file, tmp_path -): - """On Linux, plugin runs and table shows SysSettingsPlugin with a status.""" - assert sys_settings_config_file.exists() - - log_path = str(tmp_path / "logs_sys_settings_result") - result = run_cli_command( - ["--log-path", log_path, f"--plugin-configs={sys_settings_config_file}"], check=False - ) - - output = result.stdout + result.stderr - # Table or status line should mention the plugin - assert "SysSettingsPlugin" in output - - -def test_sys_settings_plugin_with_plugin_config_json(run_cli_command, plugin_config_json, tmp_path): - """Functional test: run node-scraper with project plugin_config.json (paths + directory_paths + checks).""" - assert plugin_config_json.exists(), f"Config not found: {plugin_config_json}" - - log_path = str(tmp_path / "logs_plugin_config") - result = run_cli_command( - ["--log-path", log_path, f"--plugin-configs={plugin_config_json}"], check=False - ) - - assert result.returncode in [0, 1, 2] - output = result.stdout + result.stderr - assert len(output) > 0 - assert "SysSettingsPlugin" in output - # Config exercises paths + directory_paths (/sys/class/net) + pattern check; collector/analyzer run - assert "Sysfs" in output or "sys" in output.lower() +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Functional tests for SysSettingsPlugin with --plugin-configs.""" + +from pathlib import Path + +import pytest + + +@pytest.fixture +def fixtures_dir(): + """Return path to fixtures directory.""" + return Path(__file__).parent / "fixtures" + + +@pytest.fixture +def sys_settings_config_file(fixtures_dir): + """Return path to SysSettingsPlugin config file.""" + return fixtures_dir / "sys_settings_plugin_config.json" + + +@pytest.fixture +def plugin_config_json(fixtures_dir): + """Return path to fixture plugin_config.json.""" + return fixtures_dir / "plugin_config.json" + + +def test_sys_settings_plugin_with_config_file(run_cli_command, sys_settings_config_file, tmp_path): + """Test SysSettingsPlugin using config file with collection_args and analysis_args.""" + assert sys_settings_config_file.exists(), f"Config file not found: {sys_settings_config_file}" + + log_path = str(tmp_path / "logs_sys_settings") + result = run_cli_command( + ["--log-path", log_path, f"--plugin-configs={sys_settings_config_file}"], check=False + ) + + assert result.returncode in [0, 1, 2] + output = result.stdout + result.stderr + assert len(output) > 0 + assert "SysSettingsPlugin" in output or "syssettings" in output.lower() + + +def test_sys_settings_plugin_with_run_plugins_subcommand(run_cli_command, tmp_path): + """Test SysSettingsPlugin via run-plugins subcommand (no config; collector gets no paths).""" + log_path = str(tmp_path / "logs_sys_settings_subcommand") + result = run_cli_command( + ["--log-path", log_path, "run-plugins", "SysSettingsPlugin"], check=False + ) + + assert result.returncode in [0, 1, 2] + output = result.stdout + result.stderr + assert len(output) > 0 + # Without config, plugin runs with no paths -> NOT_RAN or similar + assert "SysSettings" in output or "sys" in output.lower() + + +def test_sys_settings_plugin_output_contains_plugin_result( + run_cli_command, sys_settings_config_file, tmp_path +): + """On Linux, plugin runs and table shows SysSettingsPlugin with a status.""" + assert sys_settings_config_file.exists() + + log_path = str(tmp_path / "logs_sys_settings_result") + result = run_cli_command( + ["--log-path", log_path, f"--plugin-configs={sys_settings_config_file}"], check=False + ) + + output = result.stdout + result.stderr + # Table or status line should mention the plugin + assert "SysSettingsPlugin" in output + + +def test_sys_settings_plugin_with_plugin_config_json(run_cli_command, plugin_config_json, tmp_path): + """Functional test: run node-scraper with project plugin_config.json (paths + directory_paths + checks).""" + assert plugin_config_json.exists(), f"Config not found: {plugin_config_json}" + + log_path = str(tmp_path / "logs_plugin_config") + result = run_cli_command( + ["--log-path", log_path, f"--plugin-configs={plugin_config_json}"], check=False + ) + + assert result.returncode in [0, 1, 2] + output = result.stdout + result.stderr + assert len(output) > 0 + assert "SysSettingsPlugin" in output + # Config exercises paths + directory_paths (/sys/class/net) + pattern check; collector/analyzer run + assert "Sysfs" in output or "sys" in output.lower() diff --git a/test/unit/cli/test_cli_no_console_stdout.py b/test/unit/cli/test_cli_no_console_stdout.py index afa53d13..c762511d 100644 --- a/test/unit/cli/test_cli_no_console_stdout.py +++ b/test/unit/cli/test_cli_no_console_stdout.py @@ -1,152 +1,152 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2025 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -import io -import json -from contextlib import redirect_stdout -from unittest.mock import MagicMock, patch - -import pytest - -from nodescraper.cli.cli import main - - -def _assert_main_leaves_stdout_empty(argv: list[str]) -> None: - out = io.StringIO() - with redirect_stdout(out): - with pytest.raises(SystemExit) as exc: - main(argv) - assert out.getvalue() == "", f"Unexpected stdout: {out.getvalue()!r}" - code = exc.value.code - if code is None: - code = 0 - assert code in (0, 1), f"Unexpected exit code: {exc.value.code!r}" - - -@pytest.fixture -def no_console_base(tmp_path): - log_base = tmp_path / "logs" - log_base.mkdir(parents=True, exist_ok=True) - return ["--log-path", str(log_base), "--no-console-log", "--log-level", "ERROR"] - - -def test_describe_no_stdout(no_console_base): - _assert_main_leaves_stdout_empty( - no_console_base + ["describe", "plugin", "BiosPlugin"], - ) - - -def test_summary_no_stdout(no_console_base, tmp_path): - search = tmp_path / "search_here" - search.mkdir() - _assert_main_leaves_stdout_empty( - no_console_base + ["summary", "--search-path", str(search)], - ) - - -def test_gen_plugin_config_no_stdout(no_console_base, tmp_path): - out_dir = tmp_path / "cfg_out" - out_dir.mkdir() - _assert_main_leaves_stdout_empty( - no_console_base - + [ - "gen-plugin-config", - "--plugins", - "BiosPlugin", - "--output-path", - str(out_dir), - "--config-name", - "out_config.json", - ], - ) - - -def test_compare_runs_no_stdout(no_console_base, tmp_path): - d1 = tmp_path / "run_a" - d2 = tmp_path / "run_b" - d1.mkdir() - d2.mkdir() - _assert_main_leaves_stdout_empty( - no_console_base + ["compare-runs", str(d1), str(d2)], - ) - - -def test_run_plugins_empty_config_no_stdout(no_console_base, tmp_path): - cfg = tmp_path / "empty_plugins.json" - cfg.write_text( - json.dumps( - { - "name": "empty", - "desc": "", - "plugins": {}, - "global_args": {}, - "result_collators": {}, - } - ), - encoding="utf-8", - ) - _assert_main_leaves_stdout_empty( - no_console_base + ["run-plugins", f"--plugin-configs={cfg}"], - ) - - -@patch("nodescraper.cli.cli.get_oem_diagnostic_allowable_values", return_value=["DiagTypeA"]) -@patch("nodescraper.cli.cli.RedfishConnection") -def test_show_redfish_oem_allowable_no_stdout( - mock_conn_cls, - _mock_get_allowable, - no_console_base, - tmp_path, -): - conn_path = tmp_path / "conn.json" - conn_path.write_text( - json.dumps( - { - "RedfishConnectionManager": { - "host": "127.0.0.1", - "username": "u", - "password": "p", - "verify_ssl": False, - } - } - ), - encoding="utf-8", - ) - mock_inst = MagicMock() - mock_conn_cls.return_value = mock_inst - - _assert_main_leaves_stdout_empty( - no_console_base - + [ - "--connection-config", - str(conn_path), - "show-redfish-oem-allowable", - "--log-service-path", - "redfish/v1/Systems/1/LogServices/Logs", - ], - ) - mock_inst._ensure_session.assert_called_once() - mock_inst.close.assert_called_once() +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import io +import json +from contextlib import redirect_stdout +from unittest.mock import MagicMock, patch + +import pytest + +from nodescraper.cli.cli import main + + +def _assert_main_leaves_stdout_empty(argv: list[str]) -> None: + out = io.StringIO() + with redirect_stdout(out): + with pytest.raises(SystemExit) as exc: + main(argv) + assert out.getvalue() == "", f"Unexpected stdout: {out.getvalue()!r}" + code = exc.value.code + if code is None: + code = 0 + assert code in (0, 1), f"Unexpected exit code: {exc.value.code!r}" + + +@pytest.fixture +def no_console_base(tmp_path): + log_base = tmp_path / "logs" + log_base.mkdir(parents=True, exist_ok=True) + return ["--log-path", str(log_base), "--no-console-log", "--log-level", "ERROR"] + + +def test_describe_no_stdout(no_console_base): + _assert_main_leaves_stdout_empty( + no_console_base + ["describe", "plugin", "BiosPlugin"], + ) + + +def test_summary_no_stdout(no_console_base, tmp_path): + search = tmp_path / "search_here" + search.mkdir() + _assert_main_leaves_stdout_empty( + no_console_base + ["summary", "--search-path", str(search)], + ) + + +def test_gen_plugin_config_no_stdout(no_console_base, tmp_path): + out_dir = tmp_path / "cfg_out" + out_dir.mkdir() + _assert_main_leaves_stdout_empty( + no_console_base + + [ + "gen-plugin-config", + "--plugins", + "BiosPlugin", + "--output-path", + str(out_dir), + "--config-name", + "out_config.json", + ], + ) + + +def test_compare_runs_no_stdout(no_console_base, tmp_path): + d1 = tmp_path / "run_a" + d2 = tmp_path / "run_b" + d1.mkdir() + d2.mkdir() + _assert_main_leaves_stdout_empty( + no_console_base + ["compare-runs", str(d1), str(d2)], + ) + + +def test_run_plugins_empty_config_no_stdout(no_console_base, tmp_path): + cfg = tmp_path / "empty_plugins.json" + cfg.write_text( + json.dumps( + { + "name": "empty", + "desc": "", + "plugins": {}, + "global_args": {}, + "result_collators": {}, + } + ), + encoding="utf-8", + ) + _assert_main_leaves_stdout_empty( + no_console_base + ["run-plugins", f"--plugin-configs={cfg}"], + ) + + +@patch("nodescraper.cli.cli.get_oem_diagnostic_allowable_values", return_value=["DiagTypeA"]) +@patch("nodescraper.cli.cli.RedfishConnection") +def test_show_redfish_oem_allowable_no_stdout( + mock_conn_cls, + _mock_get_allowable, + no_console_base, + tmp_path, +): + conn_path = tmp_path / "conn.json" + conn_path.write_text( + json.dumps( + { + "RedfishConnectionManager": { + "host": "127.0.0.1", + "username": "u", + "password": "p", + "verify_ssl": False, + } + } + ), + encoding="utf-8", + ) + mock_inst = MagicMock() + mock_conn_cls.return_value = mock_inst + + _assert_main_leaves_stdout_empty( + no_console_base + + [ + "--connection-config", + str(conn_path), + "show-redfish-oem-allowable", + "--log-service-path", + "redfish/v1/Systems/1/LogServices/Logs", + ], + ) + mock_inst._ensure_session.assert_called_once() + mock_inst.close.assert_called_once() diff --git a/test/unit/cli/test_dynamic_parser_builder.py b/test/unit/cli/test_dynamic_parser_builder.py new file mode 100644 index 00000000..049e4eeb --- /dev/null +++ b/test/unit/cli/test_dynamic_parser_builder.py @@ -0,0 +1,47 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +import argparse +from typing import Literal, Optional + +from pydantic import BaseModel + +from nodescraper.cli.dynamicparserbuilder import DynamicParserBuilder +from nodescraper.typeutils import TypeUtils + + +class LiteralArgsModel(BaseModel): + mode: Literal["fast", "slow"] = "fast" + opt_mode: Optional[Literal["on", "off"]] = None + + +def _action_for(parser: argparse.ArgumentParser, option: str) -> argparse.Action: + for action in parser._actions: + if option in action.option_strings: + return action + raise AssertionError(f"no action for {option}") + + +def test_optional_literal_field_gets_choices() -> None: + """Baseline: Optional[Literal[...]] is rendered with argparse choices.""" + parser = argparse.ArgumentParser() + DynamicParserBuilder(parser, object).build_model_arg_parser(LiteralArgsModel, required=False) # type: ignore + + assert _action_for(parser, "--opt-mode").choices == ["on", "off"] + + +def test_plain_literal_field_gets_choices() -> None: + """A bare Literal[...] field must also be rendered with argparse choices.""" + parser = argparse.ArgumentParser() + DynamicParserBuilder(parser, object).build_model_arg_parser(LiteralArgsModel, required=False) # type: ignore + + assert _action_for(parser, "--mode").choices == ["fast", "slow"] + + +def test_get_literal_choices_returns_literal_values() -> None: + """get_literal_choices must return the Literal's allowed values.""" + type_classes = TypeUtils.process_type(Literal["fast", "slow"]) # type: ignore + type_class_map = {tc.type_class: tc for tc in type_classes} + + assert DynamicParserBuilder.get_literal_choices(type_class_map) == ["fast", "slow"] diff --git a/test/unit/cli/test_input_arg_types.py b/test/unit/cli/test_input_arg_types.py new file mode 100644 index 00000000..7046244a --- /dev/null +++ b/test/unit/cli/test_input_arg_types.py @@ -0,0 +1,46 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pytest +from pydantic import BaseModel + +from nodescraper.cli.inputargtypes import ModelArgHandler + + +class SampleArgs(BaseModel): + name: str = "" + count: int = 0 + + +def _write(tmp_path: Path, payload) -> str: + arg_file = tmp_path / "arg.json" + arg_file.write_text(json.dumps(payload), encoding="utf-8") + return str(arg_file) + + +def test_process_file_arg_builds_model(tmp_path: Path): + """Baseline: a valid json object is loaded into the model.""" + path = _write(tmp_path, {"name": "abc", "count": 2}) + + assert ModelArgHandler(SampleArgs).process_file_arg(path) == SampleArgs(name="abc", count=2) + + +def test_process_file_arg_invalid_value_raises_arg_type_error(tmp_path: Path): + """Baseline: a validation failure is surfaced as an argparse error.""" + path = _write(tmp_path, {"count": "not-an-int"}) + + with pytest.raises(argparse.ArgumentTypeError): + ModelArgHandler(SampleArgs).process_file_arg(path) + + +def test_process_file_arg_non_object_json_raises_arg_type_error(tmp_path: Path): + """A json file that is not an object must be reported as an argparse error.""" + path = _write(tmp_path, [{"name": "abc"}]) + + with pytest.raises(argparse.ArgumentTypeError): + ModelArgHandler(SampleArgs).process_file_arg(path) diff --git a/test/unit/cli/test_plugin_configs_cli.py b/test/unit/cli/test_plugin_configs_cli.py index 90e1171a..54c90fe9 100644 --- a/test/unit/cli/test_plugin_configs_cli.py +++ b/test/unit/cli/test_plugin_configs_cli.py @@ -1,33 +1,33 @@ -############################################################################### -# -# MIT License -# -# Copyright (C) 2026 Advanced Micro Devices, Inc. -# -############################################################################### - -from __future__ import annotations - -from nodescraper.cli.cli import build_parser -from nodescraper.configregistry import ConfigRegistry -from nodescraper.pluginrecipe.all_plugins import AllPlugins -from nodescraper.pluginrecipe.node_status import NodeStatus -from nodescraper.pluginregistry import PluginRegistry - - -def _parser(): - plugin_reg = PluginRegistry() - config_reg = ConfigRegistry(load_entry_point_configs=False) - for recipe in (NodeStatus, AllPlugins): - config_reg.configs[recipe.name()] = recipe.plugin_config() - return build_parser(plugin_reg, config_reg)[0] - - -def test_plugin_configs_equals_form_parses_csv() -> None: - ns = _parser().parse_args(["--plugin-configs=NodeStatus,AllPlugins"]) - assert ns.plugin_configs == ["NodeStatus", "AllPlugins"] - - -def test_plugin_configs_space_separated_parses() -> None: - ns = _parser().parse_args(["--plugin-configs", "NodeStatus,AllPlugins"]) - assert ns.plugin_configs == ["NodeStatus", "AllPlugins"] +############################################################################### +# +# MIT License +# +# Copyright (C) 2026 Advanced Micro Devices, Inc. +# +############################################################################### + +from __future__ import annotations + +from nodescraper.cli.cli import build_parser +from nodescraper.configregistry import ConfigRegistry +from nodescraper.pluginrecipe.all_ib_plugins import AllIbPlugins +from nodescraper.pluginrecipe.node_status import NodeStatus +from nodescraper.pluginregistry import PluginRegistry + + +def _parser(): + plugin_reg = PluginRegistry() + config_reg = ConfigRegistry(load_entry_point_configs=False) + for recipe in (NodeStatus, AllIbPlugins): + config_reg.configs[recipe.name()] = recipe.plugin_config() + return build_parser(plugin_reg, config_reg)[0] + + +def test_plugin_configs_equals_form_parses_csv() -> None: + ns = _parser().parse_args(["--plugin-configs=NodeStatus,AllIbPlugins"]) + assert ns.plugin_configs == ["NodeStatus", "AllIbPlugins"] + + +def test_plugin_configs_space_separated_parses() -> None: + ns = _parser().parse_args(["--plugin-configs", "NodeStatus,AllIbPlugins"]) + assert ns.plugin_configs == ["NodeStatus", "AllIbPlugins"] diff --git a/test/unit/connection/inband/test_shellcommand.py b/test/unit/connection/inband/test_shellcommand.py index 0a1b3fc2..3b4fc390 100644 --- a/test/unit/connection/inband/test_shellcommand.py +++ b/test/unit/connection/inband/test_shellcommand.py @@ -23,9 +23,14 @@ # SOFTWARE. # ############################################################################### +import subprocess from unittest.mock import MagicMock, patch +from paramiko.ssh_exception import SSHException + from nodescraper.connection.inband.inbandlocal import LocalShell +from nodescraper.connection.inband.inbandremote import RemoteShell +from nodescraper.connection.inband.sshparams import SSHConnectionParams @patch("nodescraper.connection.inband.inbandlocal.subprocess.run") @@ -45,4 +50,93 @@ def test_localshell_string_with_sudo(mock_run): shell = LocalShell() shell.run_command("cat /etc/shadow", sudo=True) - assert mock_run.call_args.args[0] == "sudo cat /etc/shadow" + assert mock_run.call_args.args[0] == "sudo -n cat /etc/shadow" + assert mock_run.call_args.kwargs["stdin"] is subprocess.DEVNULL + + +@patch("nodescraper.connection.inband.inbandlocal.subprocess.run") +def test_localshell_timeout_returns_exit_124(mock_run): + mock_run.side_effect = subprocess.TimeoutExpired( + cmd="sleep 5", timeout=1, output=b"", stderr=b"" + ) + + artifact = LocalShell().run_command("sleep 5", timeout=1) + + assert artifact.exit_code == 124 + assert "timed out" in artifact.stderr + + +@patch("nodescraper.connection.inband.inbandremote.paramiko.SSHClient") +def test_remoteshell_sudo_password_is_newline_terminated(mock_client_cls): + """sudo -S reads a line, so the password written to stdin must end with a newline.""" + stdin, stdout, stderr = MagicMock(), MagicMock(), MagicMock() + stdout.read.return_value = b"" + stderr.read.return_value = b"" + stdout.channel.recv_exit_status.return_value = 0 + mock_client_cls.return_value.exec_command.return_value = (stdin, stdout, stderr) + + shell = RemoteShell( + SSHConnectionParams(hostname="127.0.0.1", username="user", password="hunter2") + ) + shell.run_command("cat /etc/shadow", sudo=True) + + stdin.write.assert_called_once_with("hunter2\n") + stdout.channel.close.assert_called_once() + + +@patch("nodescraper.connection.inband.inbandremote.paramiko.SSHClient") +def test_remoteshell_ssh_exception_is_a_failed_command(mock_client_cls): + mock_client_cls.return_value.exec_command.side_effect = SSHException("Timeout opening channel.") + shell = RemoteShell( + SSHConnectionParams(hostname="127.0.0.1", username="user", password="secret") + ) + artifact = shell.run_command("mktemp") + assert artifact.exit_code == 124 + assert artifact.stderr == "Timeout opening channel." + + +def _sftp_file(data: bytes) -> MagicMock: + handle = MagicMock() + handle.read.return_value = data + handle.__enter__.return_value = handle + handle.__exit__.return_value = False + return handle + + +@patch("nodescraper.connection.inband.inbandremote.paramiko.SSHClient") +def test_reconnect_ssh_opens_a_new_client(mock_client_cls): + clients = [] + + def factory(): + client = MagicMock() + client.get_transport.return_value.channel_timeout = None + clients.append(client) + return client + + mock_client_cls.side_effect = factory + shell = RemoteShell( + SSHConnectionParams(hostname="127.0.0.1", username="user", password="secret") + ) + shell.reconnect_ssh() + assert len(clients) == 2 + clients[0].close.assert_called_once() + clients[1].connect.assert_called_once() + assert shell.client is clients[1] + assert clients[1].get_transport.return_value.channel_timeout == 60 + + +@patch("nodescraper.connection.inband.inbandremote.paramiko.SSHClient") +def test_read_files_uses_one_sftp_session(mock_client_cls): + sftp = MagicMock() + sftp.open.side_effect = [ + _sftp_file(b"HTTP/1.1 200 OK\n\n"), + _sftp_file(b"{}"), + ] + mock_client_cls.return_value.open_sftp.return_value = sftp + shell = RemoteShell( + SSHConnectionParams(hostname="127.0.0.1", username="user", password="secret") + ) + contents = shell.read_files(["/tmp/headers", "/tmp/body"]) + assert contents == [b"HTTP/1.1 200 OK\n\n", b"{}"] + mock_client_cls.return_value.open_sftp.assert_called_once() + sftp.close.assert_called_once() diff --git a/test/unit/connection/redfish/test_redfish_clear_log.py b/test/unit/connection/redfish/test_redfish_clear_log.py new file mode 100644 index 00000000..feab74d0 --- /dev/null +++ b/test/unit/connection/redfish/test_redfish_clear_log.py @@ -0,0 +1,373 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock + +from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + _odata_id_to_path, + _parse_clear_endpoint, + clear_redfish_logs, + discover_clear_log_endpoints, +) +from nodescraper.connection.redfish.redfish_connection import RedfishConnectionError + + +def _ok_resp(body: dict) -> MagicMock: + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.json.return_value = body + return resp + + +def _err_resp(status: int = 404) -> MagicMock: + resp = MagicMock() + resp.ok = False + resp.status_code = status + resp.json.return_value = {} + return resp + + +class TestOdataIdToPath: + def test_relative_path(self): + assert _odata_id_to_path("/redfish/v1/Systems", "") == "redfish/v1/Systems" + + def test_strips_leading_slash(self): + assert _odata_id_to_path("/redfish/v1", "") == "redfish/v1" + + def test_full_url_extracts_path(self): + result = _odata_id_to_path("https://bmc/redfish/v1/Systems/1", "") + assert result == "redfish/v1/Systems/1" + + def test_strips_base_url(self): + result = _odata_id_to_path( + "https://bmc/redfish/v1/Systems", + "https://bmc", + ) + assert result == "redfish/v1/Systems" + + def test_empty_returns_empty(self): + assert _odata_id_to_path("", "") == "" + + def test_none_returns_empty(self): + assert _odata_id_to_path(None, "") == "" # type: ignore[arg-type] + + +class TestParseClearEndpoint: + def test_hash_prefixed_action(self): + svc_data = { + "Actions": { + "#LogService.ClearLog": { + "target": "/redfish/v1/Systems/UBB/LogServices/DiagLogs/Actions/LogService.ClearLog" + } + }, + "OverWritePolicy": "WrapsWhenFull", + } + result = _parse_clear_endpoint(svc_data) + assert result is not None + assert ( + result.path + == "/redfish/v1/Systems/UBB/LogServices/DiagLogs/Actions/LogService.ClearLog" + ) + assert result.overwrite_policy == "WrapsWhenFull" + + def test_unprefixed_action(self): + svc_data = { + "Actions": { + "LogService.ClearLog": { + "target": "/redfish/v1/Managers/BMC/LogServices/Log/Actions/LogService.ClearLog" + } + } + } + result = _parse_clear_endpoint(svc_data) + assert result is not None + assert "ClearLog" in result.path + + def test_no_clear_action_returns_none(self): + svc_data = {"Actions": {"#LogService.CollectDiagnosticData": {"target": "/some/path"}}} + assert _parse_clear_endpoint(svc_data) is None + + def test_empty_target_returns_none(self): + svc_data = {"Actions": {"#LogService.ClearLog": {"target": ""}}} + assert _parse_clear_endpoint(svc_data) is None + + def test_missing_actions_returns_none(self): + assert _parse_clear_endpoint({}) is None + + def test_default_overwrite_policy_empty_string(self): + svc_data = {"Actions": {"#LogService.ClearLog": {"target": "/path"}}} + result = _parse_clear_endpoint(svc_data) + assert result is not None + assert result.overwrite_policy == "" + + def test_never_overwrite_policy_preserved(self): + svc_data = { + "Actions": {"#LogService.ClearLog": {"target": "/path"}}, + "OverWritePolicy": "NeverOverWrites", + } + result = _parse_clear_endpoint(svc_data) + assert result is not None + assert result.overwrite_policy == "NeverOverWrites" + + +class TestClearRedfishLogs: + def test_success(self): + conn = MagicMock() + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + conn.post.return_value = resp + + endpoints = [ClearLogEndpoint(path="/path/ClearLog")] + results = clear_redfish_logs(conn, endpoints) + assert len(results) == 1 + assert results[0].success is True + assert results[0].status_code == 200 + conn.post.assert_called_once_with("/path/ClearLog", json={}) + + def test_failure_non_ok_response(self): + conn = MagicMock() + resp = MagicMock() + resp.ok = False + resp.status_code = 500 + conn.post.return_value = resp + + endpoints = [ClearLogEndpoint(path="/path/ClearLog")] + results = clear_redfish_logs(conn, endpoints) + assert len(results) == 1 + assert results[0].success is False + assert results[0].status_code == 500 + assert "500" in results[0].error + + def test_connection_error_returns_failure(self): + conn = MagicMock() + conn.post.side_effect = RedfishConnectionError("connection refused") + + endpoints = [ClearLogEndpoint(path="/path/ClearLog")] + results = clear_redfish_logs(conn, endpoints) + assert len(results) == 1 + assert results[0].success is False + assert "connection refused" in results[0].error + + def test_never_overwrite_skipped(self): + conn = MagicMock() + endpoints = [ClearLogEndpoint(path="/path", overwrite_policy="NeverOverWrites")] + results = clear_redfish_logs(conn, endpoints) + assert results == [] + conn.post.assert_not_called() + + def test_mixed_never_overwrite_and_clearable(self): + conn = MagicMock() + resp = MagicMock() + resp.ok = True + resp.status_code = 204 + conn.post.return_value = resp + + endpoints = [ + ClearLogEndpoint(path="/clear/a"), + ClearLogEndpoint(path="/clear/b", overwrite_policy="NeverOverWrites"), + ] + results = clear_redfish_logs(conn, endpoints) + assert len(results) == 1 + assert results[0].path == "/clear/a" + conn.post.assert_called_once_with("/clear/a", json={}) + + def test_continues_on_first_failure(self): + conn = MagicMock() + + def post_side(path, json): + r = MagicMock() + r.ok = path != "/bad" + r.status_code = 500 if path == "/bad" else 200 + return r + + conn.post.side_effect = post_side + endpoints = [ClearLogEndpoint(path="/bad"), ClearLogEndpoint(path="/good")] + results = clear_redfish_logs(conn, endpoints) + assert len(results) == 2 + assert results[0].success is False + assert results[1].success is True + + def test_empty_endpoints_returns_empty(self): + conn = MagicMock() + assert clear_redfish_logs(conn, []) == [] + conn.post.assert_not_called() + + +class TestDiscoverClearLogEndpoints: + def _make_conn(self, api_root="redfish/v1", base_url="https://bmc"): + conn = MagicMock() + conn.api_root = api_root + conn.base_url = base_url + return conn + + def test_discovers_clear_endpoint(self): + conn = self._make_conn() + + systems_body = {"Members": [{"@odata.id": "/redfish/v1/Systems/UBB"}]} + ls_list_body = {"Members": [{"@odata.id": "/redfish/v1/Systems/UBB/LogServices/DiagLogs"}]} + svc_body = { + "Actions": { + "#LogService.ClearLog": { + "target": "/redfish/v1/Systems/UBB/LogServices/DiagLogs/Actions/LogService.ClearLog" + } + }, + "OverWritePolicy": "WrapsWhenFull", + } + + def get_response(path): + if "Systems" == path.strip("/").split("/")[-1] or path.endswith("/Systems"): + return _ok_resp(systems_body) + if "LogServices" in path and path.endswith("LogServices"): + return _ok_resp(ls_list_body) + if "DiagLogs" in path and "Actions" not in path: + return _ok_resp(svc_body) + return _err_resp() + + conn.get_response.side_effect = get_response + results = discover_clear_log_endpoints(conn, roots=["Systems"]) + assert len(results) == 1 + assert "ClearLog" in results[0].path + assert results[0].overwrite_policy == "WrapsWhenFull" + + def test_skips_member_with_no_log_services(self): + conn = self._make_conn() + + def get_response(path): + if path.endswith("/Systems"): + return _ok_resp({"Members": [{"@odata.id": "/redfish/v1/Systems/UBB"}]}) + if "LogServices" in path: + return _err_resp(404) + return _err_resp() + + conn.get_response.side_effect = get_response + results = discover_clear_log_endpoints(conn, roots=["Systems"]) + assert results == [] + + def test_skips_service_without_clear_action(self): + conn = self._make_conn() + + def get_response(path): + if path.endswith("/Systems"): + return _ok_resp({"Members": [{"@odata.id": "/redfish/v1/Systems/UBB"}]}) + if path.endswith("/LogServices"): + return _ok_resp( + {"Members": [{"@odata.id": "/redfish/v1/Systems/UBB/LogServices/EventLog"}]} + ) + if path.endswith("/EventLog"): + return _ok_resp( + {"Actions": {"#LogService.CollectDiagnosticData": {"target": "/collect"}}} + ) + return _err_resp() + + conn.get_response.side_effect = get_response + results = discover_clear_log_endpoints(conn, roots=["Systems"]) + assert results == [] + + def test_explicit_member_ids_skip_root_get(self): + conn = self._make_conn() + + svc_body = { + "Actions": { + "#LogService.ClearLog": { + "target": "/redfish/v1/Managers/AMC/LogServices/Log/Actions/LogService.ClearLog" + } + } + } + + def get_response(path): + if path.endswith("/LogServices"): + return _ok_resp( + {"Members": [{"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Log"}]} + ) + if path.endswith("/Log"): + return _ok_resp(svc_body) + return _err_resp() + + conn.get_response.side_effect = get_response + results = discover_clear_log_endpoints( + conn, roots=["Managers"], member_ids={"Managers": ["AMC"]} + ) + assert len(results) == 1 + # Root collection GET should never have been called + calls = [str(c) for c in conn.get_response.call_args_list] + assert not any( + "Managers'" == p.strip("'").strip("/").split("/")[-1] + for p in calls + if "LogServices" not in p and "Log" not in p + ) + + def test_connection_error_on_root_is_skipped(self): + conn = self._make_conn() + conn.get_response.side_effect = RedfishConnectionError("timeout") + results = discover_clear_log_endpoints(conn, roots=["Systems"]) + assert results == [] + + def test_non_ok_root_response_is_skipped(self): + conn = self._make_conn() + conn.get_response.return_value = _err_resp(503) + results = discover_clear_log_endpoints(conn, roots=["Systems"]) + assert results == [] + + def test_multiple_roots_combined(self): + conn = self._make_conn() + + clear_target_systems = ( + "/redfish/v1/Systems/UBB/LogServices/DiagLogs/Actions/LogService.ClearLog" + ) + clear_target_managers = ( + "/redfish/v1/Managers/AMC/LogServices/Log/Actions/LogService.ClearLog" + ) + + def get_response(path): + if path.endswith("/redfish/v1/Systems"): + return _ok_resp({"Members": [{"@odata.id": "/redfish/v1/Systems/UBB"}]}) + if path.endswith("/redfish/v1/Managers"): + return _ok_resp({"Members": [{"@odata.id": "/redfish/v1/Managers/AMC"}]}) + if path.rstrip("/") == "/redfish/v1/Systems/UBB/LogServices": + return _ok_resp( + {"Members": [{"@odata.id": "/redfish/v1/Systems/UBB/LogServices/DiagLogs"}]} + ) + if path.rstrip("/") == "/redfish/v1/Managers/AMC/LogServices": + return _ok_resp( + {"Members": [{"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Log"}]} + ) + if path.rstrip("/") == "/redfish/v1/Systems/UBB/LogServices/DiagLogs": + return _ok_resp( + {"Actions": {"#LogService.ClearLog": {"target": clear_target_systems}}} + ) + if path.rstrip("/") == "/redfish/v1/Managers/AMC/LogServices/Log": + return _ok_resp( + {"Actions": {"#LogService.ClearLog": {"target": clear_target_managers}}} + ) + return _err_resp() + + conn.get_response.side_effect = get_response + results = discover_clear_log_endpoints(conn, roots=["Systems", "Managers"]) + assert len(results) == 2 + paths = {r.path for r in results} + assert clear_target_systems in paths + assert clear_target_managers in paths diff --git a/test/unit/connection/redfish/test_redfish_connection.py b/test/unit/connection/redfish/test_redfish_connection.py new file mode 100644 index 00000000..b7cb61fa --- /dev/null +++ b/test/unit/connection/redfish/test_redfish_connection.py @@ -0,0 +1,39 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from nodescraper.connection.redfish import RedfishConnection + + +@pytest.fixture +def rf_conn() -> RedfishConnection: + return RedfishConnection( + base_url="https://bmc.example", + username="u", + password=None, + verify_ssl=False, + ) + + +@patch("nodescraper.connection.redfish.redfish_connection.requests.Session") +def test_close_clears_cached_session(mock_session_cls, rf_conn: RedfishConnection) -> None: + """Baseline: close() drops the cached session so a later call builds a new one.""" + rf_conn._ensure_session() + rf_conn.close() + + assert rf_conn._session is None + + +@patch("nodescraper.connection.redfish.redfish_connection.requests.Session") +def test_close_releases_underlying_http_session( + mock_session_cls, rf_conn: RedfishConnection +) -> None: + """close() must close the requests session so its connection pool is released.""" + session = rf_conn._ensure_session() + rf_conn.close() + + session.close.assert_called_once() # type: ignore diff --git a/test/unit/connection/redfish/test_redfish_connection_ssl.py b/test/unit/connection/redfish/test_redfish_connection_ssl.py new file mode 100644 index 00000000..aac80b2c --- /dev/null +++ b/test/unit/connection/redfish/test_redfish_connection_ssl.py @@ -0,0 +1,80 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock, patch + +import pytest + +from nodescraper.connection.redfish import RedfishConnection + + +def test_verify_ssl_false_disables_trust_env() -> None: + conn = RedfishConnection( + base_url="https://bmc.example", + username="u", + password="p", + verify_ssl=False, + use_session_auth=False, + ) + with patch( + "nodescraper.connection.redfish.redfish_connection.requests.Session" + ) as mock_session_cls: + mock_session = MagicMock() + mock_session_cls.return_value = mock_session + session = conn._ensure_session() + + assert session is mock_session + assert mock_session.verify is False + assert mock_session.trust_env is False + + +@pytest.mark.parametrize("env_var", ["REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"]) +def test_verify_ssl_false_passes_verify_on_login( + env_var: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(env_var, "/etc/ssl/certs/ca-bundle.crt") + conn = RedfishConnection( + base_url="https://bmc.example", + username="u", + password="p", + verify_ssl=False, + ) + with patch( + "nodescraper.connection.redfish.redfish_connection.requests.Session" + ) as mock_session_cls: + mock_session = MagicMock() + mock_response = MagicMock() + mock_response.ok = True + mock_response.headers = { + "X-Auth-Token": "token", + "Location": "/redfish/v1/SessionService/Sessions/1", + } + mock_session.post.return_value = mock_response + mock_session_cls.return_value = mock_session + conn._ensure_session() + + mock_session.post.assert_called_once() + assert mock_session.verify is False + assert mock_session.trust_env is False diff --git a/test/unit/connection/redfish/test_redfish_multi_target.py b/test/unit/connection/redfish/test_redfish_multi_target.py new file mode 100644 index 00000000..450f3910 --- /dev/null +++ b/test/unit/connection/redfish/test_redfish_multi_target.py @@ -0,0 +1,332 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging +from typing import cast +from unittest.mock import MagicMock, patch + +from nodescraper.base import OOBandDataPlugin, RedfishDataCollector +from nodescraper.connection.oob_ssh.oob_ssh_connection_manager import ( + OobSshConnectionManager, +) +from nodescraper.connection.redfish import ( + MultiTargetRedfishConnection, + RedfishConnection, + RedfishConnectionError, + RedfishConnectionManager, + RedfishConnectionParams, +) +from nodescraper.enums import ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import DataModel, SystemInfo, TaskResult +from nodescraper.taskresulthooks.filesystemloghook import FileSystemLogHook + + +class SampleModel(DataModel): + label: str = "" + + +class SampleCollector(RedfishDataCollector[SampleModel, None]): + DATA_MODEL = SampleModel + + def collect_data(self, args=None): + """Collect a label from the fake connection. + + Args: + args: Unused collection arguments. + + Returns: + tuple[TaskResult, Optional[SampleModel]]: Collector result and data model. + """ + marker = getattr(self.connection, "marker", "") + if marker == "boom": + raise RuntimeError("bmc down") + self.result.status = ExecutionStatus.OK + self.result.message = marker + return self.result, SampleModel(label=marker) + + +class SampleAnalyzer(DataAnalyzer[SampleModel, None]): + DATA_MODEL = SampleModel + + def analyze_data(self, data, args=None): + """Flag a bad label as an analysis error. + + Args: + data: Collected sample model. + args: Unused analysis arguments. + + Returns: + TaskResult: Analysis result for this target. + """ + if data.label == "bad": + self.result.status = ExecutionStatus.ERROR + self.result.message = "check failed" + else: + self.result.status = ExecutionStatus.OK + self.result.message = "ok" + return self.result + + +class SamplePlugin(OOBandDataPlugin[SampleModel, None, None]): + DATA_MODEL = SampleModel + COLLECTOR = SampleCollector + ANALYZER = SampleAnalyzer + + +def _manager( + system_info: SystemInfo, targets: list[RedfishConnectionParams], **extra +) -> RedfishConnectionManager: + return RedfishConnectionManager( + system_info=system_info, + connection_args=RedfishConnectionParams(targets=targets, **extra), + ) + + +@patch("nodescraper.connection.redfish.redfish_manager.RedfishConnection") +def test_partial_connect_keeps_reachable_bmc(mock_conn_cls, system_info: SystemInfo) -> None: + """One unreachable BMC leaves the reachable BMC connected at warning.""" + + def build(**kwargs): + conn = MagicMock() + if "down" in kwargs["base_url"]: + conn._ensure_session.side_effect = RedfishConnectionError("unreachable") + return conn + + mock_conn_cls.side_effect = build + manager = _manager( + system_info, + [ + RedfishConnectionParams(target_key="good", host="bmc-good.example", username="admin"), + RedfishConnectionParams(target_key="bad", host="bmc-down.example", username="admin"), + ], + max_workers=2, + ) + + result = manager.connect() + + assert result.status == ExecutionStatus.WARNING + assert isinstance(manager.connection, MultiTargetRedfishConnection) + assert set(manager.connection.target_connections) == {"good"} + assert "bad" in manager.connection.failed_targets + assert manager.connection.max_workers == 2 + + +@patch("nodescraper.connection.redfish.redfish_manager.RedfishConnection") +def test_every_target_down_fails_connect(mock_conn_cls, system_info: SystemInfo) -> None: + """Every target failing is an execution failure and does not set a connection.""" + mock_conn_cls.side_effect = lambda **kwargs: MagicMock( + _ensure_session=MagicMock(side_effect=RedfishConnectionError("unreachable")) + ) + manager = _manager( + system_info, + [ + RedfishConnectionParams(target_key="a", host="bmc-a.example", username="admin"), + RedfishConnectionParams(target_key="b", host="bmc-b.example", username="admin"), + ], + ) + + result = manager.connect() + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert manager.connection is None + + +def test_one_bmc_collection_failure_stays_warning(system_info: SystemInfo) -> None: + """A collect failure stays a warning when another target succeeds.""" + good = MagicMock() + good.marker = "good" + bad = MagicMock() + bad.marker = "boom" + multi = MultiTargetRedfishConnection( + {"good": good, "bad": bad}, + max_workers=2, + failed_targets={"offline": "Redfish connection failed for target 'offline'"}, + ) + collector = SampleCollector( + system_info=system_info, + connection=cast(RedfishConnection, multi), + logger=logging.getLogger("multi-target-test"), + parent="SamplePlugin", + ) + + result, data = collector.collect_data() + + assert data is None + assert result.status == ExecutionStatus.WARNING + assert result.status <= ExecutionStatus.WARNING + assert multi.multi_target_data["good"].label == "good" + assert "bad" not in multi.multi_target_data + assert "offline" not in result.message + assert "[good] good" in result.message + assert "bad" in result.message + + +def test_missed_connect_stays_off_the_plugin_result(system_info: SystemInfo) -> None: + """A target that never connected is omitted from the plugin result.""" + good = MagicMock() + good.marker = "good" + multi = MultiTargetRedfishConnection( + {"good": good}, + failed_targets={"offline": "Redfish connection failed for target 'offline'"}, + ) + connection_manager = MagicMock() + connection_manager.connection = multi + connection_manager.result = TaskResult( + status=ExecutionStatus.WARNING, + message="Some Redfish targets failed to connect\n[offline] Redfish connection failed", + ) + plugin = SamplePlugin( + system_info=system_info, + logger=logging.getLogger("multi-target-test"), + connection_manager=connection_manager, + ) + + result = plugin.run() + + assert result.status == ExecutionStatus.OK + assert "offline" not in result.message + assert "failed to connect" not in result.message + assert result.message == "[good] good" + + +def test_multi_target_logs_under_collector_directory(system_info: SystemInfo, tmp_path) -> None: + """Collector logs for each target are written under the collector directory.""" + good = MagicMock() + good.marker = "good" + other = MagicMock() + other.marker = "other" + multi = MultiTargetRedfishConnection({"node-a": good, "node-b": other}) + collector_dir = tmp_path / "sample_plugin" / "sample_collector" + collector_dir.mkdir(parents=True) + collector = SampleCollector( + system_info=system_info, + connection=cast(RedfishConnection, multi), + logger=logging.getLogger("multi-target-test"), + parent="SamplePlugin", + log_path=str(collector_dir), + task_result_hooks=[FileSystemLogHook(log_base_path=str(tmp_path))], + ) + + collector.collect_data() + + assert (collector_dir / "node-a" / "result.json").is_file() + assert (collector_dir / "node-b" / "result.json").is_file() + assert not (tmp_path / "sample_plugin" / "node-a").exists() + + +def test_multi_target_analyzer_logs_under_analyzer_directory( + system_info: SystemInfo, tmp_path +) -> None: + """Analyzer logs for each target are written under the analyzer directory.""" + plugin = SamplePlugin( + system_info=system_info, + log_path=str(tmp_path), + logger=logging.getLogger("multi-target-test"), + ) + multi = MultiTargetRedfishConnection({"node-a": MagicMock()}) + multi.multi_target_data["node-a"] = SampleModel(label="good") + plugin.connection_manager = MagicMock(connection=multi) + + plugin.analyze() + + assert (tmp_path / "sample_plugin" / "sample_analyzer" / "node-a" / "result.json").is_file() + assert not (tmp_path / "sample_plugin" / "node-a").exists() + + +def test_analyze_reads_data_before_disconnect(system_info: SystemInfo) -> None: + """Analysis uses data still held on the open multi-target connection.""" + multi = MultiTargetRedfishConnection({"good": MagicMock()}) + multi.multi_target_data["good"] = SampleModel(label="good") + connection_manager = MagicMock() + connection_manager.connection = multi + connection_manager._multi_target_data = {} + plugin = SamplePlugin( + system_info=system_info, + logger=logging.getLogger("multi-target-test"), + connection_manager=connection_manager, + ) + + result = plugin.analyze() + + assert result.status == ExecutionStatus.OK + + +def test_analyze_error_on_collected_data_fails(system_info: SystemInfo) -> None: + """A check failure on data that was collected still fails analysis.""" + multi = MultiTargetRedfishConnection({"bad": MagicMock(), "good": MagicMock()}) + multi.multi_target_data["bad"] = SampleModel(label="bad") + multi.multi_target_data["good"] = SampleModel(label="good") + connection_manager = MagicMock() + connection_manager.connection = multi + plugin = SamplePlugin( + system_info=system_info, + logger=logging.getLogger("multi-target-test"), + connection_manager=connection_manager, + ) + + result = plugin.analyze() + + assert result.status == ExecutionStatus.ERROR + + +def test_oob_ssh_skips_targets_without_a_single_host(system_info: SystemInfo) -> None: + """OOB SSH does not fan out across a Redfish targets list.""" + manager = OobSshConnectionManager( + system_info=system_info, + connection_args=RedfishConnectionParams( + targets=[ + RedfishConnectionParams(host="bmc-a.example", username="admin"), + RedfishConnectionParams(host="bmc-b.example", username="admin"), + ] + ), + ) + + result = manager.connect() + + assert result.status == ExecutionStatus.NOT_RAN + assert manager.connection is None + + +@patch("nodescraper.connection.oob_ssh.oob_ssh_connection_manager.RemoteShell") +def test_oob_ssh_uses_top_level_host_beside_targets(mock_shell, system_info: SystemInfo) -> None: + """A top-level host keeps OOB SSH on one BMC while Redfish uses targets.""" + mock_shell.return_value = MagicMock() + manager = OobSshConnectionManager( + system_info=system_info, + connection_args=RedfishConnectionParams( + host="bmc-ssh.example", + username="admin", + targets=[ + RedfishConnectionParams(host="bmc-a.example", username="admin"), + RedfishConnectionParams(host="bmc-b.example", username="admin"), + ], + ), + ) + + result = manager.connect() + + assert result.status == ExecutionStatus.OK + assert mock_shell.call_args.args[0].hostname == "bmc-ssh.example" diff --git a/test/unit/connection/redfish/test_redfish_oem_diag.py b/test/unit/connection/redfish/test_redfish_oem_diag.py index 727c1b69..8151f6af 100644 --- a/test/unit/connection/redfish/test_redfish_oem_diag.py +++ b/test/unit/connection/redfish/test_redfish_oem_diag.py @@ -1,186 +1,536 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import logging -from unittest.mock import MagicMock - -from requests.status_codes import codes - -from nodescraper.connection.redfish import RedfishConnectionError -from nodescraper.connection.redfish.redfish_oem_diag import ( - DEFAULT_TASK_TIMEOUT_S, - RF_ANNOTATION_ALLOWABLE, - _download_log_and_save, - _get_task_monitor_uri, - _strip_port_from_url, - get_oem_diagnostic_allowable_values, -) - - -def test_rf_annotation_allowable_constant(): - assert RF_ANNOTATION_ALLOWABLE == "OEMDiagnosticDataType@Redfish.AllowableValues" - - -def test_default_task_timeout_s(): - assert DEFAULT_TASK_TIMEOUT_S == 1800 - - -class TestGetOemDiagnosticAllowableValues: - def test_returns_list_from_collect_action(self): - conn = MagicMock() - conn.get.return_value = { - "Actions": { - "LogService.CollectDiagnosticData": { - "OEMDiagnosticDataType@Redfish.AllowableValues": ["Dmesg", "AllLogs"], - } - } - } - result = get_oem_diagnostic_allowable_values( - conn, "redfish/v1/Systems/1/LogServices/DiagLogs" - ) - assert result == ["Dmesg", "AllLogs"] - - def test_returns_list_from_octothorpe_action_key(self): - conn = MagicMock() - conn.get.return_value = { - "Actions": { - "#LogService.CollectDiagnosticData": { - "OEMDiagnosticDataType@Redfish.AllowableValues": ["JournalControl"], - } - } - } - result = get_oem_diagnostic_allowable_values( - conn, "redfish/v1/Systems/UBB/LogServices/DiagLogs" - ) - assert result == ["JournalControl"] - - def test_returns_none_on_connection_error(self): - conn = MagicMock() - conn.get.side_effect = RedfishConnectionError("fail") - result = get_oem_diagnostic_allowable_values(conn, "redfish/v1/LogServices/DiagLogs") - assert result is None - - def test_returns_none_when_data_not_dict(self): - conn = MagicMock() - conn.get.return_value = [] - result = get_oem_diagnostic_allowable_values(conn, "redfish/v1/LogServices/DiagLogs") - assert result is None - - def test_returns_none_when_no_actions(self): - conn = MagicMock() - conn.get.return_value = {} - result = get_oem_diagnostic_allowable_values(conn, "redfish/v1/LogServices/DiagLogs") - assert result is None - - -class TestStripPortFromUrl: - def test_strips_port_443(self): - url = "https://host:443/redfish/v1/TaskService/Tasks/1" - assert _strip_port_from_url(url) == "https://host/redfish/v1/TaskService/Tasks/1" - - def test_strips_other_port(self): - url = "https://host:8443/redfish/v1" - assert _strip_port_from_url(url) == "https://host/redfish/v1" - - def test_returns_none_when_no_port(self): - url = "https://host/redfish/v1" - assert _strip_port_from_url(url) is None - - def test_returns_none_for_relative_path(self): - assert _strip_port_from_url("redfish/v1/Systems/1") is None - - -class TestGetTaskMonitorUri: - def test_returns_task_monitor_from_body(self): - conn = MagicMock() - conn.base_url = "https://host/redfish/v1" - body = {"TaskMonitor": "TaskService/Tasks/1/Monitor"} - result = _get_task_monitor_uri(body, conn) - assert result == "https://host/redfish/v1/TaskService/Tasks/1/Monitor" - - def test_returns_from_odata_id_plus_monitor(self): - conn = MagicMock() - conn.base_url = "https://host/redfish/v1" - body = {"@odata.id": "TaskService/Tasks/1"} - result = _get_task_monitor_uri(body, conn) - assert result == "https://host/redfish/v1/TaskService/Tasks/1/Monitor" - - def test_returns_none_for_empty_body(self): - conn = MagicMock() - assert _get_task_monitor_uri({}, conn) is None - - def test_prefers_task_monitor_over_odata_id(self): - conn = MagicMock() - conn.base_url = "https://host/redfish/v1" - body = { - "TaskMonitor": "TaskService/Tasks/1/Monitor", - "@odata.id": "TaskService/Tasks/2", - } - result = _get_task_monitor_uri(body, conn) - assert result == "https://host/redfish/v1/TaskService/Tasks/1/Monitor" - - -class TestDownloadLogAndSave: - def test_returns_none_when_no_additional_data_uri(self): - conn = MagicMock() - log_entry_json = {"Id": "1", "Name": "LogEntry"} - result = _download_log_and_save( - conn, log_entry_json, "Dmesg", None, logging.getLogger("test") - ) - assert result is None - conn.get_response.assert_not_called() - - def test_downloads_and_returns_bytes_when_additional_data_uri_present(self): - conn = MagicMock() - conn.base_url = "https://host/redfish/v1" - resp = MagicMock() - resp.status_code = codes.ok - resp.content = b"log bytes" - conn.get_response.return_value = resp - log_entry_json = {"AdditionalDataURI": "/redfish/v1/LogServices/1/Entries/1/Attachment"} - result = _download_log_and_save( - conn, log_entry_json, "Dmesg", None, logging.getLogger("test") - ) - assert result == b"log bytes" - conn.get_response.assert_called_once() - - def test_writes_archive_and_metadata_to_output_dir(self, tmp_path): - conn = MagicMock() - conn.base_url = "https://host/redfish/v1" - resp = MagicMock() - resp.status_code = codes.ok - resp.content = b"log bytes" - conn.get_response.return_value = resp - log_entry_json = { - "AdditionalDataURI": "/redfish/v1/LogServices/1/Entries/1/Attachment", - "Id": "1", - } - result = _download_log_and_save( - conn, log_entry_json, "AllLogs", tmp_path, logging.getLogger("test") - ) - assert result == b"log bytes" - assert (tmp_path / "AllLogs.tar.xz").read_bytes() == b"log bytes" - metadata = (tmp_path / "AllLogs_log_entry.json").read_text(encoding="utf-8") - assert "Id" in metadata and "1" in metadata +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging +from unittest.mock import MagicMock, patch + +from requests.status_codes import codes + +from nodescraper.connection.redfish import RedfishConnectionError +from nodescraper.connection.redfish.redfish_oem_diag import ( + DEFAULT_TASK_TIMEOUT_S, + RF_ANNOTATION_ALLOWABLE, + _download_log_and_save, + _entries_collection_path, + _get_task_monitor_uri, + _pick_member_href, + _strip_port_from_url, + _task_resource_path, + collect_oem_diagnostic_data, + get_oem_diagnostic_allowable_values, +) + + +def test_rf_annotation_allowable_constant(): + assert RF_ANNOTATION_ALLOWABLE == "OEMDiagnosticDataType@Redfish.AllowableValues" + + +def test_default_task_timeout_s(): + assert DEFAULT_TASK_TIMEOUT_S == 1800 + + +class TestGetOemDiagnosticAllowableValues: + def test_returns_list_from_collect_action(self): + conn = MagicMock() + conn.get.return_value = { + "Actions": { + "LogService.CollectDiagnosticData": { + "OEMDiagnosticDataType@Redfish.AllowableValues": ["Dmesg", "AllLogs"], + } + } + } + result = get_oem_diagnostic_allowable_values( + conn, "redfish/v1/Systems/1/LogServices/DiagLogs" + ) + assert result == ["Dmesg", "AllLogs"] + + def test_returns_list_from_octothorpe_action_key(self): + conn = MagicMock() + conn.get.return_value = { + "Actions": { + "#LogService.CollectDiagnosticData": { + "OEMDiagnosticDataType@Redfish.AllowableValues": ["JournalControl"], + } + } + } + result = get_oem_diagnostic_allowable_values( + conn, "redfish/v1/Systems/UBB/LogServices/DiagLogs" + ) + assert result == ["JournalControl"] + + def test_returns_none_on_connection_error(self): + conn = MagicMock() + conn.get.side_effect = RedfishConnectionError("fail") + result = get_oem_diagnostic_allowable_values(conn, "redfish/v1/LogServices/DiagLogs") + assert result is None + + def test_returns_none_when_data_not_dict(self): + conn = MagicMock() + conn.get.return_value = [] + result = get_oem_diagnostic_allowable_values(conn, "redfish/v1/LogServices/DiagLogs") + assert result is None + + def test_returns_none_when_no_actions(self): + conn = MagicMock() + conn.get.return_value = {} + result = get_oem_diagnostic_allowable_values(conn, "redfish/v1/LogServices/DiagLogs") + assert result is None + + +class TestStripPortFromUrl: + def test_strips_port_443(self): + url = "https://host:443/redfish/v1/TaskService/Tasks/1" + assert _strip_port_from_url(url) == "https://host/redfish/v1/TaskService/Tasks/1" + + def test_strips_other_port(self): + url = "https://host:8443/redfish/v1" + assert _strip_port_from_url(url) == "https://host/redfish/v1" + + def test_returns_none_when_no_port(self): + url = "https://host/redfish/v1" + assert _strip_port_from_url(url) is None + + def test_returns_none_for_relative_path(self): + assert _strip_port_from_url("redfish/v1/Systems/1") is None + + +class TestGetTaskMonitorUri: + def test_returns_task_monitor_from_body(self): + conn = MagicMock() + conn.base_url = "https://host/redfish/v1" + body = {"TaskMonitor": "TaskService/Tasks/1/Monitor"} + result = _get_task_monitor_uri(body, conn) + assert result == "https://host/redfish/v1/TaskService/Tasks/1/Monitor" + + def test_returns_from_odata_id_plus_monitor(self): + conn = MagicMock() + conn.base_url = "https://host/redfish/v1" + body = {"@odata.id": "TaskService/Tasks/1"} + result = _get_task_monitor_uri(body, conn) + assert result == "https://host/redfish/v1/TaskService/Tasks/1/Monitor" + + def test_returns_none_for_empty_body(self): + conn = MagicMock() + assert _get_task_monitor_uri({}, conn) is None + + def test_prefers_task_monitor_over_odata_id(self): + conn = MagicMock() + conn.base_url = "https://host/redfish/v1" + body = { + "TaskMonitor": "TaskService/Tasks/1/Monitor", + "@odata.id": "TaskService/Tasks/2", + } + result = _get_task_monitor_uri(body, conn) + assert result == "https://host/redfish/v1/TaskService/Tasks/1/Monitor" + + +class TestDownloadLogAndSave: + def test_returns_none_when_no_additional_data_uri(self): + conn = MagicMock() + log_entry_json = {"Id": "1", "Name": "LogEntry"} + result = _download_log_and_save( + conn, log_entry_json, "Dmesg", None, logging.getLogger("test") + ) + assert result is None + conn.get_response.assert_not_called() + + def test_downloads_and_returns_bytes_when_additional_data_uri_present(self): + conn = MagicMock() + conn.base_url = "https://host/redfish/v1" + resp = MagicMock() + resp.status_code = codes.ok + resp.content = b"log bytes" + conn.get_response.return_value = resp + log_entry_json = {"AdditionalDataURI": "/redfish/v1/LogServices/1/Entries/1/Attachment"} + result = _download_log_and_save( + conn, log_entry_json, "Dmesg", None, logging.getLogger("test") + ) + assert result == b"log bytes" + conn.get_response.assert_called_once() + + def test_writes_archive_and_metadata_to_output_dir(self, tmp_path): + conn = MagicMock() + conn.base_url = "https://host/redfish/v1" + resp = MagicMock() + resp.status_code = codes.ok + resp.content = b"log bytes" + conn.get_response.return_value = resp + log_entry_json = { + "AdditionalDataURI": "/redfish/v1/LogServices/1/Entries/1/Attachment", + "Id": "1", + } + result = _download_log_and_save( + conn, log_entry_json, "AllLogs", tmp_path, logging.getLogger("test") + ) + assert result == b"log bytes" + assert (tmp_path / "AllLogs.tar.xz").read_bytes() == b"log bytes" + metadata = (tmp_path / "AllLogs_log_entry.json").read_text(encoding="utf-8") + assert "Id" in metadata and "1" in metadata + + +def test_collect_manager_diagnostic_payload(): + conn = MagicMock() + resp = MagicMock() + resp.status_code = 500 + resp.text = "fail" + conn.post.return_value = resp + collect_oem_diagnostic_data( + conn, + "redfish/v1/Managers/dummy-amc/LogServices/Dump", + diagnostic_data_type="Manager", + ) + payload = conn.post.call_args.kwargs["json"] + assert payload == {"DiagnosticDataType": "Manager"} + + +def test_task_resource_path_accepts_tasks_not_monitors(): + assert ( + _task_resource_path("/redfish/v1/TaskService/Tasks/dummy-1") + == "redfish/v1/TaskService/Tasks/dummy-1" + ) + assert _task_resource_path("redfish/v1/TaskService/TaskMonitors/1") is None + + +def test_collect_polls_task_resource_until_completed(): + conn = MagicMock() + conn.base_url = "https://bmc.example.test" + post_resp = MagicMock() + post_resp.status_code = codes.accepted + post_resp.headers = {"Location": "/redfish/v1/TaskService/TaskMonitors/1"} + post_resp.text = "" + post_resp.json.return_value = { + "@odata.id": "/redfish/v1/TaskService/Tasks/dummy-1", + "TaskState": "Running", + } + conn.post.return_value = post_resp + + running = MagicMock() + running.status_code = codes.ok + running.json.return_value = {"TaskState": "Running"} + done = MagicMock() + done.status_code = codes.ok + done.json.return_value = { + "TaskState": "Completed", + "Payload": { + "HttpHeaders": [ + "Location: /redfish/v1/Systems/dummy-system/LogServices/DiagLogs/Entries/1" + ] + }, + } + entry = MagicMock() + entry.status_code = codes.ok + entry.json.return_value = { + "Id": "1", + "AdditionalDataURI": ( + "/redfish/v1/Systems/dummy-system/LogServices/DiagLogs/Entries/1/attachment" + ), + } + attachment = MagicMock() + attachment.status_code = codes.ok + attachment.content = b"archive" + conn.get_response.side_effect = [running, done, entry, attachment] + + with patch("nodescraper.connection.redfish.redfish_oem_diag.time.sleep"): + log_bytes, metadata, err = collect_oem_diagnostic_data( + conn, + "redfish/v1/Systems/dummy-system/LogServices/DiagLogs", + oem_diagnostic_type="AllLogs", + task_timeout_s=30, + ) + assert err is None + assert log_bytes == b"archive" + assert metadata is not None + assert metadata["Id"] == "1" + polled = [c.args[0] for c in conn.get_response.call_args_list] + assert polled[0] == "redfish/v1/TaskService/Tasks/dummy-1" + assert polled[1] == "redfish/v1/TaskService/Tasks/dummy-1" + + +def test_collect_ssh_failure_during_poll_returns_error(): + conn = MagicMock() + conn.base_url = "https://bmc.example.test" + post_resp = MagicMock() + post_resp.status_code = codes.accepted + post_resp.headers = {"Location": "/redfish/v1/TaskService/TaskMonitors/1"} + post_resp.text = "" + post_resp.json.return_value = { + "@odata.id": "/redfish/v1/TaskService/Tasks/dummy-1", + "TaskState": "Running", + } + conn.post.return_value = post_resp + conn.get_response.side_effect = RedfishConnectionError("Timeout opening channel.") + + _log_bytes, _metadata, err = collect_oem_diagnostic_data( + conn, + "redfish/v1/Systems/dummy-system/LogServices/DiagLogs", + oem_diagnostic_type="AllLogs", + task_timeout_s=30, + ) + assert err == "Timeout opening channel." + + +def test_collect_taskmonitor_404_does_not_spin(): + conn = MagicMock() + conn.base_url = "https://bmc.example.test" + post_resp = MagicMock() + post_resp.status_code = codes.accepted + post_resp.headers = {"Location": "/redfish/v1/TaskService/TaskMonitors/1"} + post_resp.text = "" + post_resp.json.return_value = {} + conn.post.return_value = post_resp + missing = MagicMock() + missing.status_code = codes.not_found + conn.get_response.return_value = missing + + _log_bytes, _metadata, err = collect_oem_diagnostic_data( + conn, + "redfish/v1/Systems/dummy-system/LogServices/DiagLogs", + oem_diagnostic_type="AllLogs", + task_timeout_s=30, + ) + assert err is not None + assert "404" in err + assert conn.get_response.call_count == 1 + + +class TestCollectOemDiagnosticDataRetryAfter: + @staticmethod + def _conn_for_retry_after(retry_after: str) -> MagicMock: + conn = MagicMock() + conn.base_url = "https://host" + post_resp = MagicMock() + post_resp.status_code = codes.accepted + post_resp.headers = { + "Location": "/redfish/v1/TaskService/TaskMonitors/1", + "Retry-After": retry_after, + } + post_resp.json.return_value = {} + conn.post.return_value = post_resp + + accepted = MagicMock() + accepted.status_code = codes.accepted + accepted.json.return_value = {} + monitor_resp = MagicMock() + monitor_resp.status_code = codes.ok + monitor_resp.json.return_value = {"@odata.id": "/redfish/v1/TaskService/Tasks/1"} + task_resp = MagicMock() + task_resp.status_code = codes.ok + task_resp.json.return_value = {"TaskState": "Completed", "Payload": {"HttpHeaders": []}} + conn.get_response.side_effect = [accepted, monitor_resp, task_resp] + return conn + + @patch("nodescraper.connection.redfish.redfish_oem_diag.time.sleep") + def test_numeric_retry_after_is_used_as_poll_interval(self, mock_sleep): + """A Retry-After in seconds drives the poll interval after a 202 monitor.""" + conn = self._conn_for_retry_after("3") + + _, _, error = collect_oem_diagnostic_data( + conn, "redfish/v1/Systems/UBB/LogServices/DiagLogs", "AllLogs" + ) + + assert error == "Location header missing in task Payload.HttpHeaders" + mock_sleep.assert_called_with(3) + + @patch("nodescraper.connection.redfish.redfish_oem_diag.time.sleep") + def test_http_date_retry_after_does_not_raise(self, mock_sleep): + """An HTTP-date Retry-After must be handled, not crash the collection.""" + conn = self._conn_for_retry_after("Fri, 31 Dec 1999 23:59:59 GMT") + + _, _, error = collect_oem_diagnostic_data( + conn, "redfish/v1/Systems/UBB/LogServices/DiagLogs", "AllLogs" + ) + + assert error == "Location header missing in task Payload.HttpHeaders" + mock_sleep.assert_called_with(1) + + +def test_entries_collection_path_from_dump_entry(): + assert ( + _entries_collection_path("redfish/v1/Managers/AMC/LogServices/Dump/Entries/29") + == "redfish/v1/Managers/AMC/LogServices/Dump/Entries" + ) + + +def test_pick_member_href_prefers_wanted_id(): + body = { + "Members": [ + {"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/28"}, + {"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/29"}, + ] + } + assert _pick_member_href(body, "29") == "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/29" + assert _pick_member_href(body, "31") is None + assert _pick_member_href({"Members": []}, "29") is None + + +def _dump_collect_conn(get_side_effect): + conn = MagicMock() + conn.base_url = "http://192.0.2.1" + post_resp = MagicMock() + post_resp.status_code = codes.accepted + post_resp.headers = {"Location": "/redfish/v1/TaskService/TaskMonitors/1"} + post_resp.text = "" + post_resp.json.return_value = { + "@odata.id": "/redfish/v1/TaskService/Tasks/dummy-1", + "TaskState": "Running", + } + conn.post.return_value = post_resp + conn.get_response.side_effect = get_side_effect + return conn + + +def _ok_json(payload): + resp = MagicMock() + resp.status_code = codes.ok + resp.json.return_value = payload + return resp + + +def _status(code): + resp = MagicMock() + resp.status_code = code + resp.json.return_value = {} + return resp + + +@patch("nodescraper.connection.redfish.redfish_oem_diag.time.sleep") +def test_collect_retries_log_entry_404(mock_sleep): + """Dump LogEntry 404s are retried before failing.""" + running = _ok_json({"TaskState": "Running"}) + done = _ok_json( + { + "TaskState": "Completed", + "Payload": { + "HttpHeaders": ["Location: /redfish/v1/Managers/AMC/LogServices/Dump/Entries/29"] + }, + } + ) + entry = _ok_json( + { + "Id": "29", + "AdditionalDataURI": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/29/attachment", + } + ) + attachment = MagicMock() + attachment.status_code = codes.ok + attachment.content = b"mgr-dump" + conn = _dump_collect_conn([running, done, _status(codes.not_found), entry, attachment]) + + log_bytes, metadata, err = collect_oem_diagnostic_data( + conn, + "redfish/v1/Managers/AMC/LogServices/Dump", + diagnostic_data_type="Manager", + task_timeout_s=30, + ) + + assert err is None + assert log_bytes == b"mgr-dump" + assert metadata["Id"] == "29" + mock_sleep.assert_called() + + +@patch("nodescraper.connection.redfish.redfish_oem_diag.time.sleep") +def test_collect_log_entry_falls_back_to_entries_collection(mock_sleep): + """After LogEntry 404s, list Dump/Entries and GET the matching member.""" + running = _ok_json({"TaskState": "Running"}) + done = _ok_json( + { + "TaskState": "Completed", + "Payload": { + "HttpHeaders": ["Location: /redfish/v1/Managers/AMC/LogServices/Dump/Entries/29"] + }, + } + ) + missing = _status(codes.not_found) + coll = _ok_json( + { + "Members": [ + {"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/28"}, + {"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/29"}, + ] + } + ) + member = _ok_json( + { + "Id": "29", + "AdditionalDataURI": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/29/attachment", + } + ) + attachment = MagicMock() + attachment.status_code = codes.ok + attachment.content = b"mgr-dump" + conn = _dump_collect_conn( + [running, done, missing, missing, missing, missing, coll, member, attachment] + ) + + log_bytes, metadata, err = collect_oem_diagnostic_data( + conn, + "redfish/v1/Managers/AMC/LogServices/Dump", + diagnostic_data_type="Manager", + task_timeout_s=30, + ) + + assert err is None + assert log_bytes == b"mgr-dump" + assert metadata["Id"] == "29" + listed = [c.args[0] for c in conn.get_response.call_args_list] + assert "redfish/v1/Managers/AMC/LogServices/Dump/Entries" in listed + + +@patch("nodescraper.connection.redfish.redfish_oem_diag.time.sleep") +def test_collect_log_entry_collection_miss_keeps_entry_status(mock_sleep): + """Collection 200 without the requested Id must not report status 200.""" + running = _ok_json({"TaskState": "Running"}) + done = _ok_json( + { + "TaskState": "Completed", + "Payload": { + "HttpHeaders": ["Location: /redfish/v1/Managers/AMC/LogServices/Dump/Entries/31"] + }, + } + ) + missing = _status(codes.not_found) + coll = _ok_json( + { + "Members": [ + {"@odata.id": "/redfish/v1/Managers/AMC/LogServices/Dump/Entries/30"}, + ] + } + ) + conn = _dump_collect_conn([running, done, missing, missing, missing, missing, coll]) + + log_bytes, metadata, err = collect_oem_diagnostic_data( + conn, + "redfish/v1/Managers/AMC/LogServices/Dump", + diagnostic_data_type="Manager", + task_timeout_s=30, + ) + + assert log_bytes is None + assert metadata is None + assert err is not None + assert "status 200" not in err + assert "status 404" in err + assert "Id 31 not in redfish/v1/Managers/AMC/LogServices/Dump/Entries" in err diff --git a/test/unit/connection/redfish/test_redfish_session_refresh.py b/test/unit/connection/redfish/test_redfish_session_refresh.py new file mode 100644 index 00000000..a32fff77 --- /dev/null +++ b/test/unit/connection/redfish/test_redfish_session_refresh.py @@ -0,0 +1,134 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock, patch + +import pytest + +from nodescraper.connection.redfish import RedfishConnection, RedfishConnectionError + + +def _response(status_code: int) -> MagicMock: + response = MagicMock() + response.status_code = status_code + response.ok = status_code < 400 + return response + + +@pytest.fixture +def session_conn() -> RedfishConnection: + return RedfishConnection( + base_url="https://bmc.example", + username="root", + password="pass", + use_session_auth=True, + verify_ssl=False, + ) + + +def test_get_response_refreshes_session_after_401(session_conn: RedfishConnection) -> None: + stale = MagicMock() + fresh = MagicMock() + stale.get.return_value = _response(401) + fresh.get.return_value = _response(200) + sessions = [stale, fresh] + + def ensure_session() -> MagicMock: + session_conn._session = sessions.pop(0) + return session_conn._session + + with patch.object(session_conn, "_ensure_session", side_effect=ensure_session): + response = session_conn.get_response("/redfish/v1/Systems") + + assert response.status_code == 200 + stale.get.assert_called_once() + fresh.get.assert_called_once() + assert session_conn._session is fresh + + +def test_get_response_returns_second_401_after_refresh(session_conn: RedfishConnection) -> None: + stale = MagicMock() + fresh = MagicMock() + stale.get.return_value = _response(401) + second = _response(401) + fresh.get.return_value = second + sessions = [stale, fresh] + + def ensure_session() -> MagicMock: + session_conn._session = sessions.pop(0) + return session_conn._session + + with patch.object(session_conn, "_ensure_session", side_effect=ensure_session): + response = session_conn.get_response("/redfish/v1/Systems") + + assert response is second + assert fresh.get.call_count == 1 + + +def test_get_response_does_not_refresh_basic_auth() -> None: + conn = RedfishConnection( + base_url="https://bmc.example", + username="root", + password="pass", + use_session_auth=False, + verify_ssl=False, + ) + session = MagicMock() + session.get.return_value = _response(401) + + with patch.object(conn, "_ensure_session", return_value=session) as ensure_session: + response = conn.get_response("/redfish/v1/Systems") + + assert response.status_code == 401 + ensure_session.assert_called_once() + session.get.assert_called_once() + + +def test_get_response_does_not_refresh_other_status(session_conn: RedfishConnection) -> None: + session = MagicMock() + session.get.return_value = _response(404) + + with patch.object(session_conn, "_ensure_session", return_value=session) as ensure_session: + response = session_conn.get_response("/redfish/v1/Systems") + + assert response.status_code == 404 + ensure_session.assert_called_once() + session.get.assert_called_once() + + +def test_get_response_propagates_failed_relogin(session_conn: RedfishConnection) -> None: + stale = MagicMock() + stale.get.return_value = _response(401) + + def ensure_session() -> MagicMock: + session = session_conn._session + if not isinstance(session, MagicMock): + raise RedfishConnectionError("Session login failed: 401 Unauthorized") + return session + + session_conn._session = stale + with patch.object(session_conn, "_ensure_session", side_effect=ensure_session): + with pytest.raises(RedfishConnectionError, match="Session login failed"): + session_conn.get_response("/redfish/v1/Systems") diff --git a/test/unit/connection/redfish/test_redfish_transport_errors.py b/test/unit/connection/redfish/test_redfish_transport_errors.py new file mode 100644 index 00000000..df464660 --- /dev/null +++ b/test/unit/connection/redfish/test_redfish_transport_errors.py @@ -0,0 +1,118 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import socket +from unittest.mock import MagicMock, patch + +import pytest +import requests +from pydantic import SecretStr + +from nodescraper.connection.redfish import RedfishConnection, RedfishConnectionError +from nodescraper.connection.redfish.redfish_manager import RedfishConnectionManager +from nodescraper.connection.redfish.redfish_params import RedfishConnectionParams +from nodescraper.enums import ExecutionStatus, SystemLocation +from nodescraper.models import SystemInfo + +_TEST_HOST = "10.6.189.112" + + +@pytest.fixture +def rf_conn() -> RedfishConnection: + return RedfishConnection( + base_url=f"https://{_TEST_HOST}", + username="user", + password="pass", + use_session_auth=False, + ) + + +def test_get_response_wraps_connect_timeout(rf_conn: RedfishConnection) -> None: + with patch.object(rf_conn, "_ensure_session") as ensure_session: + session = MagicMock() + session.get.side_effect = requests.exceptions.ConnectTimeout("timed out") + ensure_session.return_value = session + + with pytest.raises(RedfishConnectionError, match="timed out"): + rf_conn.get_response("redfish/v1") + + +def test_get_response_wraps_connection_refused(rf_conn: RedfishConnection) -> None: + root = ConnectionRefusedError(111, "Connection refused") + wrapped = requests.exceptions.ConnectionError("failed", response=None) + wrapped.__cause__ = root + + with patch.object(rf_conn, "_ensure_session") as ensure_session: + session = MagicMock() + session.get.side_effect = wrapped + ensure_session.return_value = session + + with pytest.raises(RedfishConnectionError, match="connection failed"): + rf_conn.get_response("redfish/v1") + + +def test_get_response_wraps_name_resolution_error(rf_conn: RedfishConnection) -> None: + dns_error = requests.exceptions.ConnectionError( + f"HTTPSConnectionPool(host='{_TEST_HOST}', port=443): " + f"Failed to resolve '{_TEST_HOST}' " + "([Errno -5] No address associated with hostname)" + ) + dns_error.__cause__ = socket.gaierror(-5, "No address associated with hostname") + + with patch.object(rf_conn, "_ensure_session") as ensure_session: + session = MagicMock() + session.get.side_effect = dns_error + ensure_session.return_value = session + + with pytest.raises(RedfishConnectionError, match="hostname could not be resolved"): + rf_conn.get_response("redfish/v1") + + +def test_redfish_manager_connect_logs_clean_timeout() -> None: + manager = RedfishConnectionManager( + system_info=SystemInfo(name="test", location=SystemLocation.LOCAL), + connection_args=RedfishConnectionParams( + host=_TEST_HOST, + username="user", + password=SecretStr("pass"), + use_https=True, + use_session_auth=False, + ), + ) + with patch( + "nodescraper.connection.redfish.redfish_manager.RedfishConnection" + ) as connection_cls: + connection = connection_cls.return_value + connection.get_service_root.side_effect = RedfishConnectionError( + f"Redfish connection timed out: {_TEST_HOST}" + ) + + result = manager.connect() + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert len(result.events) == 1 + assert "timed out" in result.events[0].description + assert "traceback" not in result.events[0].data + assert "exception_type" not in result.events[0].data diff --git a/test/unit/connection/redfish/test_ssh_proxy_connection.py b/test/unit/connection/redfish/test_ssh_proxy_connection.py new file mode 100644 index 00000000..bbe75c6a --- /dev/null +++ b/test/unit/connection/redfish/test_ssh_proxy_connection.py @@ -0,0 +1,339 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import shlex +from typing import Optional +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import ValidationError + +from nodescraper.connection.inband.inband import CommandArtifact +from nodescraper.connection.inband.inbandremote import SSHConnectionError +from nodescraper.connection.inband.sshparams import SSHConnectionParams +from nodescraper.connection.redfish.redfish_connection import RedfishConnectionError +from nodescraper.connection.redfish.redfish_manager import MultiTargetRedfishConnection +from nodescraper.connection.redfish.ssh_proxy_connection import ( + CurlResponse, + SshProxyRedfishConnection, + parse_curl_headers, + split_curl_stdout, +) +from nodescraper.connection.redfish.ssh_proxy_manager import ( + RedfishSshProxyConnectionManager, +) +from nodescraper.connection.redfish.ssh_proxy_params import ( + RedfishSshProxyConnectionParams, +) +from nodescraper.enums import ExecutionStatus + + +class FakeRemoteShell: + def __init__(self, status=200, headers="", body=b"{}", fail_times=0, reconnect_error=None): + self.status = status + self.headers = headers + self.body = body + self.commands = [] + self.client = MagicMock() + self.fail_times = fail_times + self.reconnect_error = reconnect_error + self.reconnects = 0 + self._curls = 0 + self._proxy_ssh_down = False + + def run_command(self, command, sudo=False, timeout=30, strip=True): + self.commands.append(command) + if command.startswith("curl"): + self._curls += 1 + if self._curls <= self.fail_times: + return CommandArtifact( + command=command, + stdout="", + stderr="Server connection dropped:", + exit_code=124, + ) + parts = shlex.split(command) + trailer = parts[parts.index("-w") + 1].replace("%{http_code}", str(self.status)) + stdout = f"{self.headers}{self.body.decode('utf-8')}{trailer}" + return CommandArtifact(command=command, stdout=stdout, stderr="", exit_code=0) + return CommandArtifact(command=command, stdout="", stderr="unexpected", exit_code=1) + + def reconnect_ssh(self): + self.reconnects += 1 + if self.reconnect_error is not None: + raise self.reconnect_error + + +def test_parse_curl_headers_last_hop_wins(): + text = ( + "HTTP/1.1 301 Moved\n" + "Location: /old\n" + "\n" + "HTTP/1.1 200 OK\n" + "Location: /redfish/v1/TaskService/Tasks/1\n" + "Content-Type: application/json\n" + "\n" + ) + headers = parse_curl_headers(text) + assert headers["Location"] == "/redfish/v1/TaskService/Tasks/1" + assert headers["Content-Type"] == "application/json" + + +def test_curl_response_json_and_ok(): + resp = CurlResponse(200, b'{"Id": "root"}') + assert resp.ok + assert resp.json() == {"Id": "root"} + assert resp.reason == "OK" + + +def test_ssh_proxy_get_uses_quoted_internal_url(): + shell = FakeRemoteShell( + headers="HTTP/1.1 200 OK\nContent-Type: application/json\n\n", + body=b'{"RedfishVersion": "1.15.0"}', + ) + conn = SshProxyRedfishConnection(shell, "http://192.0.2.10:80", api_root="redfish/v1") + data = conn.get_service_root() + assert data["RedfishVersion"] == "1.15.0" + curl_cmds = [c for c in shell.commands if c.startswith("curl")] + assert len(curl_cmds) == 1 + assert "http://192.0.2.10:80/redfish/v1" in curl_cmds[0] + assert "-D -" in curl_cmds[0] + assert "-o -" in curl_cmds[0] + assert all(not command.startswith("mktemp") for command in shell.commands) + assert shell.reconnects == 0 + + +def test_split_curl_stdout_keeps_crlf_body(): + marker = "NODESCRAPERdeadbeef" + raw = ( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n" + '{"Id": "card"}\n' + f"{marker}200\n" + ) + status, headers, body = split_curl_stdout(raw, marker) + assert status == 200 + assert "Content-Type: application/json" in headers + assert body == b'{"Id": "card"}' + + +def test_ssh_proxy_reconnects_once_after_dropped_session(): + shell = FakeRemoteShell( + headers="HTTP/1.1 200 OK\nContent-Type: application/json\n\n", + body=b'{"RedfishVersion": "1.15.0"}', + fail_times=1, + ) + conn = SshProxyRedfishConnection(shell, "http://192.0.2.10:80", api_root="redfish/v1") + data = conn.get_service_root() + assert data["RedfishVersion"] == "1.15.0" + assert shell.reconnects == 1 + assert shell._curls == 2 + + +def test_ssh_proxy_does_not_retry_after_the_second_drop(): + shell = FakeRemoteShell(fail_times=5) + conn = SshProxyRedfishConnection(shell, "http://192.0.2.10:80", api_root="redfish/v1") + with pytest.raises(RedfishConnectionError, match="Server connection dropped"): + conn.get_service_root() + with pytest.raises(RedfishConnectionError, match="SSH session not active"): + conn.get_service_root() + assert shell.reconnects == 1 + assert shell._curls == 2 + + +def test_ssh_proxy_reports_reconnect_failure(): + shell = FakeRemoteShell(fail_times=1, reconnect_error=SSHConnectionError("SSH Request timeout")) + conn = SshProxyRedfishConnection(shell, "http://192.0.2.10:80", api_root="redfish/v1") + with pytest.raises(RedfishConnectionError, match="SSH proxy reconnect failed"): + conn.get_service_root() + assert shell.reconnects == 1 + assert shell._curls == 1 + + +def test_ssh_proxy_curl_timeout_does_not_reconnect(): + shell = FakeRemoteShell() + + def timed_out(command, sudo=False, timeout=30, strip=True): + shell.commands.append(command) + return CommandArtifact( + command=command, stdout="", stderr="Command timed out", exit_code=124 + ) + + shell.run_command = timed_out + conn = SshProxyRedfishConnection(shell, "http://192.0.2.10:80", api_root="redfish/v1") + with pytest.raises(RedfishConnectionError, match="Command timed out"): + conn.get_service_root() + assert shell.reconnects == 0 + + +def test_ssh_proxy_post_includes_json_body(): + shell = FakeRemoteShell(body=b'{"TaskState": "Running"}') + conn = SshProxyRedfishConnection(shell, "http://192.0.2.10:80") + resp = conn.post( + "redfish/v1/Managers/dummy-amc/LogServices/Dump/Actions/LogService.CollectDiagnosticData", + json={"DiagnosticDataType": "Manager"}, + ) + assert resp.status_code == 200 + curl_cmds = [c for c in shell.commands if c.startswith("curl")] + assert "-X POST" in curl_cmds[0] + assert "DiagnosticDataType" in curl_cmds[0] + + +def test_ssh_proxy_manager_connect_success(system_info): + shell = FakeRemoteShell(body=b'{"RedfishVersion": "1.15.0"}') + shell.connect_ssh = MagicMock() + params = { + "host": "192.0.2.10", + "port": 80, + "use_https": False, + "ssh": { + "hostname": "bmc.example.test", + "username": "testuser", + "key_filename": "/tmp/dummy_id", + }, + } + mgr = RedfishSshProxyConnectionManager(system_info=system_info, connection_args=params) + with patch( + "nodescraper.connection.redfish.ssh_proxy_manager.RemoteShell", + return_value=shell, + ): + result = mgr.connect() + assert result.status in (ExecutionStatus.UNSET, ExecutionStatus.OK) + assert mgr.connection is not None + + +def test_ssh_proxy_params_require_ssh(): + with pytest.raises(ValidationError): + RedfishSshProxyConnectionParams(host="192.0.2.10") + + +def _proxy_target( + target_key: Optional[str], amc_host: str, ssh_host: str +) -> RedfishSshProxyConnectionParams: + return RedfishSshProxyConnectionParams( + target_key=target_key, + host=amc_host, + ssh=SSHConnectionParams( + hostname=ssh_host, username="testuser", key_filename="/tmp/dummy_id" + ), + ) + + +def test_ssh_proxy_partial_connect_keeps_reachable_bmc(system_info): + """One unreachable BMC leaves the reachable BMC connected at warning.""" + shells: list[MagicMock] = [] + + def factory(_params): + shell = MagicMock() + shells.append(shell) + return shell + + def service_root(conn): + if "amc-down" in conn.base_url: + raise RedfishConnectionError("unreachable") + return {"RedfishVersion": "1.0"} + + params = RedfishSshProxyConnectionParams( + max_workers=2, + targets=[ + _proxy_target("good", "192.0.2.10", "bmc-good.example"), + _proxy_target("bad", "amc-down.example", "bmc-down.example"), + ], + ) + mgr = RedfishSshProxyConnectionManager(system_info=system_info, connection_args=params) + with ( + patch( + "nodescraper.connection.redfish.ssh_proxy_manager.RemoteShell", + side_effect=factory, + ), + patch.object(SshProxyRedfishConnection, "get_service_root", service_root), + ): + result = mgr.connect() + + assert result.status == ExecutionStatus.WARNING + assert isinstance(mgr.connection, MultiTargetRedfishConnection) + assert set(mgr.connection.target_connections) == {"good"} + assert "bad" in mgr.connection.failed_targets + assert mgr.connection.max_workers == 2 + assert shells[1].client.close.called + assert not shells[0].client.close.called + + mgr.connection.multi_target_data["good"] = {"label": "ok"} + mgr.disconnect() + assert mgr.connection is None + assert mgr._multi_target_data == {"good": {"label": "ok"}} + assert shells[0].client.close.called + + +def test_ssh_proxy_every_target_down_fails_connect(system_info): + """Every target failing is an execution failure and does not set a connection.""" + + def service_root(_conn): + raise RedfishConnectionError("unreachable") + + params = RedfishSshProxyConnectionParams( + targets=[ + _proxy_target("a", "amc-down-a.example", "bmc-a.example"), + _proxy_target("b", "amc-down-b.example", "bmc-b.example"), + ], + ) + mgr = RedfishSshProxyConnectionManager(system_info=system_info, connection_args=params) + with ( + patch( + "nodescraper.connection.redfish.ssh_proxy_manager.RemoteShell", + return_value=MagicMock(), + ), + patch.object(SshProxyRedfishConnection, "get_service_root", service_root), + ): + result = mgr.connect() + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert mgr.connection is None + + +def test_ssh_proxy_target_key_defaults_to_ssh_hostname(system_info): + """Targets that share an AMC address stay distinct by SSH hostname.""" + + def service_root(_conn): + return {"RedfishVersion": "1.0"} + + params = RedfishSshProxyConnectionParams( + targets=[ + _proxy_target(None, "192.0.2.10", "bmc-one.example"), + _proxy_target(None, "192.0.2.10", "bmc-two.example"), + ], + ) + mgr = RedfishSshProxyConnectionManager(system_info=system_info, connection_args=params) + with ( + patch( + "nodescraper.connection.redfish.ssh_proxy_manager.RemoteShell", + return_value=MagicMock(), + ), + patch.object(SshProxyRedfishConnection, "get_service_root", service_root), + ): + result = mgr.connect() + + assert result.status in (ExecutionStatus.UNSET, ExecutionStatus.OK) + assert isinstance(mgr.connection, MultiTargetRedfishConnection) + assert set(mgr.connection.target_connections) == {"bmc-one.example", "bmc-two.example"} diff --git a/test/unit/connection/test_connectionmanager.py b/test/unit/connection/test_connectionmanager.py new file mode 100644 index 00000000..def18b4b --- /dev/null +++ b/test/unit/connection/test_connectionmanager.py @@ -0,0 +1,332 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import unittest +from unittest.mock import MagicMock + +from nodescraper.connection.inband.inbandmanager import InBandConnectionManager +from nodescraper.connection.inband.inbandremote import RemoteShell +from nodescraper.connection.inband.sshparams import SSHConnectionParams +from nodescraper.connection.oob_ssh.oob_ssh_connection_manager import ( + OobSshConnectionManager, +) +from nodescraper.connection.redfish.redfish_connection import RedfishConnection +from nodescraper.connection.redfish.redfish_manager import RedfishConnectionManager +from nodescraper.connection.redfish.redfish_params import RedfishConnectionParams +from nodescraper.connection.redfish.ssh_proxy_manager import ( + RedfishSshProxyConnectionManager, +) +from nodescraper.connection.redfish.ssh_proxy_params import ( + RedfishSshProxyConnectionParams, +) +from nodescraper.enums import ExecutionStatus, SystemLocation +from nodescraper.interfaces.connectionmanager import ConnectionManager +from nodescraper.models import SystemInfo, TaskResult + + +class DummyConnection: + """Simple test connection class for testing ConnectionManager base behavior""" + + pass + + +class DummyConnectionManager(ConnectionManager[DummyConnection, None]): + """Concrete implementation of ConnectionManager for testing base class behavior""" + + CONNECTION_ARGS_MODEL = None + + def connect(self) -> TaskResult: + """Minimal connect implementation for testing""" + self.connection = DummyConnection() + return self.result + + +class TestConnectionManagerBase(unittest.TestCase): + """Test suite for ConnectionManager base behavior""" + + def test_disconnect_clears_connection(self): + """Test that disconnect clears the connection attribute""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.LOCAL, + ) + manager = DummyConnectionManager(system_info=system_info) + manager.connection = DummyConnection() + + manager.disconnect() + + self.assertIsNone(manager.connection) + + def test_disconnect_preserves_status(self): + """Test that disconnect keeps the result of the connection attempt""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.LOCAL, + ) + manager = DummyConnectionManager(system_info=system_info) + manager.connection = DummyConnection() + manager.result.status = ExecutionStatus.OK + + manager.disconnect() + + self.assertEqual(manager.result.status, ExecutionStatus.OK) + + def test_connect_success_sets_ok_and_survives_disconnect(self): + manager = DummyConnectionManager( + system_info=SystemInfo(name="test_system", location=SystemLocation.LOCAL) + ) + + result = manager.connect() + manager.disconnect() + + self.assertEqual(result.status, ExecutionStatus.OK) + self.assertEqual(manager.result.status, ExecutionStatus.OK) + + def test_connect_exception_sets_failure_with_event(self): + class FailingManager(DummyConnectionManager): + def connect(self) -> TaskResult: + raise RuntimeError("boom") + + manager = FailingManager( + system_info=SystemInfo(name="test_system", location=SystemLocation.LOCAL) + ) + + result = manager.connect() + manager.disconnect() + + self.assertEqual(result.status, ExecutionStatus.EXECUTION_FAILURE) + self.assertEqual(manager.result.status, ExecutionStatus.EXECUTION_FAILURE) + self.assertGreaterEqual(len(result.events), 1) + + def test_connect_without_connection_reports_failure_event(self): + class NoConnectionManager(DummyConnectionManager): + def connect(self) -> TaskResult: + return self.result + + manager = NoConnectionManager( + system_info=SystemInfo(name="test_system", location=SystemLocation.LOCAL) + ) + + result = manager.connect() + + self.assertEqual(result.status, ExecutionStatus.EXECUTION_FAILURE) + self.assertEqual(len(result.events), 1) + + def test_context_manager_calls_disconnect(self): + """Test that using connection manager as context manager calls disconnect on exit""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.LOCAL, + ) + manager = DummyConnectionManager(system_info=system_info) + + with manager: + manager.connection = DummyConnection() + self.assertIsNotNone(manager.connection) + + # After exiting context, disconnect should have been called + self.assertIsNone(manager.connection) + + def test_connect_decorator_initializes_result(self): + """Test that the connect decorator properly initializes the result""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.LOCAL, + ) + manager = DummyConnectionManager(system_info=system_info) + + result = manager.connect() + + self.assertIsNotNone(result) + self.assertIsInstance(manager.connection, DummyConnection) + + def test_context_manager_handles_disconnect_exception(self): + """BUG: __exit__ doesn't handle exceptions from disconnect()""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.LOCAL, + ) + manager = DummyConnectionManager(system_info=system_info) + + # Mock disconnect to raise an exception + def bad_disconnect(): + raise RuntimeError("Disconnect failed") + + manager.disconnect = bad_disconnect + + # If disconnect raises an exception in __exit__, it could mask original exceptions + with self.assertRaises(RuntimeError) as context: + with manager: + manager.connection = DummyConnection() + + self.assertIn("Disconnect failed", str(context.exception)) + + +class TestInBandConnectionManagerDisconnect(unittest.TestCase): + """Test suite specifically for InBandConnectionManager disconnect behavior""" + + def test_disconnect_remote_closes_ssh_client(self): + """Test that disconnect properly closes SSH client for remote connections""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.REMOTE, + ) + manager = InBandConnectionManager(system_info=system_info) + + # Mock a RemoteShell with a client + mock_client = MagicMock() + mock_remote_shell = MagicMock(spec=RemoteShell) + mock_remote_shell.client = mock_client + + manager.connection = mock_remote_shell + + manager.disconnect() + + # Verify client.close() was called + mock_client.close.assert_called_once() + # Verify connection was cleared by parent class + self.assertIsNone(manager.connection) + + +class TestRedfishConnectionManagerDisconnect(unittest.TestCase): + """Test suite for RedfishConnectionManager disconnect behavior""" + + def test_disconnect_closes_redfish_connection(self): + """Test that disconnect properly closes Redfish connection""" + system_info = SystemInfo( + name="test_system", + location=SystemLocation.REMOTE, + ) + + # Create manager without actual connection + manager = RedfishConnectionManager(system_info=system_info) + + # Mock the connection with a close method + mock_connection = MagicMock(spec=RedfishConnection) + + manager.connection = mock_connection + + manager.disconnect() + + # Verify connection.close() was called + mock_connection.close.assert_called_once() + # Verify connection was cleared by parent class + self.assertIsNone(manager.connection) + + +class TestConnectionArgsModelClassVar(unittest.TestCase): + """CONNECTION_ARGS_MODEL classvar drives dict -> model conversion of connection_args.""" + + def setUp(self): + self.system_info = SystemInfo(name="test_system", location=SystemLocation.REMOTE) + + def test_subclasses_declare_expected_models(self): + expected = { + InBandConnectionManager: SSHConnectionParams, + RedfishConnectionManager: RedfishConnectionParams, + RedfishSshProxyConnectionManager: RedfishSshProxyConnectionParams, + OobSshConnectionManager: RedfishConnectionParams, + } + for manager_cls, model in expected.items(): + with self.subTest(manager=manager_cls.__name__): + self.assertIs(manager_cls.CONNECTION_ARGS_MODEL, model) + + def test_classvar_not_set_on_instance(self): + manager = InBandConnectionManager(system_info=self.system_info) + self.assertNotIn("CONNECTION_ARGS_MODEL", vars(manager)) + + def test_dict_args_converted_to_model(self): + manager = InBandConnectionManager( + system_info=self.system_info, + connection_args={"hostname": "host1", "username": "user1"}, + ) + self.assertIsInstance(manager.connection_args, SSHConnectionParams) + self.assertEqual(manager.connection_args.hostname, "host1") + self.assertEqual(manager.connection_args.username, "user1") + + def test_redfish_dict_args_converted_to_model(self): + manager = RedfishConnectionManager( + system_info=self.system_info, + connection_args={"host": "bmc1", "username": "admin"}, + ) + self.assertIsInstance(manager.connection_args, RedfishConnectionParams) + self.assertEqual(manager.connection_args.host, "bmc1") + + def test_oob_ssh_dict_args_converted_to_model(self): + manager = OobSshConnectionManager( + system_info=self.system_info, + connection_args={"host": "bmc1", "username": "admin"}, + ) + self.assertIsInstance(manager.connection_args, RedfishConnectionParams) + + def test_ssh_proxy_dict_args_converted_to_model(self): + manager = RedfishSshProxyConnectionManager( + system_info=self.system_info, + connection_args={"host": "", "ssh": {"hostname": "", "username": ""}, "max_workers": 2}, + ) + self.assertIsInstance(manager.connection_args, RedfishSshProxyConnectionParams) + self.assertEqual(manager.connection_args.max_workers, 2) + + def test_model_instance_passed_through_unchanged(self): + params = SSHConnectionParams(hostname="host1", username="user1") + manager = InBandConnectionManager(system_info=self.system_info, connection_args=params) + self.assertIs(manager.connection_args, params) + + def test_none_args_stay_none(self): + manager = InBandConnectionManager(system_info=self.system_info) + self.assertIsNone(manager.connection_args) + + def test_custom_subclass_classvar_is_used(self): + class CustomManager(ConnectionManager[DummyConnection, SSHConnectionParams]): + CONNECTION_ARGS_MODEL = SSHConnectionParams + + def connect(self) -> TaskResult: + return self.result + + manager = CustomManager( + system_info=self.system_info, + connection_args={"hostname": "h", "username": "u"}, + ) + self.assertIsInstance(manager.connection_args, SSHConnectionParams) + + def test_invalid_dict_args_raise_validation_error(self): + with self.assertRaises(ValueError): + InBandConnectionManager(system_info=self.system_info, connection_args={"port": 22}) + + +if __name__ == "__main__": + unittest.main() + + +class TestConnectionManagerAnnotations(unittest.TestCase): + """Type annotations on the base connection manager must be resolvable.""" + + def test_init_type_hints_are_resolvable(self): + """get_type_hints must work on ConnectionManager.__init__ for introspection tooling""" + import typing + + hints = typing.get_type_hints(ConnectionManager.__init__) + + self.assertIn("task_result_hooks", hints) diff --git a/test/unit/connection/test_inbandmanager.py b/test/unit/connection/test_inbandmanager.py new file mode 100644 index 00000000..982b9c27 --- /dev/null +++ b/test/unit/connection/test_inbandmanager.py @@ -0,0 +1,170 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import unittest +from unittest.mock import MagicMock, patch + +from nodescraper.connection.inband.inbandlocal import LocalShell +from nodescraper.connection.inband.inbandmanager import InBandConnectionManager +from nodescraper.connection.inband.inbandremote import RemoteShell, SSHConnectionError +from nodescraper.connection.inband.sshparams import SSHConnectionParams +from nodescraper.enums import SystemLocation +from nodescraper.models import SystemInfo + + +class TestInBandConnectionManager(unittest.TestCase): + """Test suite for InBandConnectionManager""" + + def setUp(self): + """Set up test fixtures""" + self.system_info = SystemInfo( + name="test_system", + location=SystemLocation.REMOTE, + ) + + def test_connect_local(self): + """Test connecting to a local system""" + from nodescraper.enums import ExecutionStatus + + local_system_info = SystemInfo( + name="local_system", + location=SystemLocation.LOCAL, + ) + manager = InBandConnectionManager(system_info=local_system_info) + + result = manager.connect() + + self.assertIsInstance(manager.connection, LocalShell) + self.assertIn(result.status, [ExecutionStatus.OK, ExecutionStatus.UNSET]) + + @patch("nodescraper.connection.inband.inbandmanager.RemoteShell") + def test_connect_remote_success(self, mock_remote_shell_class): + """Test successful remote SSH connection""" + mock_remote_shell = MagicMock(spec=RemoteShell) + mock_remote_shell_class.return_value = mock_remote_shell + + ssh_params = SSHConnectionParams( + hostname="test.example.com", username="testuser", password="testpass" + ) + + manager = InBandConnectionManager(system_info=self.system_info, connection_args=ssh_params) + + _result = manager.connect() + + mock_remote_shell_class.assert_called_once_with(ssh_params) + mock_remote_shell.connect_ssh.assert_called_once() + self.assertEqual(manager.connection, mock_remote_shell) + + def test_connect_remote_no_credentials(self): + """Test remote connection without SSH credentials""" + from nodescraper.enums import ExecutionStatus + + manager = InBandConnectionManager(system_info=self.system_info) + + result = manager.connect() + + self.assertEqual(result.status, ExecutionStatus.EXECUTION_FAILURE) + + def test_disconnect_remote_calls_client_close(self): + """Test that disconnect calls client.close() for remote connections""" + mock_client = MagicMock() + mock_remote_shell = MagicMock(spec=RemoteShell) + mock_remote_shell.client = mock_client + + manager = InBandConnectionManager(system_info=self.system_info) + manager.connection = mock_remote_shell + + manager.disconnect() + + mock_client.close.assert_called_once() + + def test_disconnect_local_does_not_call_client_close(self): + """Test that disconnect does not call client.close() for local connections""" + mock_local_shell = MagicMock(spec=LocalShell) + + manager = InBandConnectionManager(system_info=self.system_info) + manager.connection = mock_local_shell + + # Should not raise an error even though LocalShell doesn't have a client attribute + manager.disconnect() + + # Verify that we didn't try to access client.close() + self.assertFalse(hasattr(mock_local_shell, "client")) + + def test_disconnect_remote_with_none_client_raises_error(self): + """BUG: disconnect fails when RemoteShell.client is None""" + mock_remote_shell = MagicMock(spec=RemoteShell) + mock_remote_shell.client = None + + manager = InBandConnectionManager(system_info=self.system_info) + manager.connection = mock_remote_shell + + # This should raise AttributeError because client is None + with self.assertRaises(AttributeError): + manager.disconnect() + + def test_disconnect_remote_when_client_close_raises_exception(self): + """BUG: if client.close() raises exception, super().disconnect() is never called""" + mock_client = MagicMock() + mock_client.close.side_effect = Exception("Connection already closed") + + mock_remote_shell = MagicMock(spec=RemoteShell) + mock_remote_shell.client = mock_client + + manager = InBandConnectionManager(system_info=self.system_info) + manager.connection = mock_remote_shell + + # This should raise the exception from close() + with self.assertRaises(Exception) as context: + manager.disconnect() + + self.assertIn("Connection already closed", str(context.exception)) + + # BUG: Connection is not cleared because super().disconnect() was never called + self.assertIsNotNone(manager.connection) + + @patch("nodescraper.connection.inband.inbandmanager.RemoteShell") + def test_connect_remote_failure_clears_connection(self, mock_remote_shell_class): + """A failed SSH connect must not leave an unusable connection object behind""" + mock_remote_shell = MagicMock(spec=RemoteShell) + mock_remote_shell.connect_ssh.side_effect = SSHConnectionError("SSH Authentication failed") + mock_remote_shell_class.return_value = mock_remote_shell + + manager = InBandConnectionManager( + system_info=self.system_info, + connection_args=SSHConnectionParams( + hostname="test.example.com", + username="testuser", + password="testpass", + ), + ) + + manager.connect() + + self.assertIsNone(manager.connection) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/connection/test_osdetection.py b/test/unit/connection/test_osdetection.py index d9092f04..7b53d18f 100644 --- a/test/unit/connection/test_osdetection.py +++ b/test/unit/connection/test_osdetection.py @@ -31,6 +31,7 @@ ARISTA_VERSION_CMD, DELL_VERSION_CMD, detect_network_os, + discover_and_write_os_family, parse_arista_version_output, parse_dell_sonic_version_output, ) @@ -177,7 +178,7 @@ def test_detect_network_os_falls_back_to_dell(conn_mock): assert conn_mock.run_command.call_count == 2 -def test_check_os_family_detects_arista_eos(system_info, conn_mock): +def test_discover_and_write_os_family_detects_arista_eos(system_info, conn_mock, logger): manager = InBandConnectionManager(system_info=system_info) manager.connection = conn_mock conn_mock.run_command.side_effect = [ @@ -185,18 +186,15 @@ def test_check_os_family_detects_arista_eos(system_info, conn_mock): DUMMY_ARISTA_VERSION_CMD_OK, ] - manager._check_os_family() + discover_and_write_os_family(manager, system_info, logger) assert system_info.os_family == OSFamily.EOS assert system_info.platform == "Arista EOS" assert system_info.metadata["os_version"] == DUMMY_ARISTA_VERSION["version"] assert system_info.metadata["device_model"] == DUMMY_ARISTA_VERSION["modelName"] - assert not any( - event.description == "Unable to determine SUT OS" for event in manager.result.events - ) -def test_check_os_family_detects_dell_sonic(system_info, conn_mock): +def test_discover_and_write_os_family_detects_dell_sonic(system_info, conn_mock, logger): system_info.os_family = OSFamily.UNKNOWN manager = InBandConnectionManager(system_info=system_info) manager.connection = conn_mock @@ -206,18 +204,17 @@ def test_check_os_family_detects_dell_sonic(system_info, conn_mock): DUMMY_DELL_VERSION_CMD_OK, ] - manager._check_os_family() + discover_and_write_os_family(manager, system_info, logger) assert system_info.os_family == OSFamily.SONIC assert system_info.platform == "Dell SONiC" assert system_info.metadata["os_version"] == "4.1.0-Enterprise" assert system_info.metadata["device_model"] == "DellEMC-S5248F-ON" - assert not any( - event.description == "Unable to determine SUT OS" for event in manager.result.events - ) -def test_check_os_family_still_warns_when_unknown(system_info, conn_mock): +def test_discover_and_write_os_family_leaves_unknown_when_undetected( + system_info, conn_mock, logger +): system_info.os_family = OSFamily.UNKNOWN manager = InBandConnectionManager(system_info=system_info) manager.connection = conn_mock @@ -227,21 +224,38 @@ def test_check_os_family_still_warns_when_unknown(system_info, conn_mock): DUMMY_DELL_VERSION_CMD_NON_DELL, ] - manager._check_os_family() + discover_and_write_os_family(manager, system_info, logger) assert system_info.os_family == OSFamily.UNKNOWN - assert any( - event.description == "Unable to determine SUT OS" and event.category == "UNKNOWN" - for event in manager.result.events - ) -def test_check_os_family_linux_skips_network_probes(system_info, conn_mock): +def test_discover_and_write_os_family_linux_skips_network_probes(system_info, conn_mock, logger): manager = InBandConnectionManager(system_info=system_info) manager.connection = conn_mock conn_mock.run_command.return_value = DUMMY_UNAME_LINUX - manager._check_os_family() + discover_and_write_os_family(manager, system_info, logger) assert system_info.os_family == OSFamily.LINUX conn_mock.run_command.assert_called_once_with("uname -s") + + +DUMMY_UNAME_ESXI = CommandArtifact( + command="uname -s", + stdout="VMkernel", + stderr="", + exit_code=0, +) + + +def test_discover_and_write_os_family_detects_esxi(system_info, conn_mock, logger): + """ESXi reports 'VMkernel' from `uname -s` (exit 0); classify it as ESXI and + skip the network-OS probes (same fast path as Linux).""" + manager = InBandConnectionManager(system_info=system_info) + manager.connection = conn_mock + conn_mock.run_command.return_value = DUMMY_UNAME_ESXI + + discover_and_write_os_family(manager, system_info, logger) + + assert system_info.os_family == OSFamily.ESXI + conn_mock.run_command.assert_called_once_with("uname -s") diff --git a/test/unit/framework/common/shared_utils.py b/test/unit/framework/common/shared_utils.py index 7ba16c16..0e795f87 100644 --- a/test/unit/framework/common/shared_utils.py +++ b/test/unit/framework/common/shared_utils.py @@ -36,6 +36,7 @@ class MockConnectionManager(ConnectionManager): # Class variable to store the mock connector mock_connector = None + CONNECTION_ARGS_MODEL = None def __init__( self, diff --git a/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/result.json b/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/result.json index 042a8d56..e2f6f3f1 100644 --- a/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/result.json +++ b/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/result.json @@ -1,8 +1,8 @@ -{ - "status": "OK", - "message": "SYSCTL data collected", - "task": "SysctlCollector", - "parent": "SysctlPlugin", - "start_time": "2026-02-10T18:00:00.000000", - "end_time": "2026-02-10T18:00:01.000000" -} +{ + "status": "OK", + "message": "SYSCTL data collected", + "task": "SysctlCollector", + "parent": "SysctlPlugin", + "start_time": "2026-02-10T18:00:00.000000", + "end_time": "2026-02-10T18:00:01.000000" +} diff --git a/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/sysctldatamodel.json b/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/sysctldatamodel.json index bd25b557..0780005f 100644 --- a/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/sysctldatamodel.json +++ b/test/unit/framework/fixtures/compare_runs_sysctl/run1/sysctl_plugin/sysctl_collector/sysctldatamodel.json @@ -1,13 +1,13 @@ -{ - "vm_swappiness": 60, - "vm_numa_balancing": 0, - "vm_oom_kill_allocating_task": 0, - "vm_compaction_proactiveness": 0, - "vm_compact_unevictable_allowed": 1, - "vm_extfrag_threshold": 500, - "vm_zone_reclaim_mode": 0, - "vm_dirty_background_ratio": 10, - "vm_dirty_ratio": 20, - "vm_dirty_writeback_centisecs": 500, - "kernel_numa_balancing": 0 -} +{ + "vm_swappiness": 60, + "vm_numa_balancing": 0, + "vm_oom_kill_allocating_task": 0, + "vm_compaction_proactiveness": 0, + "vm_compact_unevictable_allowed": 1, + "vm_extfrag_threshold": 500, + "vm_zone_reclaim_mode": 0, + "vm_dirty_background_ratio": 10, + "vm_dirty_ratio": 20, + "vm_dirty_writeback_centisecs": 500, + "kernel_numa_balancing": 0 +} diff --git a/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/result.json b/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/result.json index 042a8d56..e2f6f3f1 100644 --- a/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/result.json +++ b/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/result.json @@ -1,8 +1,8 @@ -{ - "status": "OK", - "message": "SYSCTL data collected", - "task": "SysctlCollector", - "parent": "SysctlPlugin", - "start_time": "2026-02-10T18:00:00.000000", - "end_time": "2026-02-10T18:00:01.000000" -} +{ + "status": "OK", + "message": "SYSCTL data collected", + "task": "SysctlCollector", + "parent": "SysctlPlugin", + "start_time": "2026-02-10T18:00:00.000000", + "end_time": "2026-02-10T18:00:01.000000" +} diff --git a/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/sysctldatamodel.json b/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/sysctldatamodel.json index 2e8df42f..21691f53 100644 --- a/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/sysctldatamodel.json +++ b/test/unit/framework/fixtures/compare_runs_sysctl/run2/sysctl_plugin/sysctl_collector/sysctldatamodel.json @@ -1,13 +1,13 @@ -{ - "vm_swappiness": 10, - "vm_numa_balancing": 0, - "vm_oom_kill_allocating_task": 0, - "vm_compaction_proactiveness": 0, - "vm_compact_unevictable_allowed": 1, - "vm_extfrag_threshold": 500, - "vm_zone_reclaim_mode": 0, - "vm_dirty_background_ratio": 5, - "vm_dirty_ratio": 15, - "vm_dirty_writeback_centisecs": 500, - "kernel_numa_balancing": 0 -} +{ + "vm_swappiness": 10, + "vm_numa_balancing": 0, + "vm_oom_kill_allocating_task": 0, + "vm_compaction_proactiveness": 0, + "vm_compact_unevictable_allowed": 1, + "vm_extfrag_threshold": 500, + "vm_zone_reclaim_mode": 0, + "vm_dirty_background_ratio": 5, + "vm_dirty_ratio": 15, + "vm_dirty_writeback_centisecs": 500, + "kernel_numa_balancing": 0 +} diff --git a/test/unit/framework/fixtures/example.json b/test/unit/framework/fixtures/valid_configs/example.json similarity index 100% rename from test/unit/framework/fixtures/example.json rename to test/unit/framework/fixtures/valid_configs/example.json diff --git a/test/unit/framework/fixtures/example_config.json b/test/unit/framework/fixtures/valid_configs/example_config.json similarity index 100% rename from test/unit/framework/fixtures/example_config.json rename to test/unit/framework/fixtures/valid_configs/example_config.json diff --git a/test/unit/framework/test_cli.py b/test/unit/framework/test_cli.py index 8df56e95..8679b1da 100644 --- a/test/unit/framework/test_cli.py +++ b/test/unit/framework/test_cli.py @@ -68,7 +68,9 @@ def test_dict_arg(): def test_json_arg(framework_fixtures_path): - assert json_arg(os.path.join(framework_fixtures_path, "example.json")) == {"test": 123} + assert json_arg(os.path.join(framework_fixtures_path, "valid_configs", "example.json")) == { + "test": 123 + } with pytest.raises(argparse.ArgumentTypeError): json_arg(os.path.join(framework_fixtures_path, "invalid.json")) @@ -79,7 +81,7 @@ class TestArg(BaseModel): arg_handler = ModelArgHandler(TestArg) assert arg_handler.process_file_arg( - os.path.join(framework_fixtures_path, "example.json") + os.path.join(framework_fixtures_path, "valid_configs", "example.json") ) == TestArg(test=123) with pytest.raises(argparse.ArgumentTypeError): @@ -96,7 +98,10 @@ def test_system_info_builder(): system_config=None, ) ) == SystemInfo( - name="test_name", sku="test_sku", platform="test_plat", location=SystemLocation.LOCAL + name="test_name", + sku="test_sku", + platform="test_plat", + location=SystemLocation.LOCAL, ) with pytest.raises(argparse.ArgumentTypeError): @@ -122,7 +127,18 @@ def test_system_info_builder(): ( ["--sys-name", "test-sys", "--sys-sku", "test-sku", "run-plugins", "-h"], ["TestPlugin1", "TestPlugin2"], - (["--sys-name", "test-sys", "--sys-sku", "test-sku", "run-plugins", "-h"], {}, []), + ( + [ + "--sys-name", + "test-sys", + "--sys-sku", + "test-sku", + "run-plugins", + "-h", + ], + {}, + [], + ), ), ( [ diff --git a/test/unit/framework/test_cli_helper.py b/test/unit/framework/test_cli_helper.py index b0f666d0..911588d3 100644 --- a/test/unit/framework/test_cli_helper.py +++ b/test/unit/framework/test_cli_helper.py @@ -76,10 +76,37 @@ def test_generate_reference_config(plugin_registry): ] ref_config = generate_reference_config(results, plugin_registry, logging.getLogger()) - dump = ref_config.dict() + dump = ref_config.model_dump() assert dump["plugins"] == {"TestPluginA": {"analysis_args": {"model_attr": 17}}} +def test_generate_reference_config_missing_data_model_warning(plugin_registry, caplog): + """The 'data model not found' warning must be formattable.""" + caplog.set_level(logging.WARNING) + results = [ + PluginResult( + status=ExecutionStatus.OK, + source="TestPluginA", + message="Plugin tasks completed successfully", + result_data=DataPluginResult( + system_data=None, + collection_result=TaskResult( + status=ExecutionStatus.OK, + task="BiosCollector", + parent="TestPluginA", + artifacts=[], + ), + ), + ) + ] + + generate_reference_config(results, plugin_registry, logging.getLogger()) + + records = [r for r in caplog.records if "data model not found" in r.msg] + assert len(records) == 1 + assert "TestPluginA" in records[0].getMessage() + + def test_get_plugin_configs(): with pytest.raises(argparse.ArgumentTypeError): get_plugin_configs( @@ -136,7 +163,7 @@ def test_config_builder(plugin_registry): config = build_config( config_reg=ConfigRegistry( - config_path=os.path.join(os.path.dirname(__file__), "fixtures"), + config_path=os.path.join(os.path.dirname(__file__), "fixtures", "valid_configs"), load_entry_point_configs=False, ), plugin_reg=plugin_registry, @@ -299,3 +326,50 @@ def test_generate_summary(tmp_path): rows = list(csv.DictReader(f)) assert len(rows) == 1 assert rows[0]["plugin"] == "PluginA" + + +def test_get_plugin_configs_must_copy_to_build_its_configs_to_avoid_mutation(): + """cli --skip-sudo mutates the returned config in place; it must not corrupt the built-in.""" + built_in_configs = {"MyConfig": PluginConfig(name="MyConfig")} + + plugin_configs = get_plugin_configs( + system_interaction_level="INTERACTIVE", + plugin_config_input=["MyConfig"], + built_in_configs=built_in_configs, + parsed_plugin_args={}, + plugin_subparser_map={}, + ) + + # this is exactly what nodescraper/cli/cli.py does for --skip-sudo + plugin_configs[-1].global_args.setdefault("collection_args", {})["skip_sudo"] = True + # Ensure that skip_sudo is there + assert plugin_configs[-1].global_args["collection_args"]["skip_sudo"] is True + + assert built_in_configs["MyConfig"].global_args == {} + + +def test_dump_to_csv_logs_success_when_written(tmp_path: Path): + """Baseline: a successful write reports the output file.""" + from unittest.mock import MagicMock + + mock_logger = MagicMock() + out_file = str(tmp_path / "out.csv") + + dump_to_csv([{"a": "1"}], out_file, ["a"], mock_logger) + + mock_logger.error.assert_not_called() + mock_logger.info.assert_called_once() + + +def test_dump_to_csv_does_not_log_success_when_write_fails(tmp_path: Path): + """A failed write must not be reported as data written to the csv file.""" + from unittest.mock import MagicMock + + mock_logger = MagicMock() + # parent dir does not exist, so open() fails + out_file = str(tmp_path / "missing_dir" / "out.csv") + + dump_to_csv([{"a": "1"}], out_file, ["a"], mock_logger) + + mock_logger.error.assert_called_once() + mock_logger.info.assert_not_called() diff --git a/test/unit/framework/test_collectorargs.py b/test/unit/framework/test_collectorargs.py new file mode 100644 index 00000000..09eb7326 --- /dev/null +++ b/test/unit/framework/test_collectorargs.py @@ -0,0 +1,30 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +from typing import Optional + +import pytest + +from nodescraper.models.collectorargs import CollectorArgs + + +class MyCollectorArgs(CollectorArgs): + opt_arg: Optional[str] = None + set_arg: Optional[str] = None + + +def test_collector_args_forbids_extra(): + """Baseline: extra="forbid" from model_config is honored.""" + with pytest.raises(ValueError): + MyCollectorArgs(not_a_field=1) + + +def test_collector_args_dumps_none(): + """model_config declares should serialize all args.""" + args = MyCollectorArgs(set_arg="abc") + assert args.model_dump() == { + "html_view": False, + "set_arg": "abc", + "opt_arg": None, + } diff --git a/test/unit/framework/test_compare_runs.py b/test/unit/framework/test_compare_runs.py index 75c37cc6..b9fad638 100644 --- a/test/unit/framework/test_compare_runs.py +++ b/test/unit/framework/test_compare_runs.py @@ -23,11 +23,13 @@ # SOFTWARE. # ############################################################################### +import json import logging from nodescraper.cli.compare_runs import ( _diff_value, _format_value, + _load_plugin_data_from_run, run_compare_runs, ) from nodescraper.pluginregistry import PluginRegistry @@ -169,3 +171,38 @@ def test_run_compare_runs_one_run_missing_plugin(caplog, framework_fixtures_path assert "Loading run 1" in caplog.text assert "Loading run 2" in caplog.text assert "not found in run 2" in caplog.text or "NOT_RAN" in caplog.text + + +def test_load_plugin_data_skips_result_json_failing_model_validation( + caplog, framework_fixtures_path, tmp_path +): + """An unparsable result.json must be skipped, not abort the whole run load.""" + caplog.set_level(logging.WARNING) + logger = logging.getLogger() + + fixture_collector = framework_fixtures_path / "log_dir" / "collector" + run = tmp_path / "run" + + # A collector dir whose result.json fails TaskResult validation (bad status name) + bad_collector = run / "bad_collector" + bad_collector.mkdir(parents=True) + (bad_collector / "result.json").write_text( + json.dumps({"status": "BOGUS", "task": "BiosCollector", "parent": "BiosPlugin"}), + encoding="utf-8", + ) + (bad_collector / "biosdatamodel.json").write_text( + (fixture_collector / "biosdatamodel.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + # A valid collector dir that should still be loaded + good_collector = run / "zz_collector" + good_collector.mkdir(parents=True) + for name in ("result.json", "biosdatamodel.json"): + (good_collector / name).write_text( + (fixture_collector / name).read_text(encoding="utf-8"), encoding="utf-8" + ) + + data = _load_plugin_data_from_run(str(run), PluginRegistry(), logger) + + assert "BiosPlugin" in data diff --git a/test/unit/framework/test_config_registry.py b/test/unit/framework/test_config_registry.py index 5ba78fd1..0330196c 100644 --- a/test/unit/framework/test_config_registry.py +++ b/test/unit/framework/test_config_registry.py @@ -32,7 +32,7 @@ def test_config_registry(): config_registry = ConfigRegistry( - config_path=os.path.join(os.path.dirname(__file__), "fixtures"), + config_path=os.path.join(os.path.dirname(__file__), "fixtures", "valid_configs"), load_entry_point_configs=False, ) @@ -48,3 +48,16 @@ def test_config_registry(): global_args={}, plugins={}, result_collators={}, name=None, desc=None ), } + + +def test_config_registry_raises_on_invalid_json(tmp_path): + """A JSON file whose top level is not an object must be skipped, not crash the registry.""" + (tmp_path / "list.json").write_text("[1, 2, 3]", encoding="utf-8") + (tmp_path / "good.json").write_text('{"name": "GoodConfig", "plugins": {}}', encoding="utf-8") + import pytest + + with pytest.raises(RuntimeError, match="Failed to load config from"): + ConfigRegistry( + config_path=str(tmp_path), + load_entry_point_configs=False, + ) diff --git a/test/unit/framework/test_connection_manager_entrypoints.py b/test/unit/framework/test_connection_manager_entrypoints.py index a11d5b50..c5344c54 100644 --- a/test/unit/framework/test_connection_manager_entrypoints.py +++ b/test/unit/framework/test_connection_manager_entrypoints.py @@ -74,3 +74,21 @@ def test_plugin_registry_merges_entry_point_connection_managers(): def test_plugin_registry_can_disable_entry_point_connection_managers(): reg = PluginRegistry(load_entry_point_connection_managers=False) assert "InBandConnectionManager" in reg.connection_managers + + +def test_load_connection_managers_cache_is_not_mutable_by_callers(): + """The entry point cache must not be aliased out to callers (cf. load_plugins_from_entry_points).""" + mock_ep = MagicMock() + mock_ep.name = "AliasInBand" + mock_ep.load.return_value = InBandConnectionManager + PluginRegistry.clear_caches() + + with patch("nodescraper.pluginregistry.importlib.metadata.entry_points") as mock_eps: + mock_eps.side_effect = lambda *a, **k: _entry_points_side_effect_cm_only(mock_ep, *a, **k) + PluginRegistry.load_connection_managers_from_entry_points() + warm = PluginRegistry.load_connection_managers_from_entry_points() + warm["InjectedByCaller"] = InBandConnectionManager + + refetched = PluginRegistry.load_connection_managers_from_entry_points() + + assert "InjectedByCaller" not in refetched diff --git a/test/unit/framework/test_dataanalyzer.py b/test/unit/framework/test_dataanalyzer.py index ce13e3a8..b5436911 100644 --- a/test/unit/framework/test_dataanalyzer.py +++ b/test/unit/framework/test_dataanalyzer.py @@ -23,9 +23,13 @@ # SOFTWARE. # ############################################################################### +from typing import Optional + from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.interfaces.dataanalyzertask import analyze_decorator +from nodescraper.models.event import Event +from nodescraper.models.taskresult import TaskResult def test_invalid_data(mock_analyzer, dummy_data_model, system_info): @@ -89,3 +93,93 @@ def fake_err(self, data: dummy_data_model, args: dummy_arg): assert any( "Exception during data analysis: some_err" in event.description for event in result.events ) + + +def test_analyzer_subclass_with_none_data_model_raises_type_error(dummy_data_model): + """Baseline: DATA_MODEL explicitly set to None is rejected with a TypeError.""" + import pytest + + from nodescraper.interfaces.dataanalyzertask import DataAnalyzer + + with pytest.raises(TypeError): + + class NoneModelAnalyzer(DataAnalyzer): + DATA_MODEL = None + + def analyze_data(self, data, args=None): + return self.result + + +def test_analyzer_subclass_without_data_model_raises_type_error(): + """A concrete analyzer that never declares DATA_MODEL must raise TypeError, not AttributeError.""" + import pytest + + from nodescraper.interfaces.dataanalyzertask import DataAnalyzer + + with pytest.raises(TypeError): + + class MissingModelAnalyzer(DataAnalyzer): + def analyze_data(self, data, args=None): + return self.result + + +def _event(priority: EventPriority, description: str, count: Optional[int] = None) -> Event: + data = {"count": count} if count is not None else {} + return Event(category="OS", description=description, priority=priority, data=data) + + +def test_event_summary_empty_when_no_events(): + assert TaskResult()._get_event_summary() == "" + + +def test_event_summary_counts_warnings_and_errors(): + result = TaskResult( + events=[ + _event(EventPriority.WARNING, "w1"), + _event(EventPriority.WARNING, "w2", count=2), + _event(EventPriority.ERROR, "e1"), + ] + ) + assert result._get_event_summary() == "3 warnings; 1 errors: e1" + + +def test_event_summary_top_three_errors_by_count_with_omission_note(): + result = TaskResult( + events=[ + _event(EventPriority.ERROR, "e1"), + _event(EventPriority.ERROR, "e2", count=4), + _event(EventPriority.ERROR, "e1"), + _event(EventPriority.ERROR, "e3"), + _event(EventPriority.ERROR, "e4"), + ] + ) + assert result._get_event_summary() == ( + "8 errors: e2 (x4), e1 (x2), e3 (omitted 1 descriptions)" + ) + + +def test_event_summary_critical_counted_as_error_and_listed(): + result = TaskResult(events=[_event(EventPriority.CRITICAL, "c1")]) + assert result._get_event_summary() == "1 errors; critical: c1" + + +def test_event_summary_three_critical_no_omission_note(): + result = TaskResult(events=[_event(EventPriority.CRITICAL, f"c{i}") for i in range(3)]) + summary = result._get_event_summary() + assert summary == "3 errors; critical: c0, c1, c2" + assert "omitted" not in summary + + +def test_event_summary_top_three_critical_by_count_with_omission_note(): + result = TaskResult( + events=[ + _event(EventPriority.CRITICAL, "rare"), + _event(EventPriority.CRITICAL, "common", count=5), + _event(EventPriority.CRITICAL, "mid", count=3), + _event(EventPriority.CRITICAL, "also_rare"), + _event(EventPriority.CRITICAL, "second", count=4), + ] + ) + assert result._get_event_summary() == ( + "14 errors; critical: common (x5), second (x4), mid (x3) (omitted 2 descriptions)" + ) diff --git a/test/unit/framework/test_datacollector.py b/test/unit/framework/test_datacollector.py index 410b4d85..cc1798e6 100644 --- a/test/unit/framework/test_datacollector.py +++ b/test/unit/framework/test_datacollector.py @@ -49,6 +49,9 @@ def finalize(self, logger): pass +DUMMY_GIVES_THIS = DummyDataModel(foo=0xC0FFEE) + + class DummyCollector(DataCollector[None, DummyDataModel, None]): SUPPORTED_SKUS = {"GOOD"} SUPPORTED_PLATFORMS = {"X"} @@ -67,7 +70,7 @@ def _init_result(self): def collect_data(self, args=None) -> Tuple[TaskResult, Optional[DummyDataModel]]: self.result.status = ExecutionStatus.OK - return self.result, None + return self.result, DUMMY_GIVES_THIS def test_ok(system_info, conn_mock): @@ -82,7 +85,8 @@ def test_ok(system_info, conn_mock): result, data = dc.collect_data() assert result.status == ExecutionStatus.OK - assert ("hook", result, None) in calls + assert data == DUMMY_GIVES_THIS + assert ("hook", result, DUMMY_GIVES_THIS) in calls def test_exception(system_info, conn_mock): @@ -194,3 +198,86 @@ def _init_result(self): def collect_data(self, args=None): return self.result, None + + +class LogPathCollector(DataCollector[None, DummyDataModel, None]): + DATA_MODEL = DummyDataModel + + def collect_data(self, args=None) -> Tuple[TaskResult, Optional[DummyDataModel]]: + return self.result, None + + +def test_data_collector_keeps_log_path(system_info, conn_mock): + """Baseline: the base collector stores the log path passed by DataPlugin.collect.""" + collector = LogPathCollector(system_info, conn_mock, log_path="/tmp/run/collector") + + assert collector.log_path == "/tmp/run/collector" + + +def test_inband_collector_keeps_log_path(system_info, conn_mock): + """InBandDataCollector must forward log_path to the base collector, not drop it.""" + from nodescraper.base.inbandcollectortask import InBandDataCollector + from nodescraper.enums import OSFamily + + class InBandLogPathCollector(InBandDataCollector[DummyDataModel, None]): + DATA_MODEL = DummyDataModel + + def collect_data(self, args=None) -> Tuple[TaskResult, Optional[DummyDataModel]]: + return self.result, None + + system_info.os_family = OSFamily.LINUX + collector = InBandLogPathCollector(system_info, conn_mock, log_path="/tmp/run/collector") + + assert collector.log_path == "/tmp/run/collector" + + +class UnsetStatusWithDataCollector(DataCollector[None, DummyDataModel, None]): + DATA_MODEL = DummyDataModel + + def collect_data(self, args=None) -> Tuple[TaskResult, Optional[DummyDataModel]]: + return self.result, DummyDataModel(foo=1) + + +class UnsetStatusNoDataCollector(DataCollector[None, DummyDataModel, None]): + DATA_MODEL = DummyDataModel + + def collect_data(self, args=None) -> Tuple[TaskResult, Optional[DummyDataModel]]: + return self.result, None + + +class OkStatusNoDataCollector(DataCollector[None, DummyDataModel, None]): + DATA_MODEL = DummyDataModel + + def collect_data(self, args=None) -> Tuple[TaskResult, Optional[DummyDataModel]]: + self.result.status = ExecutionStatus.OK + return self.result, None + + +def test_collector_returning_data_with_unset_status_is_ok(system_info, conn_mock): + """Baseline: data collected with an unset status finalizes to OK.""" + collector = UnsetStatusWithDataCollector(system_info, conn_mock) + + result, data = collector.collect_data() + + assert data is not None + assert result.status == ExecutionStatus.OK + + +def test_collector_returning_no_data_with_unset_status_is_execution_failure(system_info, conn_mock): + """A collector that returns no data and never sets a status must not be reported as OK.""" + collector = UnsetStatusNoDataCollector(system_info, conn_mock) + + result, data = collector.collect_data() + + assert data is None + assert result.status == ExecutionStatus.EXECUTION_FAILURE + + +def test_collector_returning_no_data_with_ok_status_is_execution_failure(system_info, conn_mock): + """A collector that returns no data and never sets a status must not be reported as OK.""" + collector = UnsetStatusNoDataCollector(system_info, conn_mock) + + result, data = collector.collect_data() + + assert data is None + assert result.status == ExecutionStatus.EXECUTION_FAILURE diff --git a/test/unit/framework/test_datamodel.py b/test/unit/framework/test_datamodel.py new file mode 100644 index 00000000..c2bdaa99 --- /dev/null +++ b/test/unit/framework/test_datamodel.py @@ -0,0 +1,87 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +import base64 +import json +import os +from pathlib import Path + +from nodescraper.models.datamodel import DataModel, FileModel + + +class FolderDataModel(DataModel): + value: str = "" + + @classmethod + def build_from_folder(cls, folder_path: str) -> "FolderDataModel": + return cls(value=os.path.basename(folder_path)) + + +class CperListDataModel(DataModel): + value: str = "ok" + cper_data: list[FileModel] = [] + + +def test_import_model_from_dict(): + """Baseline: a dict is passed straight to the model constructor.""" + assert FolderDataModel.import_model({"value": "abc"}).value == "abc" + + +def test_import_model_from_json_file(tmp_path: Path): + """Baseline: a json file path is read and parsed.""" + model_file = tmp_path / "model.json" + model_file.write_text(json.dumps({"value": "from-file"}), encoding="utf-8") + + assert FolderDataModel.import_model(str(model_file)).value == "from-file" + + +def test_import_model_from_directory_uses_build_from_folder(tmp_path: Path): + """A directory path must be dispatched to build_from_folder.""" + folder = tmp_path / "collected" + folder.mkdir() + + assert FolderDataModel.import_model(str(folder)).value == "collected" + + +def test_filemodel_json_roundtrips_binary_cper_bytes(): + """Non-UTF-8 CPER bytes serialize as base64 and reload intact as bytes.""" + cper_bytes = b"CPER\x00\x01\xff\xff\xff\xff" + model = FileModel(file_contents=cper_bytes, file_name="corrected-1.cper") + dumped = model.model_dump_json() + payload = json.loads(dumped) + assert payload["file_contents"] == base64.urlsafe_b64encode(cper_bytes).decode("ascii") + + reloaded = FileModel.model_validate_json(dumped) + assert isinstance(reloaded.file_contents, bytes) + assert reloaded.file_contents == cper_bytes + + +def test_filemodel_python_strings_are_utf8_not_base64(): + """A plain string that is also valid base64 must be stored as UTF-8 text.""" + text = "VGVzdA==" + model = FileModel(file_contents=text, file_name="t.txt") + assert model.file_contents == text.encode("utf-8") + assert FileModel(file_contents="hello", file_name="t.txt").file_contents == b"hello" + + +def test_log_model_keeps_cper_list_in_json(tmp_path: Path): + """CPER-like bytes must not crash log_model and remain in the JSON dump.""" + cper_bytes = b"CPER\x00\x01\xff\xff\xff\xff" + model = CperListDataModel( + value="ok", + cper_data=[FileModel(file_contents=cper_bytes, file_name="corrected-1.cper")], + ) + + model.log_model(str(tmp_path)) + + cper_path = tmp_path / "corrected-1.cper" + assert cper_path.read_bytes() == cper_bytes + + json_files = list(tmp_path.glob("*.json")) + assert len(json_files) == 1 + payload = json.loads(json_files[0].read_text(encoding="utf-8")) + assert payload["value"] == "ok" + assert len(payload["cper_data"]) == 1 + assert payload["cper_data"][0]["file_name"] == "corrected-1.cper" + assert base64.urlsafe_b64decode(payload["cper_data"][0]["file_contents"]) == cper_bytes diff --git a/test/unit/framework/test_dataplugin.py b/test/unit/framework/test_dataplugin.py index 67b92fb9..48b41a3c 100644 --- a/test/unit/framework/test_dataplugin.py +++ b/test/unit/framework/test_dataplugin.py @@ -31,11 +31,18 @@ import pytest from framework.common.shared_utils import MockConnectionManager -from nodescraper.enums import EventPriority, ExecutionStatus, SystemInteractionLevel +from nodescraper import utils +from nodescraper.enums import ( + EventCategory, + EventPriority, + ExecutionStatus, + SystemInteractionLevel, +) from nodescraper.interfaces.dataanalyzertask import DataAnalyzer from nodescraper.interfaces.datacollectortask import DataCollector from nodescraper.interfaces.dataplugin import DataPlugin -from nodescraper.models import CollectorArgs, DataModel, TaskResult +from nodescraper.interfaces.task import SystemCompatibilityError +from nodescraper.models import CollectorArgs, DataModel, DataPluginResult, TaskResult class StandardDataModel(DataModel): @@ -124,6 +131,27 @@ def test_data_property(self, plugin): assert isinstance(plugin.data, StandardDataModel) assert plugin.data.value == "dict_value" + def test_default_analysis_result_is_poplulated(self, plugin): + assert plugin.analysis_result.status == ExecutionStatus.NOT_RAN + assert "not ran" in plugin.analysis_result.message + assert "analysis" in plugin.analysis_result.message + assert plugin.analysis_result.parent is not None + assert plugin.analysis_result.task is not None + + def test_default_collection_result_is_poplulated(self, plugin): + assert plugin.collection_result.status == ExecutionStatus.NOT_RAN + assert "not ran" in plugin.collection_result.message + assert "collection" in plugin.collection_result.message + assert plugin.collection_result.parent is not None + assert plugin.collection_result.task is not None + + def test_data_setter_error_names_expected_model(self, plugin): + """Invalid data should report the expected DATA_MODEL name, not its metaclass.""" + with pytest.raises(ValueError) as exc_info: + plugin.data = 12345 + + assert "StandardDataModel" in str(exc_info.value) + def test_collect_creates_connection_manager(self, plugin, conn_mock, system_info, logger): assert plugin.connection_manager is None @@ -314,7 +342,7 @@ def test_run_with_parameters(self, plugin_with_conn): analysis=True, max_event_priority_level=EventPriority.ERROR, system_interaction_level=SystemInteractionLevel.PASSIVE, - preserve_connection=True, + preserve_connection=False, data=data, collection_args=collection_args, analysis_args=analysis_args, @@ -323,17 +351,16 @@ def test_run_with_parameters(self, plugin_with_conn): mock_collect.assert_called_once_with( max_event_priority_level=EventPriority.ERROR, system_interaction_level=SystemInteractionLevel.PASSIVE, + preserve_connection=False, collection_args=collection_args, - preserve_connection=True, ) mock_analyze.assert_called_once_with( max_event_priority_level=EventPriority.ERROR, analysis_args=analysis_args, data=data ) - def test_collect_preserve_connection(self, plugin_with_conn): - """Test the behavior of preserve_connection parameter in collect method.""" - # Test with preserve_connection=True + def test_collect_leaves_externally_owned_connection_open(self, plugin_with_conn): + """A connection manager passed in by the caller is closed by its owner, not the plugin.""" with patch.object(BaseDataCollector, "collect_data") as mock_collect: with patch.object(MockConnectionManager, "disconnect") as mock_disconnect: mock_collect.return_value = ( @@ -341,17 +368,31 @@ def test_collect_preserve_connection(self, plugin_with_conn): StandardDataModel(), ) - # Call collect with preserve_connection=True - result = plugin_with_conn.collect(preserve_connection=True) + result = plugin_with_conn.collect() - # Verify collect_data was called and result is OK mock_collect.assert_called_once() assert result.status == ExecutionStatus.OK - - # Verify disconnect was NOT called when preserve_connection=True mock_disconnect.assert_not_called() - # Test with preserve_connection=False (default) + def test_collect_closes_passed_in_connection_when_preserve_connection_false( + self, plugin_with_conn + ): + with patch.object(MockConnectionManager, "disconnect") as mock_disconnect: + result = plugin_with_conn.collect(preserve_connection=False) + + assert result.status == ExecutionStatus.OK + mock_disconnect.assert_called_once() + + def test_run_forwards_preserve_connection_to_collect(self, plugin_with_conn): + with patch.object(MockConnectionManager, "disconnect") as mock_disconnect: + plugin_with_conn.run(analysis=False, preserve_connection=False) + assert mock_disconnect.call_count == 1 + + plugin_with_conn.run(analysis=False) + assert mock_disconnect.call_count == 1 + + def test_collect_closes_connection_it_created(self, plugin): + """A connection manager created by the plugin itself is closed after collection.""" with patch.object(BaseDataCollector, "collect_data") as mock_collect: with patch.object(MockConnectionManager, "disconnect") as mock_disconnect: mock_collect.return_value = ( @@ -359,14 +400,10 @@ def test_collect_preserve_connection(self, plugin_with_conn): StandardDataModel(), ) - # Call collect with preserve_connection=False (default) - result = plugin_with_conn.collect(preserve_connection=False) + result = plugin.collect() - # Verify collect_data was called and result is OK mock_collect.assert_called_once() assert result.status == ExecutionStatus.OK - - # Verify disconnect WAS called when preserve_connection=False mock_disconnect.assert_called_once() def test_run_with_data_file_no_collection(self, plugin_with_conn, tmp_path): @@ -507,6 +544,30 @@ def test_find_datamodel_path_success(self, tmp_path: Path) -> None: assert found is not None assert found.endswith("contentmodel.json") + def test_find_datamodel_path_prefers_datamodel_json_over_log( + self, tmp_path: Path, monkeypatch + ) -> None: + """A collector dir holding both files must yield the datamodel json, not the .log.""" + import os + + collector_dir = tmp_path / "extract_plugin" / "content_collector" + collector_dir.mkdir(parents=True) + (collector_dir / "result.json").write_text( + json.dumps({"parent": "ExtractPlugin"}), encoding="utf-8" + ) + (collector_dir / "contentmodel.json").write_text( + json.dumps({"value": "from_run"}), encoding="utf-8" + ) + (collector_dir / "capture.log").write_text("raw log", encoding="utf-8") + + # make directory listing order deterministic: "capture.log" sorts first + real_listdir = os.listdir + monkeypatch.setattr(os, "listdir", lambda path: sorted(real_listdir(path))) + + found = ExtractPlugin.find_datamodel_path_in_run(str(tmp_path)) + assert found is not None + assert found.endswith("contentmodel.json") + def test_find_datamodel_path_wrong_parent(self, tmp_path: Path) -> None: collector_dir = tmp_path / "extract_plugin" / "content_collector" collector_dir.mkdir(parents=True) @@ -682,3 +743,702 @@ def test_find_datamodel_path_in_run_checks_all_collectors(self, tmp_path: Path) found = MultiCollectorPlugin.find_datamodel_path_in_run(str(tmp_path)) assert found is not None assert found.endswith("multipartdatamodel.json") + + def test_log_path_creates_collector_subdirectories(self, plugin_with_conn, tmp_path): + """Test that log_path creates subdirectories for each collector.""" + log_path = tmp_path / "test_logs" + + # Create plugin with log_path + multi_plugin = MultiCollectorPlugin( + system_info=plugin_with_conn.system_info, + logger=plugin_with_conn.logger, + connection_manager=plugin_with_conn.connection_manager, + log_path=str(log_path), + ) + + with ( + patch.object(AlphaCollector, "__init__", return_value=None) as alpha_init, + patch.object(BetaCollector, "__init__", return_value=None) as beta_init, + patch.object(AlphaCollector, "collect_data") as alpha_collect, + patch.object(BetaCollector, "collect_data") as beta_collect, + ): + alpha_collect.return_value = ( + TaskResult(status=ExecutionStatus.OK, task="AlphaCollector"), + MultiPartDataModel(alpha="alpha-value"), + ) + beta_collect.return_value = ( + TaskResult(status=ExecutionStatus.OK, task="BetaCollector"), + MultiPartDataModel(beta="beta-value"), + ) + + multi_plugin.collect() + + # Verify that AlphaCollector was initialized with correct log_path + alpha_call_kwargs = alpha_init.call_args[1] + assert "log_path" in alpha_call_kwargs + alpha_log_path = Path(alpha_call_kwargs["log_path"]) + assert alpha_log_path.parent.parent == log_path + assert alpha_log_path.parent.name == "multi_collector_plugin" + assert alpha_log_path.name == "alpha_collector" + + # Verify that BetaCollector was initialized with correct log_path + beta_call_kwargs = beta_init.call_args[1] + assert "log_path" in beta_call_kwargs + beta_log_path = Path(beta_call_kwargs["log_path"]) + assert beta_log_path.parent.parent == log_path + assert beta_log_path.parent.name == "multi_collector_plugin" + assert beta_log_path.name == "beta_collector" + + # Verify directories were created + assert alpha_log_path.exists() + assert beta_log_path.exists() + + def test_log_path_none_does_not_create_directories(self, plugin_with_conn): + """Test that when log_path is None, no directories are created.""" + multi_plugin = MultiCollectorPlugin( + system_info=plugin_with_conn.system_info, + logger=plugin_with_conn.logger, + connection_manager=plugin_with_conn.connection_manager, + log_path=None, + ) + + with ( + patch.object(AlphaCollector, "__init__", return_value=None) as alpha_init, + patch.object(BetaCollector, "__init__", return_value=None) as beta_init, + patch.object(AlphaCollector, "collect_data") as alpha_collect, + patch.object(BetaCollector, "collect_data") as beta_collect, + ): + alpha_collect.return_value = ( + TaskResult(status=ExecutionStatus.OK, task="AlphaCollector"), + MultiPartDataModel(alpha="alpha-value"), + ) + beta_collect.return_value = ( + TaskResult(status=ExecutionStatus.OK, task="BetaCollector"), + MultiPartDataModel(beta="beta-value"), + ) + + multi_plugin.collect() + + # Verify that collectors were initialized with log_path=None + alpha_call_kwargs = alpha_init.call_args[1] + assert alpha_call_kwargs["log_path"] is None + + beta_call_kwargs = beta_init.call_args[1] + assert beta_call_kwargs["log_path"] is None + + +class OverrideDataModel(DataModel): + value: str = "test" + + +class OverrideCollector(DataCollector): + DATA_MODEL = OverrideDataModel + + def collect_data(self, args=None): + self.result.status = ExecutionStatus.OK + return self.result, OverrideDataModel(value="collected") + + +class OverrideAnalyzer(DataAnalyzer): + DATA_MODEL = OverrideDataModel + + def analyze_data(self, data, args=None): + return TaskResult(status=ExecutionStatus.OK) + + +class OverrideLogDirPlugin(DataPlugin): + DATA_MODEL = OverrideDataModel + CONNECTION_TYPE = MockConnectionManager + COLLECTOR = OverrideCollector + ANALYZER = OverrideAnalyzer + + +class TestDataPluginLogDirNameOverride: + """find_datamodel_path_in_run must read from the dirs collect() actually writes to.""" + + @pytest.fixture(autouse=True) + def registered_overrides(self, monkeypatch): + monkeypatch.setitem( + utils._LOG_DIR_NAME_OVERRIDES, "OverrideLogDirPlugin", "override_LOGDIR_plugin" + ) + monkeypatch.setitem( + utils._LOG_DIR_NAME_OVERRIDES, "OverrideCollector", "override_LOGDIR_collector" + ) + + def test_find_datamodel_path_honors_registered_log_dir_name( + self, plugin_with_conn, tmp_path: Path + ) -> None: + run_path = tmp_path / "scraper_logs_run" + plugin = OverrideLogDirPlugin( + system_info=plugin_with_conn.system_info, + logger=plugin_with_conn.logger, + connection_manager=plugin_with_conn.connection_manager, + log_path=str(run_path), + ) + + assert plugin.collect().status == ExecutionStatus.OK + + # collect() wrote the collector output under the registered override names + collector_dir = run_path / "override_LOGDIR_plugin" / "override_LOGDIR_collector" + assert (collector_dir / "result.json").is_file() + assert (collector_dir / "overridedatamodel.json").is_file() + + found = OverrideLogDirPlugin.find_datamodel_path_in_run(str(run_path)) + assert found is not None + assert Path(found).parent == collector_dir + + def test_load_run_data_honors_registered_log_dir_name( + self, plugin_with_conn, tmp_path: Path + ) -> None: + run_path = tmp_path / "scraper_logs_run" + plugin = OverrideLogDirPlugin( + system_info=plugin_with_conn.system_info, + logger=plugin_with_conn.logger, + connection_manager=plugin_with_conn.connection_manager, + log_path=str(run_path), + ) + plugin.collect() + + loaded = OverrideLogDirPlugin.load_run_data(str(run_path)) + assert loaded is not None + assert loaded["value"] == "collected" + + +class TestDataPluginConnectionResult: + def test_run_includes_connection_result_when_connection_initiated(self, plugin_with_conn): + plugin_with_conn.connection_manager.result.status = ExecutionStatus.EXECUTION_FAILURE + plugin_with_conn.connection_manager.connection = None + + result = plugin_with_conn.run(analysis=False) + + assert result.result_data.connection_result is not None + assert result.result_data.connection_result.status == ExecutionStatus.EXECUTION_FAILURE + assert result.result_data.collection_result.status == ExecutionStatus.NOT_RAN + + def test_run_has_no_connection_result_without_connection_attempt(self, plugin): + with patch.object(CoreDataPlugin, "CONNECTION_TYPE", None): + result = plugin.run(analysis=False) + + assert result.result_data.connection_result is None + + +class FailingConnectionManager(MockConnectionManager): + """Connection manager whose connect() fails without producing a connection.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.connection = None + self.result = TaskResult(task=self.__class__.__name__, parent=self.parent) + + def connect(self): + self._log_event( + category=EventCategory.SSH, + description="Unable to reach host", + priority=EventPriority.CRITICAL, + ) + self.result.status = ExecutionStatus.EXECUTION_FAILURE + return self.result + + +class FailingConnectionPlugin(CoreDataPlugin): + CONNECTION_TYPE = FailingConnectionManager + + +class SilentFailingConnectionManager(FailingConnectionManager): + """Connection manager that returns without a connection, status or event.""" + + def connect(self): + return self.result + + +class SilentFailingConnectionPlugin(CoreDataPlugin): + CONNECTION_TYPE = SilentFailingConnectionManager + + +class TestDataPluginResultConnection: + """connection_result is saved in DataPluginResult whenever a connection is initiated.""" + + def test_default_is_none(self): + assert DataPluginResult().connection_result is None + + def test_successful_connection_saved_after_disconnect(self, plugin): + result = plugin.run() + + connection_result = result.result_data.connection_result + assert connection_result is not None + assert connection_result.status == ExecutionStatus.OK + assert result.status == ExecutionStatus.OK + + def test_connection_result_reported_for_externally_owned_connection(self, plugin_with_conn): + result = plugin_with_conn.run() + + assert result.result_data.connection_result.status == ExecutionStatus.OK + + def test_failed_connection_saved_with_event(self, system_info, logger): + plugin = FailingConnectionPlugin(system_info=system_info, logger=logger) + + result = plugin.run() + + connection_result = result.result_data.connection_result + assert connection_result is not None + assert connection_result.status == ExecutionStatus.EXECUTION_FAILURE + assert any(e.description == "Unable to reach host" for e in connection_result.events) + assert result.result_data.collection_result.status == ExecutionStatus.NOT_RAN + assert result.result_data.collection_result.message == ( + "Connection not available, data collection skipped" + ) + + def test_failed_connection_not_reset_by_disconnect(self, system_info, logger): + plugin = FailingConnectionPlugin(system_info=system_info, logger=logger) + + plugin.collect() + plugin.connection_manager.disconnect() + + assert plugin.connection_manager.result.status == ExecutionStatus.EXECUTION_FAILURE + + def test_failed_connection_is_not_retried(self, system_info, logger): + plugin = FailingConnectionPlugin(system_info=system_info, logger=logger) + + with patch.object( + FailingConnectionManager, "connect", autospec=True, side_effect=lambda s: s.result + ) as mock_connect: + plugin.collect() + plugin.connection_manager.result.status = ExecutionStatus.EXECUTION_FAILURE + plugin.collect() + plugin.collect() + + assert mock_connect.call_count == 1 + + def test_silent_connect_failure_gets_status_and_event(self, system_info, logger): + plugin = SilentFailingConnectionPlugin(system_info=system_info, logger=logger) + + result = plugin.run() + + connection_result = result.result_data.connection_result + assert connection_result.status == ExecutionStatus.EXECUTION_FAILURE + assert len(connection_result.events) >= 1 + + def test_reconnects_after_successful_disconnect(self, plugin_with_conn): + manager = plugin_with_conn.connection_manager + manager.connection = None + manager.result.status = ExecutionStatus.OK + + with patch.object(MockConnectionManager, "connect", autospec=True) as mock_connect: + mock_connect.side_effect = lambda s: setattr(s, "connection", MagicMock()) or s.result + plugin_with_conn.collect() + + mock_connect.assert_called_once() + + def test_connection_result_is_a_snapshot(self, plugin_with_conn): + result = plugin_with_conn.run(analysis=False) + + plugin_with_conn.connection_manager.result.status = ExecutionStatus.EXECUTION_FAILURE + + assert result.result_data.connection_result.status == ExecutionStatus.OK + + def test_connection_result_serializes(self, plugin_with_conn): + result = plugin_with_conn.run(analysis=False) + + dumped = result.result_data.model_dump(mode="json") + + assert dumped["connection_result"]["status"] == "OK" + + +class WarningConnectionManager(MockConnectionManager): + """Connection manager whose successful connect() reports one warning event.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.connection = None + self.result = TaskResult(task=self.__class__.__name__, parent=self.parent) + + def connect(self): + self._log_event( + category=EventCategory.SSH, + description="Slow handshake", + priority=EventPriority.WARNING, + ) + self.connection = MagicMock() + self.result.status = ExecutionStatus.OK + return self.result + + +class WarningConnectionPlugin(CoreDataPlugin): + CONNECTION_TYPE = WarningConnectionManager + + +class TestDataPluginSavesConnectionResult: + """The plugin saves the connection result when it closes the connection itself.""" + + @staticmethod + def _conn_dir(log_path: Path) -> Path: + # manager created by the plugin: //connection + return ( + log_path + / utils.resolve_log_dir_name("WarningConnectionPlugin") + / utils.resolve_log_dir_name("BaseDataCollector") + / "connection" + ) + + def test_successful_connection_saved_when_plugin_owns_connection( + self, system_info, logger, tmp_path + ): + plugin = WarningConnectionPlugin( + system_info=system_info, logger=logger, log_path=str(tmp_path) + ) + + plugin.run(analysis=False) + + conn_dir = self._conn_dir(tmp_path) + assert json.loads((conn_dir / "result.json").read_text())["status"] == "OK" + # connect() already saved the warning; closing the connection must not save it again + events = json.loads((conn_dir / "events.json").read_text()) + assert [e["description"] for e in events] == ["Slow handshake"] + + def test_failed_connection_result_saved_when_plugin_owns_connection( + self, system_info, logger, tmp_path + ): + plugin = FailingConnectionPlugin( + system_info=system_info, logger=logger, log_path=str(tmp_path) + ) + + plugin.run(analysis=False) + + conn_dir = ( + tmp_path + / utils.resolve_log_dir_name("FailingConnectionPlugin") + / utils.resolve_log_dir_name("BaseDataCollector") + / "connection" + ) + assert json.loads((conn_dir / "result.json").read_text())["status"] == "EXECUTION_FAILURE" + events = json.loads((conn_dir / "events.json").read_text()) + assert [e["description"] for e in events] == ["Unable to reach host"] + + +class LifecycleConnectionManager(MockConnectionManager): + """Starts disconnected, connects on demand and counts disconnects per instance.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.connection = None + self.result = self._init_result() + self.disconnect_calls = 0 + + def connect(self): + self.connection = MagicMock() + self.result.status = ExecutionStatus.OK + return self.result + + def disconnect(self): + self.disconnect_calls += 1 + self.connection = None + + +class RaisingConnectManager(LifecycleConnectionManager): + def connect(self): + raise RuntimeError("connect boom") + + +class RaisingInitManager(LifecycleConnectionManager): + def __init__(self, *args, **kwargs): + raise ValueError("bad connection args") + + +class RaisingDisconnectManager(LifecycleConnectionManager): + def disconnect(self): + super().disconnect() + raise OSError("close boom") + + +class InitRaisingCollector(BaseDataCollector): + def __init__(self, *args, **kwargs): + raise RuntimeError("collector init boom") + + +class CollectRaisingCollector(BaseDataCollector): + def collect_data(self, args=None): + raise RuntimeError("collect boom") + + +class IncompatibleCollector(BaseDataCollector): + def __init__(self, *args, **kwargs): + raise SystemCompatibilityError("unsupported system") + + +def _own_connection_plugin(manager_cls, collector_cls=BaseDataCollector): + class OwnConnectionPlugin(DataPlugin): + DATA_MODEL = StandardDataModel + CONNECTION_TYPE = manager_cls + COLLECTOR = collector_cls + ANALYZER = StandardAnalyzer + + return OwnConnectionPlugin + + +class TestDataPluginOwnConnection: + """The DataPlugin creates the connection manager itself and must clean it up.""" + + @staticmethod + def _conn_dir(log_path: Path, collector_cls=BaseDataCollector) -> Path: + return ( + log_path + / utils.resolve_log_dir_name("OwnConnectionPlugin") + / utils.resolve_log_dir_name(collector_cls.__name__) + / "connection" + ) + + @staticmethod + def _external_conn_dir(log_path: Path) -> Path: + # a manager supplied by the caller keeps the caller's hooks: / + return ( + log_path + / utils.resolve_log_dir_name("OwnConnectionPlugin") + / utils.resolve_log_dir_name("LifecycleConnectionManager") + ) + + @staticmethod + def _saved_result(conn_dir: Path) -> dict: + return json.loads((conn_dir / "result.json").read_text()) + + @staticmethod + def _assert_only_plugin_folder(log_path: Path): + # nothing may be written next to (or outside) the plugin's own folder + assert {p.name for p in log_path.iterdir() if p.is_dir()} == { + utils.resolve_log_dir_name("OwnConnectionPlugin") + } + + def _build(self, system_info, logger, tmp_path, manager_cls, collector_cls=BaseDataCollector): + plugin = _own_connection_plugin(manager_cls, collector_cls)( + system_info=system_info, logger=logger, log_path=str(tmp_path) + ) + assert plugin.connection_manager is None + return plugin + + def test_created_connection_is_closed_and_saved_in_plugin_folder( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, LifecycleConnectionManager) + + result = plugin.run() + + manager = plugin.connection_manager + assert isinstance(manager, LifecycleConnectionManager) + assert manager.disconnect_calls == 1 + assert manager.connection is None + conn_dir = self._conn_dir(tmp_path) + saved = self._saved_result(conn_dir) + assert saved["status"] == "OK" + assert saved["task"] == "LifecycleConnectionManager" + assert saved["parent"] == "OwnConnectionPlugin" + assert not (conn_dir / "events.json").exists() + self._assert_only_plugin_folder(tmp_path) + assert result.result_data.connection_result.status == ExecutionStatus.OK + + def test_connection_result_is_kept_apart_from_collector_files( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, LifecycleConnectionManager) + + plugin.collect() + + collector_dir = ( + tmp_path + / utils.resolve_log_dir_name("OwnConnectionPlugin") + / utils.resolve_log_dir_name("BaseDataCollector") + ) + conn_dir = collector_dir / "connection" + assert json.loads((conn_dir / "result.json").read_text())["task"] == ( + "LifecycleConnectionManager" + ) + # the connection's files are not mixed into the collector folder itself + assert not (collector_dir / "result.json").exists() + assert not (collector_dir / "events.json").exists() + # nothing is written under a per-manager folder next to the collector + assert not ( + tmp_path + / utils.resolve_log_dir_name("OwnConnectionPlugin") + / utils.resolve_log_dir_name("LifecycleConnectionManager") + ).exists() + + def test_externally_owned_connection_is_left_open_for_its_owner( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, LifecycleConnectionManager) + manager = LifecycleConnectionManager( + system_info=system_info, + logger=logger, + parent="OwnConnectionPlugin", + task_result_hooks=plugin.task_result_hooks, + ) + plugin.connection_manager = manager + + plugin.collect() + + # the plugin did not create the manager, so it must not close it + assert manager.disconnect_calls == 0 + assert manager.connection is not None + conn_dir = self._external_conn_dir(tmp_path) + assert self._saved_result(conn_dir)["status"] == "OK" + + # the owner closes its own connection + manager.disconnect() + assert manager.disconnect_calls == 1 + assert self._saved_result(conn_dir)["status"] == "OK" + self._assert_only_plugin_folder(tmp_path) + + def test_passed_in_connection_closed_and_saved_when_preserve_connection_false( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, LifecycleConnectionManager) + manager = LifecycleConnectionManager( + system_info=system_info, + logger=logger, + parent="OwnConnectionPlugin", + task_result_hooks=plugin.task_result_hooks, + ) + plugin.connection_manager = manager + + plugin.collect(preserve_connection=False) + + assert manager.disconnect_calls == 1 + assert manager.connection is None + saved = self._saved_result(self._external_conn_dir(tmp_path)) + assert saved["status"] == "OK" + self._assert_only_plugin_folder(tmp_path) + + def test_owned_connection_closed_even_when_preserve_connection_true( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, LifecycleConnectionManager) + + plugin.collect(preserve_connection=True) + + assert plugin.connection_manager.disconnect_calls == 1 + + def test_externally_owned_failed_connection_is_not_closed_by_plugin( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, RaisingConnectManager) + manager = RaisingConnectManager( + system_info=system_info, + logger=logger, + parent="OwnConnectionPlugin", + task_result_hooks=plugin.task_result_hooks, + ) + plugin.connection_manager = manager + + plugin.collect() + + assert manager.disconnect_calls == 0 + + def test_manager_built_from_connection_args_is_owned_and_closed( + self, system_info, logger, tmp_path + ): + plugin = _own_connection_plugin(LifecycleConnectionManager)( + system_info=system_info, + logger=logger, + log_path=str(tmp_path), + connection_args={"host": "example"}, + ) + manager = plugin.connection_manager + assert isinstance(manager, LifecycleConnectionManager) + + plugin.collect() + + assert manager.disconnect_calls == 1 + saved = self._saved_result(self._conn_dir(tmp_path)) + assert saved["status"] == "OK" + + @pytest.mark.parametrize( + "collector_cls, expected_status", + [ + (InitRaisingCollector, ExecutionStatus.EXECUTION_FAILURE), + (CollectRaisingCollector, ExecutionStatus.EXECUTION_FAILURE), + (IncompatibleCollector, ExecutionStatus.NOT_RAN), + ], + ) + def test_collector_exception_still_closes_and_saves_connection( + self, system_info, logger, tmp_path, collector_cls, expected_status + ): + plugin = self._build( + system_info, logger, tmp_path, LifecycleConnectionManager, collector_cls + ) + + result = plugin.collect() + + assert result.status == expected_status + manager = plugin.connection_manager + assert manager.disconnect_calls == 1 + assert manager.connection is None + # the connection itself was fine, so it is saved as OK in the plugin folder + saved = self._saved_result(self._conn_dir(tmp_path, collector_cls)) + assert saved["status"] == "OK" + self._assert_only_plugin_folder(tmp_path) + + def test_connect_exception_is_saved_as_failure_and_connection_closed( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, RaisingConnectManager) + + result = plugin.run(analysis=False) + + assert result.result_data.collection_result.status == ExecutionStatus.NOT_RAN + manager = plugin.connection_manager + assert manager.disconnect_calls == 1 + conn_dir = self._conn_dir(tmp_path) + assert self._saved_result(conn_dir)["status"] == "EXECUTION_FAILURE" + events = json.loads((conn_dir / "events.json").read_text()) + # reported once by connect(); closing the connection must not repeat it + assert len(events) == 1 + assert events[0]["priority"] == "CRITICAL" + assert events[0]["data"]["exception_type"] == "RuntimeError" + assert "connect boom" in events[0]["description"] + self._assert_only_plugin_folder(tmp_path) + + def test_connection_manager_creation_failure_does_not_raise( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, RaisingInitManager) + + result = plugin.collect() + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert "bad connection args" in result.message + assert plugin.connection_manager is None + assert list(tmp_path.iterdir()) == [] + + def test_disconnect_exception_still_saves_result_and_does_not_raise( + self, system_info, logger, tmp_path + ): + plugin = self._build(system_info, logger, tmp_path, RaisingDisconnectManager) + + result = plugin.collect() + + assert result.status == ExecutionStatus.OK + assert plugin.connection_manager.disconnect_calls == 1 + saved = self._saved_result(self._conn_dir(tmp_path)) + assert saved["status"] == "OK" + + def test_without_log_path_connection_is_closed_and_nothing_written( + self, system_info, logger, tmp_path, monkeypatch + ): + monkeypatch.chdir(tmp_path) + plugin = _own_connection_plugin(LifecycleConnectionManager)( + system_info=system_info, logger=logger + ) + + result = plugin.collect() + + assert result.status == ExecutionStatus.OK + assert plugin.connection_manager.disconnect_calls == 1 + assert list(tmp_path.iterdir()) == [] + + def test_reconnects_and_saves_again_on_second_collect(self, system_info, logger, tmp_path): + plugin = self._build(system_info, logger, tmp_path, LifecycleConnectionManager) + + plugin.collect() + plugin.collect() + + assert plugin.connection_manager.disconnect_calls == 2 + saved = self._saved_result(self._conn_dir(tmp_path)) + assert saved["status"] == "OK" diff --git a/test/unit/framework/test_file_artifact.py b/test/unit/framework/test_file_artifact.py index 991fbf48..87e048aa 100644 --- a/test/unit/framework/test_file_artifact.py +++ b/test/unit/framework/test_file_artifact.py @@ -72,3 +72,15 @@ def test_log_model_binary(tmp_path: Path): output_path = tmp_path / "binary.bin" assert output_path.exists() assert output_path.read_bytes() == binary_data + + +def test_log_model_absolute_filename_stays_in_log_path(tmp_path: Path): + """An artifact filename that is an absolute path must not write outside the log dir.""" + log_dir = tmp_path / "logs" + log_dir.mkdir() + outside = tmp_path / "outside.txt" + + artifact = TextFileArtifact(filename=str(outside), contents="leaked") + artifact.log_model(str(log_dir)) + + assert not outside.exists(), "log_model wrote outside of the provided log path" diff --git a/test/unit/framework/test_filesystemloghook.py b/test/unit/framework/test_filesystemloghook.py new file mode 100644 index 00000000..b31e6b08 --- /dev/null +++ b/test/unit/framework/test_filesystemloghook.py @@ -0,0 +1,135 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import json + +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.models import DataModel, Event, TaskResult +from nodescraper.taskresulthooks import FileSystemLogHook + + +class _Model(DataModel): + value: str = "x" + + +def _result(task="MyTask", parent="MyParent", message="", events=()): + return TaskResult( + task=task, + parent=parent, + status=ExecutionStatus.OK, + message=message, + events=[ + Event( + reporter="test", + category=EventCategory.RUNTIME, + description=d, + priority=EventPriority.WARNING, + ) + for d in events + ], + ) + + +def _names(path): + return sorted(p.name for p in path.iterdir()) + + +def _read(path): + return json.loads(path.read_text()) + + +def test_first_result_uses_plain_file_names(tmp_path): + hook = FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False) + + hook.process_result(_result(events=["one"])) + + assert _names(tmp_path) == ["events.json", "result.json"] + + +def test_later_results_in_same_directory_get_an_integer_and_keep_earlier_logs(tmp_path): + hook = FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False) + + hook.process_result(_result(message="first", events=["one"])) + hook.process_result(_result(message="second", events=["two"])) + hook.process_result(_result(message="third", events=["three"])) + + assert _names(tmp_path) == [ + "events(1).json", + "events(2).json", + "events.json", + "result(1).json", + "result(2).json", + "result.json", + ] + assert _read(tmp_path / "result.json")["message"] == "first" + assert _read(tmp_path / "result(1).json")["message"] == "second" + assert _read(tmp_path / "result(2).json")["message"] == "third" + # events are not merged across results: each result keeps its own events file + assert [e["description"] for e in _read(tmp_path / "events.json")] == ["one"] + assert [e["description"] for e in _read(tmp_path / "events(1).json")] == ["two"] + assert [e["description"] for e in _read(tmp_path / "events(2).json")] == ["three"] + + +def test_result_and_events_share_the_same_index_when_earlier_result_had_no_events(tmp_path): + hook = FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False) + + hook.process_result(_result(message="first")) + hook.process_result(_result(message="second", events=["only"])) + + assert _names(tmp_path) == ["events(1).json", "result(1).json", "result.json"] + + +def test_results_for_different_tasks_do_not_get_an_integer(tmp_path): + hook = FileSystemLogHook(log_base_path=str(tmp_path)) + + hook.process_result(_result(task="TaskA")) + hook.process_result(_result(task="TaskB")) + + assert (tmp_path / "my_parent" / "task_a" / "result.json").is_file() + assert (tmp_path / "my_parent" / "task_b" / "result.json").is_file() + assert not list(tmp_path.rglob("result(1).json")) + + +def test_a_new_hook_instance_does_not_overwrite_logs_already_on_disk(tmp_path): + FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False).process_result( + _result(message="old") + ) + + FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False).process_result( + _result(message="new") + ) + + assert _read(tmp_path / "result.json")["message"] == "old" + assert _read(tmp_path / "result(1).json")["message"] == "new" + + +def test_data_model_is_logged_next_to_each_result_without_overwriting(tmp_path): + hook = FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False) + + hook.process_result(_result(), data=_Model(value="a")) + hook.process_result(_result(), data=_Model(value="b")) + + assert _read(tmp_path / "_model.json")["value"] == "a" + assert _read(tmp_path / "_model(1).json")["value"] == "b" diff --git a/test/unit/framework/test_match_ignore.py b/test/unit/framework/test_match_ignore.py index bf29752e..c2e7ccaa 100644 --- a/test/unit/framework/test_match_ignore.py +++ b/test/unit/framework/test_match_ignore.py @@ -110,3 +110,9 @@ def test_should_ignore_match_mce_banks_only_when_all_banks_ignored(): def test_extract_mce_bank_from_line(): line = "[Hardware Error]: Machine Check: CPU0 MC21_STATUS[0xcafe|CE|Misc]: 0x0" assert extract_mce_bank_from_line(line) == 21 + + +def test_parse_mce_bank_spec_rejects_negative_bank_string(): + """A negative bank given as a string must be reported as an invalid bank number.""" + with pytest.raises(ValueError, match="Invalid MCE bank"): + parse_mce_bank_spec(["-5"]) diff --git a/test/unit/framework/test_plugin_execution_target.py b/test/unit/framework/test_plugin_execution_target.py new file mode 100644 index 00000000..91ef30be --- /dev/null +++ b/test/unit/framework/test_plugin_execution_target.py @@ -0,0 +1,90 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from framework.common.shared_utils import DummyDataModel + +from nodescraper.base.inbanddataplugin import InBandDataPlugin +from nodescraper.base.oobanddataplugin import OOBandDataPlugin +from nodescraper.enums import SystemLocation +from nodescraper.helpers.plugin_execution_target import ( + format_in_band_target_summary, + format_plugin_execution_target, +) +from nodescraper.models import SystemInfo + + +class _DummyOobPlugin(OOBandDataPlugin): + DATA_MODEL = DummyDataModel + + +class _DummyInBandPlugin(InBandDataPlugin): + DATA_MODEL = DummyDataModel + + +def test_format_in_band_target_summary_local(): + summary = format_in_band_target_summary(SystemInfo(name="workstation01")) + assert summary == "In-band default: local host (workstation01)" + + +def test_format_in_band_target_summary_remote(): + summary = format_in_band_target_summary( + SystemInfo(name="workstation01", location=SystemLocation.REMOTE), + connection_configs={ + "InBandConnectionManager": { + "hostname": "sut.example.com", + } + }, + ) + assert summary == "In-band default: remote host via SSH (sut.example.com)" + + +def test_format_plugin_execution_target_redfish(): + target = format_plugin_execution_target( + _DummyOobPlugin, + system_info=SystemInfo(name="workstation01"), + connection_configs={ + "RedfishConnectionManager": {"host": "bmc.example.com"}, + }, + ) + assert target == "Execution target: BMC via Redfish OOB (bmc.example.com)" + + +def test_format_plugin_execution_target_inband_local(): + target = format_plugin_execution_target( + _DummyInBandPlugin, + system_info=SystemInfo(name="workstation01", location=SystemLocation.LOCAL), + ) + assert target == "Execution target: local host (workstation01)" + + +def test_format_plugin_execution_target_inband_remote(): + target = format_plugin_execution_target( + _DummyInBandPlugin, + system_info=SystemInfo(name="workstation01", location=SystemLocation.REMOTE), + connection_configs={ + "InBandConnectionManager": {"hostname": "sut.example.com"}, + }, + ) + assert target == "Execution target: remote host via SSH (sut.example.com)" diff --git a/test/unit/framework/test_plugin_executor.py b/test/unit/framework/test_plugin_executor.py index 494551ce..a4c4c77d 100644 --- a/test/unit/framework/test_plugin_executor.py +++ b/test/unit/framework/test_plugin_executor.py @@ -23,17 +23,35 @@ # SOFTWARE. # ############################################################################### +import abc +import json +import logging +from typing import Optional +from unittest.mock import MagicMock, Mock, patch + import pytest from framework.common.shared_utils import DummyDataModel, MockConnectionManager from pydantic import BaseModel -from nodescraper.enums import ExecutionStatus +from nodescraper.base.oobsshdataplugin import OOBSSHDataPlugin +from nodescraper.connection.inband.inbandmanager import InBandConnectionManager +from nodescraper.connection.oob_ssh import OobSshConnectionManager +from nodescraper.enums import ExecutionStatus, OSFamily, SystemLocation from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.systeminteraction import SystemInteractionLevel -from nodescraper.interfaces import PluginInterface -from nodescraper.models import PluginConfig, PluginResult +from nodescraper.interfaces import ( + DataPlugin, + PluginInterface, + PluginResultCollator, + TaskResultHook, +) +from nodescraper.interfaces.datacollectortask import DataCollector +from nodescraper.models import PluginConfig, PluginResult, SystemInfo, TaskResult +from nodescraper.models.postactioncondition import PostActionCondition +from nodescraper.models.postactionpluginconfig import PostActionPluginConfig from nodescraper.pluginexecutor import PluginExecutor from nodescraper.pluginregistry import PluginRegistry +from nodescraper.utils import resolve_log_dir_name class DummyArgs(BaseModel): @@ -42,12 +60,13 @@ class DummyArgs(BaseModel): class TestPluginA(PluginInterface[MockConnectionManager, None]): + __test__ = False # Tells pytest to ignore this class + CONNECTION_TYPE = MockConnectionManager COLLECTOR_ARGS = DummyArgs(foo="initial") ANALYZER_ARGS = DummyArgs(foo="initial") collection = False analysis = False - preserve_connection = False data = DummyDataModel(some_version="1") max_event_priority_level = EventPriority.INFO system_interaction_level = SystemInteractionLevel.PASSIVE @@ -59,6 +78,8 @@ def run(self): class TestPluginB(PluginInterface[MockConnectionManager, None]): + __test__ = False # Tells pytest to ignore this class + CONNECTION_TYPE = MockConnectionManager def run(self, test_arg=None): @@ -67,10 +88,38 @@ def run(self, test_arg=None): ) +class PostActionPlugin(PluginInterface[MockConnectionManager, None]): + """Minimal plugin used as a post-action target in tests.""" + + __test__ = False # Tells pytest to ignore this class + + CONNECTION_TYPE = MockConnectionManager + + def run(self, **kwargs): + return PluginResult(source="PostActionPlugin", status=ExecutionStatus.OK) + + +class TestPluginCapture(PluginInterface[MockConnectionManager, None]): + """Records kwargs passed to run() for merge-order regression tests.""" + + __test__ = False # Tells pytest to ignore this class + + CONNECTION_TYPE = MockConnectionManager + last_run_kwargs: Optional[dict] = None + + def run(self, **kwargs): + TestPluginCapture.last_run_kwargs = kwargs + return PluginResult(source="TestPluginCapture", status=ExecutionStatus.OK) + + @pytest.fixture def plugin_registry(): registry = PluginRegistry() - registry.plugins = {"TestPluginA": TestPluginA, "TestPluginB": TestPluginB} + registry.plugins = { + "TestPluginA": TestPluginA, + "TestPluginB": TestPluginB, + "PostActionPlugin": PostActionPlugin, + } registry.connection_managers = {"MockConnectionManager": MockConnectionManager} return registry @@ -137,12 +186,106 @@ def test_queue_callback(plugin_registry): assert results[1].status == ExecutionStatus.OK +def test_merge_plugin_run_args_plugin_overrides_global(): + global_run_args = { + "system_interaction_level": "INTERACTIVE", + "collection_args": {"foo": "global", "shared": "global"}, + "analysis_args": {"bar": "global"}, + } + plugin_args = { + "system_interaction_level": "PASSIVE", + "collection_args": {"foo": "plugin"}, + } + + merged = PluginExecutor.merge_plugin_run_args(plugin_args, global_run_args) + + assert merged["system_interaction_level"] == "PASSIVE" + assert merged["collection_args"] == {"foo": "plugin", "shared": "global"} + assert merged["analysis_args"] == {"bar": "global"} + + +@pytest.mark.parametrize( + "plugin_args,global_run_args,expected_key,expected_value", + [ + ( + {"system_interaction_level": SystemInteractionLevel.PASSIVE}, + {"system_interaction_level": SystemInteractionLevel.INTERACTIVE}, + "system_interaction_level", + SystemInteractionLevel.PASSIVE, + ), + ( + {"system_interaction_level": "PASSIVE"}, + {"system_interaction_level": "INTERACTIVE"}, + "system_interaction_level", + "PASSIVE", + ), + ( + {"analysis": False, "collection": True}, + {"analysis": True, "collection": True}, + "analysis", + False, + ), + ( + {}, + {"system_interaction_level": SystemInteractionLevel.INTERACTIVE}, + "system_interaction_level", + SystemInteractionLevel.INTERACTIVE, + ), + ], +) +def test_merge_plugin_run_args_plugin_wins_on_conflict( + plugin_args, global_run_args, expected_key, expected_value +): + merged = PluginExecutor.merge_plugin_run_args(plugin_args, global_run_args) + assert merged[expected_key] == expected_value + + +def test_merge_plugin_run_args_analysis_args_plugin_overrides_global(): + merged = PluginExecutor.merge_plugin_run_args( + {"analysis_args": {"exp_speed": 4, "shared": "plugin"}}, + {"analysis_args": {"exp_speed": 5, "shared": "global", "exp_width": 16}}, + ) + assert merged["analysis_args"] == { + "exp_speed": 4, + "shared": "plugin", + "exp_width": 16, + } + + +def test_run_queue_plugin_system_interaction_level_overrides_global(plugin_registry): + """Per-plugin run() kwargs must win over global_args (PciePlugin PASSIVE case).""" + plugin_registry.plugins["TestPluginCapture"] = TestPluginCapture + TestPluginCapture.last_run_kwargs = None + + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + global_args={"system_interaction_level": SystemInteractionLevel.INTERACTIVE}, + plugins={ + "TestPluginCapture": { + "system_interaction_level": SystemInteractionLevel.PASSIVE, + } + }, + ) + ], + plugin_registry=plugin_registry, + ) + results = executor.run_queue() + + assert len(results) == 1 + assert results[0].source == "TestPluginCapture" + assert TestPluginCapture.last_run_kwargs is not None + assert ( + TestPluginCapture.last_run_kwargs["system_interaction_level"] + == SystemInteractionLevel.PASSIVE + ) + + def test_apply_global_args_to_plugin(): plugin = TestPluginA() global_args = { "collection": True, "analysis": True, - "preserve_connection": True, "data": {"some_version": "1"}, "max_event_priority_level": 4, "system_interaction_level": "INTERACTIVE", @@ -155,7 +298,6 @@ def test_apply_global_args_to_plugin(): assert run_payload["collection"] is True assert run_payload["analysis"] is True - assert run_payload["preserve_connection"] is True assert run_payload["data"]["some_version"] == "1" assert run_payload["max_event_priority_level"] == 4 assert run_payload["system_interaction_level"] == "INTERACTIVE" @@ -201,3 +343,1091 @@ def hook(res: PluginResult) -> None: ) executor.run_queue() assert seen == ["testB"] + + +# --------------------------------------------------------------------------- +# merge_configs: post_action_plugins concatenation +# --------------------------------------------------------------------------- + + +def test_merge_configs_concatenates_post_action_plugins(): + """post_action_plugins lists from multiple configs are concatenated.""" + pa1 = PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + pa2 = PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="WARNING")], + ) + configs = [ + PluginConfig(post_action_plugins=[pa1]), + PluginConfig(post_action_plugins=[pa2]), + ] + merged = PluginExecutor.merge_configs(configs) + assert len(merged.post_action_plugins) == 2 + assert merged.post_action_plugins[0] is pa1 + assert merged.post_action_plugins[1] is pa2 + + +def test_merge_configs_empty_post_action_plugins(): + """Merging configs with no post_action_plugins yields an empty list.""" + configs = [PluginConfig(plugins={"TestPluginB": {}})] + merged = PluginExecutor.merge_configs(configs) + assert merged.post_action_plugins == [] + + +def test_plugin_config_merge_concatenates_post_action_plugins(): + """PluginConfig.merge() (recipe merge) also concatenates post_action_plugins.""" + pa1 = PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + pa2 = PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="WARNING")], + ) + merged = PluginConfig.merge( + PluginConfig(plugins={"TestPluginA": {}}, post_action_plugins=[pa1]), + PluginConfig(plugins={"TestPluginB": {}}, post_action_plugins=[pa2]), + ) + # Plugins from both configs are present + assert "TestPluginA" in merged.plugins + assert "TestPluginB" in merged.plugins + # Post-action plugins from both configs are concatenated + assert len(merged.post_action_plugins) == 2 + assert merged.post_action_plugins[0] is pa1 + assert merged.post_action_plugins[1] is pa2 + + +def test_plugin_config_merge_empty_post_action_plugins(): + """PluginConfig.merge() with no post_action_plugins yields an empty list.""" + merged = PluginConfig.merge( + PluginConfig(plugins={"TestPluginA": {}}), + PluginConfig(plugins={"TestPluginB": {}}), + ) + assert merged.post_action_plugins == [] + + +# --------------------------------------------------------------------------- +# run_queue: post-action execution +# --------------------------------------------------------------------------- + + +def test_post_action_runs_when_condition_met(plugin_registry): + """Post-action plugin fires when primary result meets the condition. + + TestPluginA returns ERROR and also queues TestPluginB via _update_queue, + so the primary run produces two results (testA + testB). The post-action + fires on the ERROR status, giving a total of 3 results. + """ + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginA": {}}, # TestPluginA always returns ERROR + post_action_plugins=[ + PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + ], + ) + ], + plugin_registry=plugin_registry, + ) + results = executor.run_queue() + + sources = [r.source for r in results] + assert "testA" in sources + assert "testB" in sources # queued by TestPluginA via _update_queue + assert "PostActionPlugin" in sources + assert len(results) == 3 + + +def test_post_action_does_not_run_when_condition_not_met(plugin_registry): + """Post-action plugin is skipped when no primary result meets the condition.""" + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginB": {}}, # TestPluginB always returns OK + post_action_plugins=[ + PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + ], + ) + ], + plugin_registry=plugin_registry, + ) + results = executor.run_queue() + + assert len(results) == 1 + assert results[0].source == "testB" + + +def test_post_action_result_appended_to_run_queue_return(plugin_registry): + """The post-action PluginResult is present in the list returned by run_queue().""" + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginA": {}}, + post_action_plugins=[ + PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + ], + ) + ], + plugin_registry=plugin_registry, + ) + results = executor.run_queue() + + post_action_results = [r for r in results if r.source == "PostActionPlugin"] + assert len(post_action_results) == 1 + assert post_action_results[0].status == ExecutionStatus.OK + + +def test_post_action_result_hooks_called(plugin_registry): + """plugin_run_result_hooks are invoked for post-action results too.""" + seen: list[str] = [] + + def hook(res: PluginResult) -> None: + seen.append(res.source) + + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginA": {}}, + post_action_plugins=[ + PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + ], + ) + ], + plugin_registry=plugin_registry, + plugin_run_result_hooks=[hook], + ) + executor.run_queue() + + assert "testA" in seen + assert "PostActionPlugin" in seen + + +def test_multiple_post_actions_selective_firing(plugin_registry): + """With two post-actions, only the one whose condition is met runs. + + testA returns ERROR (value=40). The first condition requires ERROR (40 >= 40) → fires. + The second condition requires EXECUTION_FAILURE (50); ERROR (40) < 50 → does not fire. + """ + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginA": {}}, # returns ERROR + post_action_plugins=[ + PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], # fires: ERROR >= ERROR + ), + PostActionPluginConfig( + plugin="PostActionPlugin", + # does not fire: testA is ERROR (40) < EXECUTION_FAILURE (50) + conditions=[ + PostActionCondition(plugin="testA", status="EXECUTION_FAILURE") + ], + ), + ], + ) + ], + plugin_registry=plugin_registry, + ) + results = executor.run_queue() + + post_action_results = [r for r in results if r.source == "PostActionPlugin"] + assert len(post_action_results) == 1 # only first post-action fired + + +def test_post_action_invalid_plugin_name_logs_error_and_continues(plugin_registry, caplog): + """An unregistered post-action plugin name is logged as an error; run completes.""" + with caplog.at_level(logging.ERROR): + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginA": {}}, + post_action_plugins=[ + PostActionPluginConfig( + plugin="NonExistentPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + ], + ) + ], + plugin_registry=plugin_registry, + ) + results = executor.run_queue() + + # Primary result still returned; no exception raised + assert any(r.source == "testA" for r in results) + assert any("NonExistentPlugin" in record.message for record in caplog.records) + + +def test_closing_connections_logged_after_post_actions(plugin_registry, caplog): + """'Closing connections' log appears after the post-action run log, not before.""" + with caplog.at_level(logging.INFO): + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"TestPluginA": {}}, + post_action_plugins=[ + PostActionPluginConfig( + plugin="PostActionPlugin", + conditions=[PostActionCondition(status="ERROR")], + ) + ], + ) + ], + plugin_registry=plugin_registry, + ) + executor.run_queue() + + messages = [r.message for r in caplog.records] + + post_action_idx = next( + (i for i, m in enumerate(messages) if "post-action plugin" in m.lower()), None + ) + closing_idx = next( + (i for i, m in enumerate(messages) if "closing connections" in m.lower()), None + ) + + assert post_action_idx is not None, "Expected a post-action log message" + assert closing_idx is not None, "Expected a 'Closing connections' log message" + assert ( + post_action_idx < closing_idx + ), "'Closing connections' must be logged after the post-action plugin runs" + + +def test_discover_os_info_detects_linux(system_info): + """discover_os_info() should detect Linux OS when uname succeeds.""" + from unittest.mock import MagicMock + + from nodescraper.connection.inband import CommandArtifact + from nodescraper.connection.inband.inbandmanager import InBandConnectionManager + from nodescraper.enums import OSFamily + from nodescraper.models.taskresult import TaskResult + + # Mock the connection to return Linux + mock_conn = MagicMock() + mock_conn.run_command.return_value = CommandArtifact( + command="uname -s", + stdout="Linux", + stderr="", + exit_code=0, + ) + + # Mock InBandConnectionManager instance + mock_manager = MagicMock(spec=InBandConnectionManager) + mock_manager.connection = mock_conn + mock_manager.connect.return_value = TaskResult(status=ExecutionStatus.OK) + mock_manager.disconnect.return_value = None + + # Create a mock class that returns our mock manager instance + mock_class = Mock(return_value=mock_manager) + mock_class.__name__ = "InBandConnectionManager" + + # Store original and patch + import nodescraper.pluginexecutor + + original = nodescraper.pluginexecutor.InBandConnectionManager + nodescraper.pluginexecutor.InBandConnectionManager = mock_class + + try: + # Create executor + executor = PluginExecutor( + plugin_configs=[PluginConfig(plugins={})], + system_info=system_info, + ) + + # Run OS discovery + executor.discover_os_info() + finally: + # Restore original + nodescraper.pluginexecutor.InBandConnectionManager = original + + # Verify Linux was detected + assert system_info.os_family == OSFamily.LINUX + mock_conn.run_command.assert_called_once_with("uname -s") + + +def test_discover_os_info_detects_arista_eos(system_info): + """discover_os_info() should detect Arista EOS when uname fails but Arista command succeeds.""" + import json + from unittest.mock import MagicMock + + from nodescraper.connection.inband import CommandArtifact + from nodescraper.connection.inband.inbandmanager import InBandConnectionManager + from nodescraper.enums import OSFamily + from nodescraper.models.taskresult import TaskResult + + arista_version = { + "mfgName": "Arista Networks", + "version": "4.32.1F", + "modelName": "DCS-7280CR3-32P4", + } + + # Mock the connection to fail uname but succeed with Arista command + mock_conn = MagicMock() + mock_conn.run_command.side_effect = [ + CommandArtifact(command="uname -s", stdout="", stderr="invalid", exit_code=1), + CommandArtifact( + command="show version | json | no-more", + stdout=json.dumps(arista_version), + stderr="", + exit_code=0, + ), + ] + + # Mock InBandConnectionManager instance + mock_manager = MagicMock(spec=InBandConnectionManager) + mock_manager.connection = mock_conn + mock_manager.connect.return_value = TaskResult(status=ExecutionStatus.OK) + mock_manager.disconnect.return_value = None + + # Create a mock class that returns our mock manager instance + mock_class = Mock(return_value=mock_manager) + mock_class.__name__ = "InBandConnectionManager" + + # Store original and patch + import nodescraper.pluginexecutor + + original = nodescraper.pluginexecutor.InBandConnectionManager + nodescraper.pluginexecutor.InBandConnectionManager = mock_class + + try: + # Create executor + executor = PluginExecutor( + plugin_configs=[PluginConfig(plugins={})], + system_info=system_info, + ) + + # Run OS discovery + executor.discover_os_info() + finally: + # Restore original + nodescraper.pluginexecutor.InBandConnectionManager = original + + # Verify Arista EOS was detected + assert system_info.os_family == OSFamily.EOS + assert system_info.platform == "Arista EOS" + assert system_info.metadata["os_version"] == "4.32.1F" + assert system_info.metadata["device_model"] == "DCS-7280CR3-32P4" + + +def test_discover_os_info_skips_when_connection_fails(system_info, caplog): + """discover_os_info() should skip detection and log when InBandConnectionManager fails to connect.""" + from unittest.mock import MagicMock + + from nodescraper.connection.inband.inbandmanager import InBandConnectionManager + from nodescraper.enums import OSFamily + from nodescraper.models.taskresult import TaskResult + + # Mock InBandConnectionManager instance to fail connection + mock_manager = MagicMock(spec=InBandConnectionManager) + mock_manager.connect.return_value = TaskResult( + status=ExecutionStatus.ERROR, message="Connection failed" + ) + mock_manager.disconnect.return_value = None + + # Create a mock class that returns our mock manager instance + mock_class = Mock(return_value=mock_manager) + mock_class.__name__ = "InBandConnectionManager" + + # Store original and patch + import nodescraper.pluginexecutor + + original = nodescraper.pluginexecutor.InBandConnectionManager + nodescraper.pluginexecutor.InBandConnectionManager = mock_class + + try: + # Create executor + executor = PluginExecutor( + plugin_configs=[PluginConfig(plugins={})], + system_info=system_info, + ) + + # Run OS discovery + with caplog.at_level(logging.INFO): + executor.discover_os_info() + finally: + # Restore original + nodescraper.pluginexecutor.InBandConnectionManager = original + + # Verify OS detection was skipped + assert system_info.os_family == OSFamily.LINUX # Default from fixture + assert any("Skipping OS discovery" in record.message for record in caplog.records) + + +class RaisingPlugin(PluginInterface[MockConnectionManager, None]): + __test__ = False # Tells pytest to ignore this class + + CONNECTION_TYPE = MockConnectionManager + + def run(self, **kwargs): + raise RuntimeError("plugin boom") + + +class FailingInitConnectionManager(MockConnectionManager): + def __init__(self, *args, **kwargs): + raise ValueError("bad connection args") + + +class FailingInitPlugin(PluginInterface[FailingInitConnectionManager, None]): + __test__ = False # Tells pytest to ignore this class + + CONNECTION_TYPE = FailingInitConnectionManager + + def run(self, **kwargs): + return PluginResult(source="FailingInitPlugin", status=ExecutionStatus.OK) + + +class RaisingCollator(PluginResultCollator): + def collate_results(self, plugin_results, connection_results, **kwargs): + raise RuntimeError("collator boom") + + +def _isolated_registry(plugins: dict, managers: dict) -> PluginRegistry: + registry = PluginRegistry( + load_internal_plugins=False, + load_entry_point_plugins=False, + load_entry_point_connection_managers=False, + ) + registry.plugins = plugins + registry.connection_managers = managers + return registry + + +def _read_events(path) -> list[dict]: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def test_connection_manager_instantiation_failure_is_logged_and_queue_continues(tmp_path, caplog): + """A connection manager that cannot be built is logged and skipped, not fatal for the queue.""" + registry = _isolated_registry( + {"FailingInitPlugin": FailingInitPlugin, "TestPluginB": TestPluginB}, + { + "FailingInitConnectionManager": FailingInitConnectionManager, + "MockConnectionManager": MockConnectionManager, + }, + ) + executor = PluginExecutor( + plugin_configs=[PluginConfig(plugins={"FailingInitPlugin": {}, "TestPluginB": {}})], + plugin_registry=registry, + system_info=SystemInfo(os_family=OSFamily.LINUX), + log_path=str(tmp_path), + ) + + with caplog.at_level(logging.ERROR): + results = executor.run_queue() + + assert [r.source for r in results] == ["testB"] + assert any( + "FailingInitConnectionManager" in r.message and "bad connection args" in r.message + for r in caplog.records + ) + assert not ( + tmp_path / "connection" / resolve_log_dir_name("FailingInitConnectionManager") + ).exists() + + +def test_connections_closed_when_cleanup_step_raises(tmp_path): + """Connections are closed even if a collator raises.""" + registry = _isolated_registry( + {"RaisingPlugin": RaisingPlugin}, + {"MockConnectionManager": MockConnectionManager}, + ) + registry.result_collators = {"RaisingCollator": RaisingCollator} + executor = PluginExecutor( + plugin_configs=[ + PluginConfig(plugins={"RaisingPlugin": {}}, result_collators={"RaisingCollator": {}}) + ], + plugin_registry=registry, + system_info=SystemInfo(os_family=OSFamily.LINUX), + log_path=str(tmp_path), + ) + + with patch.object(MockConnectionManager, "disconnect") as disconnect: + with pytest.raises(RuntimeError, match="collator boom"): + executor.run_queue() + + assert disconnect.call_count == 2 + + +def test_discover_os_info_result_saved_separately_from_connection_results(tmp_path): + """The OS discovery connection result goes to os_discovery/, not to connection/.""" + executor = PluginExecutor( + plugin_configs=[PluginConfig(plugins={})], + plugin_registry=_isolated_registry({}, {}), + system_info=SystemInfo(location=SystemLocation.REMOTE), + log_path=str(tmp_path), + ) + + executor.discover_os_info() + + events = _read_events(tmp_path / "os_discovery" / "events.json") + assert events[0]["priority"] == "CRITICAL" + assert events[0]["data"]["parent"] == "os_discovery" + assert not (tmp_path / "connection").exists() + + +# --------------------------------------------------------------------------- +# Connection cleanup when exceptions are raised at different points of a run +# --------------------------------------------------------------------------- + + +class TrackedManager(MockConnectionManager): + """Starts disconnected, connects on demand and records every disconnect.""" + + calls: list = [] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.connection = None + self.result = self._init_result() + + def connect(self): + TrackedManager.calls.append(("connect", type(self).__name__)) + self.connection = MagicMock() + self.result.status = ExecutionStatus.OK + return self.result + + def disconnect(self): + TrackedManager.calls.append(("disconnect", type(self).__name__)) + self.connection = None + + +class ManagerA(TrackedManager): + pass + + +class ManagerB(TrackedManager): + pass + + +class AbstractManager(TrackedManager): + @abc.abstractmethod + def extra(self): ... + + +def _make_plugin(manager_cls, name, outcome="ok"): + """Plugin that connects, then returns OK / ERROR or raises.""" + + class _Plugin(PluginInterface[manager_cls, None]): + CONNECTION_TYPE = manager_cls + + def run(self, **kwargs): + TrackedManager.calls.append(("run", name)) + self.connection_manager.connect() + if outcome == "raise": + raise RuntimeError(f"{name} boom") + status = ExecutionStatus.ERROR if outcome == "error" else ExecutionStatus.OK + return PluginResult(source=name, status=status) + + _Plugin.__name__ = name + return _Plugin + + +class InitFailsPlugin(PluginInterface[TrackedManager, None]): + __test__ = False + + CONNECTION_TYPE = TrackedManager + + def __init__(self, *args, **kwargs): + raise RuntimeError("init boom") + + def run(self, **kwargs): + return PluginResult(source="init_fails", status=ExecutionStatus.OK) + + +@pytest.fixture +def tracked(): + TrackedManager.calls = [] + yield TrackedManager.calls + TrackedManager.calls = [] + + +def _executor(tmp_path, plugins, managers, config_plugins, **kwargs): + config_kwargs = kwargs.pop("config_kwargs", {}) + registry = _isolated_registry(plugins, managers) + registry.result_collators = kwargs.pop("collators", {}) + return PluginExecutor( + plugin_configs=[PluginConfig(plugins=config_plugins, **config_kwargs)], + plugin_registry=registry, + system_info=kwargs.pop("system_info", SystemInfo(os_family=OSFamily.LINUX)), + log_path=str(tmp_path), + **kwargs, + ) + + +def _conn_dir(tmp_path, manager_cls): + return tmp_path / "connection" / resolve_log_dir_name(manager_cls.__name__) + + +def _disconnects(calls, name="TrackedManager"): + return [c for c in calls if c == ("disconnect", name)] + + +def _failing_and_ok_plugins(): + return ( + {"Failing": InitFailsPlugin, "Ok": _make_plugin(TrackedManager, "Ok")}, + {"TrackedManager": TrackedManager}, + {"Failing": {}, "Ok": {}}, + ) + + +@pytest.mark.parametrize("spot", ["plugin_init", "global_args", "plugin_run", "result_hook"]) +def test_exception_at_each_plugin_spot_closes_connection(tmp_path, tracked, spot): + """Whichever step of running a plugin raises, its connection is closed right away + and the next plugin still runs.""" + plugins, managers, config = _failing_and_ok_plugins() + if spot != "plugin_init": + plugins["Failing"] = _make_plugin( + TrackedManager, "Failing", "raise" if spot == "plugin_run" else "ok" + ) + + def hook(result): + if result.source == "Failing": + raise RuntimeError("hook boom") + + executor = _executor( + tmp_path, + plugins, + managers, + config, + plugin_run_result_hooks=[hook] if spot == "result_hook" else None, + ) + + if spot == "global_args": + real = PluginExecutor.apply_global_args_to_plugin + + def flaky(self, plugin_inst, plugin_class, global_args): + if plugin_class.__name__ == "Failing": + raise TypeError("bad global args") + return real(self, plugin_inst, plugin_class, global_args) + + with patch.object(PluginExecutor, "apply_global_args_to_plugin", flaky): + results = executor.run_queue() + else: + results = executor.run_queue() + + assert "Ok" in [r.source for r in results] + # closed straight after the failure (before the next plugin ran) and again during cleanup + assert len(_disconnects(tracked)) == 2 + assert tracked.index(("disconnect", "TrackedManager")) < tracked.index(("run", "Ok")) + + +def test_invalid_global_args_skips_plugin_without_connection_failure(tmp_path, tracked): + """A ValueError from global args skips the plugin: no connection failure, closed at cleanup.""" + executor = _executor( + tmp_path, + {"Ok": _make_plugin(TrackedManager, "Ok")}, + {"TrackedManager": TrackedManager}, + {"Ok": {}}, + ) + + with patch.object( + PluginExecutor, "apply_global_args_to_plugin", side_effect=ValueError("bad value") + ): + results = executor.run_queue() + + assert results == [] + assert _disconnects(tracked) == [("disconnect", "TrackedManager")] + assert not (tmp_path / "connection").exists() + + +def test_unresolvable_connection_manager_is_logged_and_queue_continues(tmp_path, tracked, caplog): + """An abstract / unregistered CONNECTION_TYPE is logged as an error and the plugin skipped.""" + abstract_plugin = _make_plugin(AbstractManager, "Abstract") + executor = _executor( + tmp_path, + {"Abstract": abstract_plugin, "Ok": _make_plugin(TrackedManager, "Ok")}, + {"TrackedManager": TrackedManager}, + {"Abstract": {}, "Ok": {}}, + ) + + with caplog.at_level(logging.ERROR): + results = executor.run_queue() + + assert [r.source for r in results] == ["Ok"] + assert any("AbstractManager" in r.message and "Abstract" in r.message for r in caplog.records) + assert not _conn_dir(tmp_path, AbstractManager).exists() + + +def test_oob_plugin_without_redfish_config_is_logged_and_queue_continues(tmp_path, tracked, caplog): + """An OOB plugin without a RedfishConnectionManager config logs an error and is skipped.""" + + class NoConfigOobPlugin(OOBSSHDataPlugin): + DATA_MODEL = DummyDataModel + + executor = _executor( + tmp_path, + {"NoConfigOobPlugin": NoConfigOobPlugin, "Ok": _make_plugin(TrackedManager, "Ok")}, + {"TrackedManager": TrackedManager}, + {"NoConfigOobPlugin": {}, "Ok": {}}, + ) + + with caplog.at_level(logging.ERROR): + results = executor.run_queue() + + assert [r.source for r in results] == ["Ok"] + assert any("RedfishConnectionManager" in r.message for r in caplog.records) + assert not _conn_dir(tmp_path, OobSshConnectionManager).exists() + + +def test_post_action_plugin_exception_closes_connection(tmp_path, tracked): + """A post-action plugin raising is handled like a primary plugin failure.""" + executor = _executor( + tmp_path, + { + "Primary": _make_plugin(TrackedManager, "Primary", "error"), + "PostAction": _make_plugin(TrackedManager, "PostAction", "raise"), + }, + {"TrackedManager": TrackedManager}, + {"Primary": {}}, + config_kwargs={ + "post_action_plugins": [ + PostActionPluginConfig( + plugin="PostAction", conditions=[PostActionCondition(status="ERROR")] + ) + ] + }, + ) + + results = executor.run_queue() + + assert [r.source for r in results] == ["Primary"] + assert len(_disconnects(tracked)) == 2 + + +def test_queue_level_exception_still_saves_and_closes_connections(tmp_path, tracked): + """An exception escaping _run_single_plugin aborts the queue but connections are cleaned up.""" + plugins = { + "First": _make_plugin(TrackedManager, "First"), + "Second": _make_plugin(TrackedManager, "Second"), + } + executor = _executor( + tmp_path, plugins, {"TrackedManager": TrackedManager}, {"First": {}, "Second": {}} + ) + real = PluginExecutor._run_single_plugin + + def flaky(self, plugin_name, *args, **kwargs): + if plugin_name == "Second": + raise RuntimeError("queue boom") + return real(self, plugin_name, *args, **kwargs) + + with patch.object(PluginExecutor, "_run_single_plugin", flaky): + results = executor.run_queue() + + assert [r.source for r in results] == ["First"] + assert ("run", "Second") not in tracked + assert _disconnects(tracked) == [("disconnect", "TrackedManager")] + with open(_conn_dir(tmp_path, TrackedManager) / "result.json", encoding="utf-8") as f: + assert json.load(f)["status"] == "OK" + + +def test_post_action_evaluation_exception_still_closes_connections(tmp_path, tracked): + """If post-action handling itself raises, connections are still saved and closed.""" + executor = _executor( + tmp_path, + {"First": _make_plugin(TrackedManager, "First")}, + {"TrackedManager": TrackedManager}, + {"First": {}}, + ) + + with patch.object(PluginExecutor, "_run_post_actions", side_effect=RuntimeError("post boom")): + with pytest.raises(RuntimeError, match="post boom"): + executor.run_queue() + + assert _disconnects(tracked) == [("disconnect", "TrackedManager")] + assert (_conn_dir(tmp_path, TrackedManager) / "result.json").is_file() + + +def test_cleanup_failure_for_one_connection_does_not_block_the_others(tmp_path, tracked): + """A manager whose disconnect raises must not stop the other managers being closed.""" + executor = _executor( + tmp_path, + { + "PluginA": _make_plugin(ManagerA, "PluginA"), + "PluginB": _make_plugin(ManagerB, "PluginB"), + }, + {"ManagerA": ManagerA, "ManagerB": ManagerB}, + {"PluginA": {}, "PluginB": {}}, + ) + + with patch.object(ManagerA, "disconnect", side_effect=OSError("cleanup boom")): + results = executor.run_queue() + + assert {r.source for r in results} == {"PluginA", "PluginB"} + assert _disconnects(tracked, "ManagerB") == [("disconnect", "ManagerB")] + with open(_conn_dir(tmp_path, ManagerB) / "result.json", encoding="utf-8") as f: + assert json.load(f)["status"] == "OK" + + +def test_failing_connection_hook_does_not_stop_cleanup(tmp_path, tracked): + """A hook that raises while saving a recorded failure must not abort cleanup.""" + + class RaisingOnFailureHook(TaskResultHook): + def process_result(self, task_result, data=None, **kwargs): + if task_result.status == ExecutionStatus.EXECUTION_FAILURE: + raise OSError("disk full") + + executor = _executor( + tmp_path, + { + "Failing": _make_plugin(ManagerA, "Failing", "raise"), + "Ok": _make_plugin(ManagerB, "Ok"), + }, + {"ManagerA": ManagerA, "ManagerB": ManagerB}, + {"Failing": {}, "Ok": {}}, + ) + executor.connection_result_hooks.append(RaisingOnFailureHook()) + + results = executor.run_queue() + + assert [r.source for r in results] == ["Ok"] + assert len(_disconnects(tracked, "ManagerA")) == 2 + assert len(_disconnects(tracked, "ManagerB")) == 1 + + +def test_failure_close_error_does_not_mask_plugin_failure(tmp_path, tracked): + """If disconnect raises while closing after a plugin failure, the queue carries on.""" + executor = _executor( + tmp_path, + {"Failing": _make_plugin(ManagerA, "Failing", "raise"), "Ok": _make_plugin(ManagerB, "Ok")}, + {"ManagerA": ManagerA, "ManagerB": ManagerB}, + {"Failing": {}, "Ok": {}}, + ) + + with patch.object(ManagerA, "disconnect", side_effect=OSError("close boom")): + results = executor.run_queue() + + assert [r.source for r in results] == ["Ok"] + + +# --------------------------------------------------------------------------- +# OS discovery +# --------------------------------------------------------------------------- + + +def _discovery_executor(tmp_path, **system_kwargs): + return PluginExecutor( + plugin_configs=[PluginConfig(plugins={})], + plugin_registry=_isolated_registry({}, {}), + system_info=SystemInfo(**system_kwargs), + log_path=str(tmp_path), + ) + + +def test_discover_os_info_success_saves_result_in_os_discovery_dir(tmp_path): + executor = _discovery_executor(tmp_path) + + with ( + patch("nodescraper.pluginexecutor.discover_and_write_os_family") as detect, + patch.object(InBandConnectionManager, "disconnect", autospec=True) as disconnect, + ): + executor.discover_os_info() + + detect.assert_called_once() + disconnect.assert_called_once() + with open(tmp_path / "os_discovery" / "result.json", encoding="utf-8") as f: + saved = json.load(f) + assert saved["status"] == "OK" + assert saved["parent"] == "os_discovery" + assert not (tmp_path / "os_discovery" / "events.json").exists() + assert not (tmp_path / "connection").exists() + + +@pytest.mark.parametrize( + "spot, expected_priority", + [ + ("connect_raises", "CRITICAL"), + ("no_credentials", "CRITICAL"), + # the connection itself succeeded; the detection error is only logged + ("detection_raises", None), + ], +) +def test_discover_os_info_failures_are_saved_in_os_discovery_dir_and_closed( + tmp_path, spot, expected_priority +): + system_kwargs = {"location": SystemLocation.REMOTE} if spot == "no_credentials" else {} + executor = _discovery_executor(tmp_path, **system_kwargs) + original_family = executor.system_info.os_family + + patches = [patch.object(InBandConnectionManager, "disconnect", autospec=True)] + if spot == "connect_raises": + patches.append( + patch( + "nodescraper.connection.inband.inbandmanager.LocalShell", + side_effect=OSError("no shell"), + ) + ) + elif spot == "detection_raises": + patches.append( + patch( + "nodescraper.pluginexecutor.discover_and_write_os_family", + side_effect=RuntimeError("detect boom"), + ) + ) + with patches[0] as disconnect: + for extra in patches[1:]: + extra.start() + try: + executor.discover_os_info() + finally: + for extra in patches[1:]: + extra.stop() + + disconnect.assert_called_once() + assert executor.system_info.os_family == original_family + if expected_priority is None: + assert not (tmp_path / "os_discovery" / "events.json").exists() + else: + events = _read_events(tmp_path / "os_discovery" / "events.json") + assert events[-1]["priority"] == expected_priority + assert events[-1]["data"]["parent"] == "os_discovery" + assert not (tmp_path / "connection").exists() + + +def test_discover_os_info_disconnect_error_does_not_propagate(tmp_path): + executor = _discovery_executor(tmp_path) + + with ( + patch("nodescraper.pluginexecutor.discover_and_write_os_family"), + patch.object(InBandConnectionManager, "disconnect", side_effect=OSError("close boom")), + ): + executor.discover_os_info() + + +def test_discover_os_info_constructor_failure_does_not_raise(tmp_path): + executor = _discovery_executor(tmp_path) + + with patch( + "nodescraper.pluginexecutor.InBandConnectionManager", + side_effect=ValueError("bad connection args"), + ): + executor.discover_os_info() + + assert not (tmp_path / "connection").exists() + + +def test_discovery_and_plugin_results_go_to_separate_folders(tmp_path, tracked): + """OS discovery output and plugin connection output never share a folder.""" + executor = _executor( + tmp_path, + {"Failing": _make_plugin(TrackedManager, "Failing", "raise")}, + {"TrackedManager": TrackedManager}, + {"Failing": {}}, + system_info=SystemInfo(os_family=OSFamily.UNKNOWN), + ) + + with patch("nodescraper.pluginexecutor.discover_and_write_os_family"): + executor.run_queue() + + discovery = json.loads((tmp_path / "os_discovery" / "result.json").read_text()) + assert discovery["task"] == "InBandConnectionManager" + assert discovery["status"] == "OK" + assert not (tmp_path / "os_discovery" / "events.json").exists() + + assert not (tmp_path / "connection" / "os_discovery").exists() + assert not (tmp_path / "connection" / resolve_log_dir_name("InBandConnectionManager")).exists() + + +def test_run_queue_continues_when_discovery_connection_cannot_be_built(tmp_path, tracked): + executor = _executor( + tmp_path, + {"Ok": _make_plugin(TrackedManager, "Ok")}, + {"TrackedManager": TrackedManager}, + {"Ok": {}}, + system_info=SystemInfo(os_family=OSFamily.UNKNOWN), + ) + + with patch( + "nodescraper.pluginexecutor.InBandConnectionManager", side_effect=ValueError("bad args") + ): + results = executor.run_queue() + + assert [r.source for r in results] == ["Ok"] + + +class _OkCollector(DataCollector): + DATA_MODEL = DummyDataModel + + def collect_data(self, args=None): + return TaskResult(status=ExecutionStatus.OK), DummyDataModel() + + +def _make_data_plugin(name): + class _DataPlugin(DataPlugin): + DATA_MODEL = DummyDataModel + CONNECTION_TYPE = TrackedManager + COLLECTOR = _OkCollector + + _DataPlugin.__name__ = name + return _DataPlugin + + +def test_data_plugins_share_one_connection_by_default(tmp_path, tracked): + """No argument is needed for plugins to share the executor's connection.""" + executor = _executor( + tmp_path, + {"First": _make_data_plugin("First"), "Second": _make_data_plugin("Second")}, + {"TrackedManager": TrackedManager}, + {"First": {"analysis": False}, "Second": {"analysis": False}}, + ) + + results = executor.run_queue() + + assert [r.status for r in results] == [ExecutionStatus.OK, ExecutionStatus.OK] + assert tracked == [("connect", "TrackedManager"), ("disconnect", "TrackedManager")] + + +def test_plugin_config_preserve_connection_false_is_honored(tmp_path, tracked): + """A plugin configured with preserve_connection=False closes the shared connection and the + next plugin reconnects; the executor no longer overrides the value.""" + executor = _executor( + tmp_path, + {"First": _make_data_plugin("First"), "Second": _make_data_plugin("Second")}, + {"TrackedManager": TrackedManager}, + { + "First": {"analysis": False, "preserve_connection": False}, + "Second": {"analysis": False}, + }, + ) + + results = executor.run_queue() + + assert [r.status for r in results] == [ExecutionStatus.OK, ExecutionStatus.OK] + assert tracked == [ + ("connect", "TrackedManager"), + ("disconnect", "TrackedManager"), # First closed it + ("connect", "TrackedManager"), # Second reconnected + ("disconnect", "TrackedManager"), # executor cleanup + ] + + +def test_global_args_preserve_connection_false_is_honored(tmp_path, tracked): + executor = _executor( + tmp_path, + {"First": _make_data_plugin("First"), "Second": _make_data_plugin("Second")}, + {"TrackedManager": TrackedManager}, + {"First": {"analysis": False}, "Second": {"analysis": False}}, + config_kwargs={"global_args": {"preserve_connection": False}}, + ) + + executor.run_queue() + + # each plugin closes it after collecting, then the executor's cleanup closes it once more + assert len(_disconnects(tracked)) == 3 + assert len([c for c in tracked if c[0] == "connect"]) == 2 diff --git a/test/unit/framework/test_plugin_interface.py b/test/unit/framework/test_plugin_interface.py new file mode 100644 index 00000000..6913d325 --- /dev/null +++ b/test/unit/framework/test_plugin_interface.py @@ -0,0 +1,38 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +from nodescraper.enums import ExecutionStatus +from nodescraper.interfaces.plugin import PluginInterface +from nodescraper.interfaces.taskresulthook import TaskResultHook +from nodescraper.models import PluginResult, TaskResult +from nodescraper.taskresulthooks.filesystemloghook import FileSystemLogHook + + +class DummyHook(TaskResultHook): + def process_result(self, task_result: TaskResult, **kwargs): + return None + + +class DummyPlugin(PluginInterface): + def run(self, **kwargs) -> PluginResult: + return PluginResult(source=self.__class__.__name__, status=ExecutionStatus.OK) + + +def test_log_path_adds_filesystem_hook(): + """Baseline: a plugin given a log path gets a filesystem hook for that path.""" + plugin = DummyPlugin(log_path="/tmp/run-a") + + hooks = [h for h in plugin.task_result_hooks if isinstance(h, FileSystemLogHook)] + assert [h.log_base_path for h in hooks] == ["/tmp/run-a"] + + +def test_shared_hook_list_does_not_leak_log_path_between_plugins(): + """A hook list shared by two plugins must not give the second plugin the first's log path.""" + shared_hooks: list = [DummyHook()] + + DummyPlugin(log_path="/tmp/run-a", task_result_hooks=shared_hooks) + plugin_b = DummyPlugin(log_path="/tmp/run-b", task_result_hooks=shared_hooks) + + hooks = [h for h in plugin_b.task_result_hooks if isinstance(h, FileSystemLogHook)] + assert "/tmp/run-b" in [h.log_base_path for h in hooks] diff --git a/test/unit/framework/test_plugin_registry_lazy.py b/test/unit/framework/test_plugin_registry_lazy.py new file mode 100644 index 00000000..92f17270 --- /dev/null +++ b/test/unit/framework/test_plugin_registry_lazy.py @@ -0,0 +1,154 @@ +############################################################################### +# +# MIT License +# +# Copyright (C) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock, patch + +import pytest + +import nodescraper.plugins as internal_plugins +from nodescraper.interfaces import PluginInterface +from nodescraper.pluginregistry import LazyPluginMapping, PluginRegistry +from nodescraper.plugins.inband.bios.bios_plugin import BiosPlugin +from nodescraper.plugins.inband.kernel.kernel_plugin import KernelPlugin + +BIOS_VALUE = "nodescraper.plugins.inband.bios.bios_plugin:BiosPlugin" +KERNEL_VALUE = "nodescraper.plugins.inband.kernel.kernel_plugin:KernelPlugin" + + +@pytest.fixture(autouse=True) +def clear_cache(): + PluginRegistry.clear_caches() + yield + PluginRegistry.clear_caches() + + +def _entry_point(name, value, loaded): + ep = MagicMock() + ep.name = name + ep.value = value + if isinstance(loaded, Exception): + ep.load.side_effect = loaded + else: + ep.load.return_value = loaded + return ep + + +def _patch_plugin_entry_points(eps): + def side_effect(*args, **kwargs): + return eps if kwargs.get("group") == "nodescraper.plugins" else [] + + return patch( + "nodescraper.pluginregistry.importlib.metadata.entry_points", side_effect=side_effect + ) + + +def test_lazy_registry_uses_lazy_mapping(): + reg = PluginRegistry(lazy_load=True) + assert reg.lazy + assert isinstance(reg.plugins, LazyPluginMapping) + + +def test_lazy_internal_plugins_match_eager_scan(): + """Every internal plugin must be declared in the ``nodescraper.plugins`` entry points.""" + lazy = PluginRegistry( + lazy_load=True, load_entry_point_plugins=False, load_entry_point_connection_managers=False + ) + eager = PluginRegistry.load_plugins(PluginInterface, [internal_plugins]) + assert sorted(lazy.plugins) == sorted(eager) + for name, cls in eager.items(): + assert lazy.plugins[name] is cls + + +def test_lazy_entry_points_not_loaded_until_accessed(): + bios_ep = _entry_point("BiosPlugin", BIOS_VALUE, BiosPlugin) + kernel_ep = _entry_point("KernelPlugin", KERNEL_VALUE, KernelPlugin) + with _patch_plugin_entry_points([bios_ep, kernel_ep]): + reg = PluginRegistry( + lazy_load=True, + load_entry_point_connection_managers=False, + ) + assert sorted(reg.plugins) == ["BiosPlugin", "KernelPlugin"] + bios_ep.load.assert_not_called() + kernel_ep.load.assert_not_called() + + assert reg.plugins["BiosPlugin"] is BiosPlugin + bios_ep.load.assert_called_once() + kernel_ep.load.assert_not_called() + assert reg.plugins.is_resolved("BiosPlugin") + assert not reg.plugins.is_resolved("KernelPlugin") + + assert reg.plugins["BiosPlugin"] is BiosPlugin + bios_ep.load.assert_called_once() + + +def test_lazy_internal_only_skips_external_entry_points(): + bios_ep = _entry_point("BiosPlugin", BIOS_VALUE, BiosPlugin) + external_ep = _entry_point("ExtPlugin", "ext_pkg.ext_plugin:ExtPlugin", KernelPlugin) + with _patch_plugin_entry_points([bios_ep, external_ep]): + reg = PluginRegistry( + lazy_load=True, + load_entry_point_plugins=False, + load_entry_point_connection_managers=False, + ) + assert list(reg.plugins) == ["BiosPlugin"] + external_ep.load.assert_not_called() + + +def test_lazy_external_entry_point_keyed_by_class_name(): + external_ep = _entry_point("alias", "ext_pkg.ext_plugin:KernelPlugin", KernelPlugin) + with _patch_plugin_entry_points([external_ep]): + reg = PluginRegistry( + lazy_load=True, + load_internal_plugins=False, + load_entry_point_connection_managers=False, + ) + assert list(reg.plugins) == ["KernelPlugin"] + assert reg.plugins["KernelPlugin"] is KernelPlugin + + +def test_lazy_broken_entry_point_does_not_affect_others(): + broken_ep = _entry_point( + "BrokenPlugin", "nodescraper.plugins.nope:BrokenPlugin", ImportError("boom") + ) + bios_ep = _entry_point("BiosPlugin", BIOS_VALUE, BiosPlugin) + with _patch_plugin_entry_points([broken_ep, bios_ep]): + reg = PluginRegistry(lazy_load=True, load_entry_point_connection_managers=False) + assert "BrokenPlugin" not in reg.plugins + assert "BrokenPlugin" not in list(reg.plugins) + assert reg.plugins["BiosPlugin"] is BiosPlugin + with pytest.raises(KeyError): + reg.plugins["BrokenPlugin"] + + +def test_lazy_non_plugin_entry_point_is_treated_as_missing(): + bad_ep = _entry_point("NotAPlugin", "nodescraper.plugins.x:NotAPlugin", dict) + with _patch_plugin_entry_points([bad_ep]): + reg = PluginRegistry( + lazy_load=True, + load_internal_plugins=False, + load_entry_point_connection_managers=False, + ) + assert "NotAPlugin" not in reg.plugins + assert len(reg.plugins) == 0 diff --git a/test/unit/framework/test_pluginrecipe.py b/test/unit/framework/test_pluginrecipe.py index 55b65450..9adeb30f 100644 --- a/test/unit/framework/test_pluginrecipe.py +++ b/test/unit/framework/test_pluginrecipe.py @@ -12,8 +12,9 @@ import pytest +from nodescraper.connection.inband import InBandConnectionManager from nodescraper.models import PluginConfig -from nodescraper.pluginrecipe.all_plugins import AllPlugins +from nodescraper.pluginrecipe.all_ib_plugins import AllIbPlugins from nodescraper.pluginrecipe.discovery import PluginDiscovery from nodescraper.pluginrecipe.node_status import NodeStatus from nodescraper.pluginrecipe.pluginrecipe import ( @@ -77,9 +78,21 @@ def test_node_status_recipe_matches_registered_plugins() -> None: assert set(NodeStatus.plugin_names()) == expected & available -def test_all_plugins_recipe_matches_registry() -> None: - plugin_reg = PluginRegistry() - assert set(AllPlugins.plugin_names()) == set(plugin_reg.plugins) +def test_all_ib_plugins_recipe_returns_only_ib_plugins() -> None: + discovery = PluginDiscovery() + for name in AllIbPlugins.plugin_names(): + plugin_class = discovery.load_plugin_class(name) + assert getattr(plugin_class, "CONNECTION_TYPE", None) is InBandConnectionManager + + +def test_all_ib_plugins_recipe_excludes_oob_plugins() -> None: + all_names = set(PluginRegistry().plugins) + ib_names = set(AllIbPlugins.plugin_names()) + discovery = PluginDiscovery() + non_ib = all_names - ib_names + for name in non_ib: + plugin_class = discovery.load_plugin_class(name) + assert getattr(plugin_class, "CONNECTION_TYPE", None) is not InBandConnectionManager def test_node_status_plugin_config_shape() -> None: @@ -90,11 +103,11 @@ def test_node_status_plugin_config_shape() -> None: assert config.plugins["DmesgPlugin"] == COLLECT_AND_ANALYZE.as_config() -def test_all_plugins_plugin_config_shape() -> None: - config = AllPlugins.plugin_config() - assert config.name == "AllPlugins" - assert config.desc == "Run all registered plugins with default arguments." - assert len(config.plugins) == len(PluginRegistry().plugins) +def test_all_ib_plugins_plugin_config_shape() -> None: + config = AllIbPlugins.plugin_config() + assert config.name == "AllIbPlugins" + assert config.desc == "Run all registered in-band plugins with default arguments." + assert len(config.plugins) == len(AllIbPlugins.plugin_names()) def test_collector_only_recipe_sets_analysis_false() -> None: diff --git a/test/unit/framework/test_post_action_condition.py b/test/unit/framework/test_post_action_condition.py new file mode 100644 index 00000000..99fe151a --- /dev/null +++ b/test/unit/framework/test_post_action_condition.py @@ -0,0 +1,362 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Unit tests for PostActionCondition matching logic.""" +from typing import Union + +import pytest + +from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.models import DataPluginResult, Event, PluginResult, TaskResult +from nodescraper.models.postactioncondition import PostActionCondition + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_event( + category: str, + description: str, + priority: EventPriority = EventPriority.ERROR, +) -> Event: + return Event( + category=category, + description=description, + priority=priority, + reporter="test", + ) + + +def _make_result( + source: str = "SomePlugin", + status: ExecutionStatus = ExecutionStatus.OK, + analysis_events: Union[list[Event], None] = None, + collection_events: Union[list[Event], None] = None, +) -> PluginResult: + """Build a PluginResult with optional events in analysis and/or collection results.""" + return PluginResult( + status=status, + source=source, + result_data=DataPluginResult( + analysis_result=TaskResult(status=status, events=analysis_events or []), + collection_result=TaskResult(status=status, events=collection_events or []), + ), + ) + + +# --------------------------------------------------------------------------- +# No-field conditions (vacuously true) +# --------------------------------------------------------------------------- + + +def test_no_fields_matches_any_result(): + """A condition with all fields None matches any result.""" + condition = PostActionCondition() + result = _make_result(status=ExecutionStatus.OK) + assert condition.is_met([result]) is True + + +def test_no_fields_matches_error_result(): + condition = PostActionCondition() + result = _make_result(status=ExecutionStatus.ERROR) + assert condition.is_met([result]) is True + + +def test_no_results_returns_false(): + """With an empty results list there are no candidates — always False.""" + condition = PostActionCondition() + assert condition.is_met([]) is False + + +# --------------------------------------------------------------------------- +# status field +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "condition_status, result_status, expected", + [ + ("WARNING", ExecutionStatus.WARNING, True), # exact match at threshold + ("WARNING", ExecutionStatus.ERROR, True), # above threshold + ("WARNING", ExecutionStatus.EXECUTION_FAILURE, True), + ("ERROR", ExecutionStatus.ERROR, True), + ("ERROR", ExecutionStatus.OK, False), # below threshold + ("ERROR", ExecutionStatus.WARNING, False), + ("EXECUTION_FAILURE", ExecutionStatus.ERROR, False), + ("OK", ExecutionStatus.OK, True), + ("OK", ExecutionStatus.WARNING, True), + ], +) +def test_status_threshold(condition_status, result_status, expected): + condition = PostActionCondition(status=condition_status) + result = _make_result(status=result_status) + assert condition.is_met([result]) is expected + + +def test_status_invalid_name_raises(): + """An unrecognised status name raises ValidationError at construction time.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + PostActionCondition(status="NONEXISTENT_STATUS") + + +# --------------------------------------------------------------------------- +# event_category field +# --------------------------------------------------------------------------- + + +def test_event_category_matches_analysis_event(): + condition = PostActionCondition(event_category="WIDGET_ERROR") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("WIDGET_ERROR", "widget fault detected")], + ) + assert condition.is_met([result]) is True + + +def test_event_category_matches_collection_event(): + """event_category also matches events from collection_result.""" + condition = PostActionCondition(event_category="COLLECTION_WARN") + result = _make_result( + status=ExecutionStatus.WARNING, + collection_events=[_make_event("COLLECTION_WARN", "something", EventPriority.WARNING)], + ) + assert condition.is_met([result]) is True + + +def test_event_category_no_match(): + condition = PostActionCondition(event_category="WIDGET_ERROR") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("GIZMO_ERROR", "gizmo fault")], + ) + assert condition.is_met([result]) is False + + +def test_event_category_normalisation(): + """Category matching normalises spaces and hyphens to underscores and uppercases.""" + condition = PostActionCondition(event_category="widget-error") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("WIDGET_ERROR", "widget fault")], + ) + assert condition.is_met([result]) is True + + +def test_event_category_no_events_returns_false(): + condition = PostActionCondition(event_category="WIDGET_ERROR") + result = _make_result(status=ExecutionStatus.ERROR) # no events + assert condition.is_met([result]) is False + + +# --------------------------------------------------------------------------- +# event_priority field +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "condition_priority, event_priority, expected", + [ + ("WARNING", EventPriority.WARNING, True), + ("WARNING", EventPriority.ERROR, True), + ("WARNING", EventPriority.CRITICAL, True), + ("ERROR", EventPriority.ERROR, True), + ("ERROR", EventPriority.CRITICAL, True), + ("ERROR", EventPriority.WARNING, False), + ("CRITICAL", EventPriority.ERROR, False), + ("CRITICAL", EventPriority.CRITICAL, True), + ], +) +def test_event_priority_threshold(condition_priority, event_priority, expected): + condition = PostActionCondition(event_priority=condition_priority) + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("SOME_CAT", "some description", event_priority)], + ) + assert condition.is_met([result]) is expected + + +def test_event_priority_invalid_name_raises(): + """An unrecognised event_priority name raises ValidationError at construction time.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError): + PostActionCondition(event_priority="SUPER_CRITICAL") + + +def test_event_priority_no_events_returns_false(): + condition = PostActionCondition(event_priority="WARNING") + result = _make_result(status=ExecutionStatus.ERROR) + assert condition.is_met([result]) is False + + +# --------------------------------------------------------------------------- +# event_description_contains field +# --------------------------------------------------------------------------- + + +def test_event_description_contains_match(): + condition = PostActionCondition(event_description_contains="widget fault") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("WIDGET_ERROR", "widget fault detected on unit 0")], + ) + assert condition.is_met([result]) is True + + +def test_event_description_contains_no_match(): + condition = PostActionCondition(event_description_contains="widget fault") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("GIZMO_ERROR", "gizmo fault")], + ) + assert condition.is_met([result]) is False + + +def test_event_description_contains_case_sensitive(): + """Substring match is case-sensitive.""" + condition = PostActionCondition(event_description_contains="Widget Fault") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("WIDGET_ERROR", "widget fault detected")], + ) + assert condition.is_met([result]) is False + + +def test_event_description_contains_no_events_returns_false(): + condition = PostActionCondition(event_description_contains="anything") + result = _make_result(status=ExecutionStatus.ERROR) + assert condition.is_met([result]) is False + + +# --------------------------------------------------------------------------- +# AND semantics within a single condition +# --------------------------------------------------------------------------- + + +def test_and_semantics_status_matches_category_does_not(): + """Both fields specified; status matches but category doesn't → False.""" + condition = PostActionCondition(status="ERROR", event_category="WIDGET_ERROR") + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("GIZMO_ERROR", "gizmo fault")], + ) + assert condition.is_met([result]) is False + + +def test_and_semantics_category_matches_status_does_not(): + """Both fields specified; category matches but status doesn't → False.""" + condition = PostActionCondition(status="ERROR", event_category="WIDGET_ERROR") + result = _make_result( + status=ExecutionStatus.WARNING, + analysis_events=[_make_event("WIDGET_ERROR", "widget fault")], + ) + assert condition.is_met([result]) is False + + +def test_and_semantics_all_fields_match(): + """All four fields specified and all matched → True.""" + condition = PostActionCondition( + status="WARNING", + event_category="WIDGET_ERROR", + event_priority="ERROR", + event_description_contains="widget fault", + ) + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("WIDGET_ERROR", "widget fault detected", EventPriority.ERROR)], + ) + assert condition.is_met([result]) is True + + +def test_and_semantics_three_fields_one_missing(): + """Three fields specified; the one unmatched field causes False.""" + condition = PostActionCondition( + status="ERROR", + event_category="WIDGET_ERROR", + event_description_contains="widget fault", + ) + result = _make_result( + status=ExecutionStatus.ERROR, + analysis_events=[_make_event("WIDGET_ERROR", "gizmo fault")], # description doesn't match + ) + assert condition.is_met([result]) is False + + +# --------------------------------------------------------------------------- +# plugin filter +# --------------------------------------------------------------------------- + + +def test_plugin_filter_restricts_to_named_source(): + """With plugin set, only that plugin's result is a candidate.""" + condition = PostActionCondition(plugin="PluginA", status="ERROR") + result_a = _make_result(source="PluginA", status=ExecutionStatus.ERROR) + result_b = _make_result(source="PluginB", status=ExecutionStatus.ERROR) + assert condition.is_met([result_a, result_b]) is True + + +def test_plugin_filter_excludes_other_source(): + """Named plugin's result doesn't meet the condition; other results are excluded.""" + condition = PostActionCondition(plugin="PluginA", status="ERROR") + result_a = _make_result(source="PluginA", status=ExecutionStatus.OK) + result_b = _make_result(source="PluginB", status=ExecutionStatus.ERROR) + assert condition.is_met([result_a, result_b]) is False + + +def test_plugin_filter_none_checks_all_results(): + """With plugin=None, any result may satisfy the condition.""" + condition = PostActionCondition(plugin=None, status="ERROR") + result_a = _make_result(source="PluginA", status=ExecutionStatus.OK) + result_b = _make_result(source="PluginB", status=ExecutionStatus.ERROR) + assert condition.is_met([result_a, result_b]) is True + + +def test_plugin_filter_named_plugin_not_in_results(): + """Named plugin not present in results at all → no candidates → False.""" + condition = PostActionCondition(plugin="MissingPlugin", status="ERROR") + result = _make_result(source="OtherPlugin", status=ExecutionStatus.ERROR) + assert condition.is_met([result]) is False + + +# --------------------------------------------------------------------------- +# result_data edge cases +# --------------------------------------------------------------------------- + + +def test_result_with_no_result_data_status_only(): + """A bare PluginResult (no result_data) can still match on status.""" + condition = PostActionCondition(status="ERROR") + result = PluginResult(status=ExecutionStatus.ERROR, source="BarePlugin") + assert condition.is_met([result]) is True + + +def test_result_with_no_result_data_event_field_returns_false(): + """A bare PluginResult has no events; event-based conditions are False.""" + condition = PostActionCondition(event_category="WIDGET_ERROR") + result = PluginResult(status=ExecutionStatus.ERROR, source="BarePlugin") + assert condition.is_met([result]) is False diff --git a/test/unit/framework/test_post_action_plugin_config.py b/test/unit/framework/test_post_action_plugin_config.py new file mode 100644 index 00000000..019d8e04 --- /dev/null +++ b/test/unit/framework/test_post_action_plugin_config.py @@ -0,0 +1,232 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Unit tests for PostActionPluginConfig.should_run OR semantics.""" +from typing import Union + +import pytest + +from nodescraper.enums import ExecutionStatus +from nodescraper.models import PluginResult +from nodescraper.models.postactioncondition import PostActionCondition +from nodescraper.models.postactionpluginconfig import PostActionPluginConfig + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_result(source: str, status: ExecutionStatus) -> PluginResult: + return PluginResult(status=status, source=source) + + +def _cond(status, plugin: Union[str, None] = None) -> PostActionCondition: + """Shorthand: a condition that fires when *source* has at least *status*.""" + return PostActionCondition(plugin=plugin, status=status) + + +# --------------------------------------------------------------------------- +# Empty conditions +# --------------------------------------------------------------------------- + + +def test_no_conditions_never_fires(): + """An empty conditions list always returns False — no accidental unconditional runs.""" + cfg = PostActionPluginConfig(plugin="SomePlugin", conditions=[]) + results = [_make_result("Primary", ExecutionStatus.EXECUTION_FAILURE)] + assert cfg.should_run(results) is False + + +def test_no_conditions_empty_results_also_false(): + cfg = PostActionPluginConfig(plugin="SomePlugin", conditions=[]) + assert cfg.should_run([]) is False + + +# --------------------------------------------------------------------------- +# OR semantics across conditions +# --------------------------------------------------------------------------- + + +def test_or_first_condition_met_second_not(): + """should_run is True when the first condition fires even if the second doesn't.""" + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[ + _cond("ERROR", plugin="PluginA"), # PluginA has ERROR → True + _cond("ERROR", plugin="PluginB"), # PluginB has OK → False + ], + ) + results = [ + _make_result("PluginA", ExecutionStatus.ERROR), + _make_result("PluginB", ExecutionStatus.OK), + ] + assert cfg.should_run(results) is True + + +def test_or_first_condition_not_met_second_met(): + """should_run is True when only the second condition fires.""" + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[ + _cond("ERROR", plugin="PluginA"), # PluginA has OK → False + _cond("ERROR", plugin="PluginB"), # PluginB has ERROR → True + ], + ) + results = [ + _make_result("PluginA", ExecutionStatus.OK), + _make_result("PluginB", ExecutionStatus.ERROR), + ] + assert cfg.should_run(results) is True + + +def test_or_no_conditions_met(): + """should_run is False when no condition is satisfied.""" + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[ + _cond("ERROR", plugin="PluginA"), + _cond("ERROR", plugin="PluginB"), + ], + ) + results = [ + _make_result("PluginA", ExecutionStatus.OK), + _make_result("PluginB", ExecutionStatus.WARNING), + ] + assert cfg.should_run(results) is False + + +def test_or_all_conditions_met(): + """should_run is True when every condition is satisfied (OR short-circuits at first).""" + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[ + _cond("ERROR", plugin="PluginA"), + _cond("ERROR", plugin="PluginB"), + ], + ) + results = [ + _make_result("PluginA", ExecutionStatus.ERROR), + _make_result("PluginB", ExecutionStatus.ERROR), + ] + assert cfg.should_run(results) is True + + +def test_single_condition_met(): + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[_cond("WARNING")], + ) + results = [_make_result("Primary", ExecutionStatus.ERROR)] + assert cfg.should_run(results) is True + + +def test_single_condition_not_met(): + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[_cond("ERROR")], + ) + results = [_make_result("Primary", ExecutionStatus.OK)] + assert cfg.should_run(results) is False + + +# --------------------------------------------------------------------------- +# Empty results list +# --------------------------------------------------------------------------- + + +def test_conditions_present_empty_results_returns_false(): + """Conditions exist but there are no primary results to match against.""" + cfg = PostActionPluginConfig( + plugin="SomePlugin", + conditions=[_cond("ERROR")], + ) + assert cfg.should_run([]) is False + + +# --------------------------------------------------------------------------- +# plugin field validation +# --------------------------------------------------------------------------- + + +def test_empty_plugin_name_raises(): + """An empty string plugin name raises ValidationError at construction time.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError, match="must not be empty or whitespace"): + PostActionPluginConfig(plugin="", conditions=[_cond("ERROR")]) + + +def test_whitespace_only_plugin_name_raises(): + """A whitespace-only plugin name raises ValidationError at construction time.""" + from pydantic import ValidationError + + with pytest.raises(ValidationError, match="must not be empty or whitespace"): + PostActionPluginConfig(plugin=" ", conditions=[_cond("ERROR")]) + + +# --------------------------------------------------------------------------- +# plugin_args field validation +# --------------------------------------------------------------------------- + + +def test_plugin_args_defaults_to_empty_dict(): + cfg = PostActionPluginConfig(plugin="SomePlugin", conditions=[_cond("ERROR")]) + assert cfg.plugin_args == {} + + +def test_plugin_args_passed_through(): + cfg = PostActionPluginConfig( + plugin="SomePlugin", + plugin_args={"collection": True, "analysis": False}, + conditions=[_cond("ERROR")], + ) + assert cfg.plugin_args == {"collection": True, "analysis": False} + + +# --------------------------------------------------------------------------- +# JSON round-trip (Pydantic model_validate from dict) +# --------------------------------------------------------------------------- + + +def test_model_validate_from_dict(): + """PostActionPluginConfig can be constructed from a plain dict (as from JSON config).""" + from nodescraper.enums import EventPriority, ExecutionStatus + + raw = { + "plugin": "RemediationPlugin", + "plugin_args": {"collection": True}, + "conditions": [ + {"plugin": "PrimaryPlugin", "status": "ERROR"}, + {"event_priority": "CRITICAL", "event_description_contains": "widget fault"}, + ], + } + cfg = PostActionPluginConfig.model_validate(raw) + assert cfg.plugin == "RemediationPlugin" + assert cfg.plugin_args == {"collection": True} + assert len(cfg.conditions) == 2 + assert cfg.conditions[0].plugin == "PrimaryPlugin" + assert cfg.conditions[0].status == ExecutionStatus.ERROR + assert cfg.conditions[1].event_priority == EventPriority.CRITICAL + assert cfg.conditions[1].event_description_contains == "widget fault" diff --git a/test/unit/framework/test_redfish_connection_selection.py b/test/unit/framework/test_redfish_connection_selection.py new file mode 100644 index 00000000..21d08614 --- /dev/null +++ b/test/unit/framework/test_redfish_connection_selection.py @@ -0,0 +1,128 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import unittest +from typing import Any, Optional + +from nodescraper.connection.redfish import ( + RedfishConnectionManager, + RedfishSshProxyConnectionManager, +) +from nodescraper.enums import OSFamily, SystemLocation +from nodescraper.interfaces import ConnectionManager +from nodescraper.models import PluginConfig, SystemInfo +from nodescraper.pluginexecutor import PluginExecutor + +REDFISH_PLUGIN = "RedfishEndpointPlugin" +PROXY_PLUGIN = "AmcRedfishDiagPlugin" + +DIRECT_CONFIG: dict[str, Any] = { + "RedfishConnectionManager": {"host": "bmc.example.com", "username": "root", "password": "x"} +} +PROXY_CONFIG: dict[str, Any] = { + "RedfishSshProxyConnectionManager": { + "host": "192.0.2.10", + "ssh": {"hostname": "bmc.example.com", "username": "root", "password": "x"}, + } +} +BOTH_CONFIG: dict[str, Any] = {**DIRECT_CONFIG, **PROXY_CONFIG} + + +def _executor(connections: dict) -> PluginExecutor: + return PluginExecutor( + plugin_configs=[PluginConfig(plugins={REDFISH_PLUGIN: {}})], + connections=connections, + system_info=SystemInfo( + name="test", location=SystemLocation.REMOTE, os_family=OSFamily.LINUX + ), + ) + + +def _resolve(ex: PluginExecutor, plugin_name: str = REDFISH_PLUGIN) -> Optional[ConnectionManager]: + plugin_cls = ex.plugin_registry.plugins[plugin_name] + return ex._get_connection_manager_for_plugin(plugin_cls, plugin_name) + + +class TestRedfishConnectionSelection(unittest.TestCase): + def test_plugin_declares_direct_redfish(self): + ex = _executor({}) + plugin_cls = ex.plugin_registry.plugins[REDFISH_PLUGIN] + self.assertIs(plugin_cls.CONNECTION_TYPE, RedfishConnectionManager) + + def test_direct_only_uses_direct(self): + ex = _executor(DIRECT_CONFIG) + self.assertIs(_resolve(ex), ex.connection_library[RedfishConnectionManager]) + + def test_proxy_only_falls_back_to_proxy(self): + ex = _executor(PROXY_CONFIG) + self.assertIs(_resolve(ex), ex.connection_library[RedfishSshProxyConnectionManager]) + + def test_both_prefers_direct(self): + ex = _executor(BOTH_CONFIG) + self.assertIs(_resolve(ex), ex.connection_library[RedfishConnectionManager]) + + def test_neither_skips_plugin(self): + ex = _executor({}) + self.assertIsNone(_resolve(ex)) + self.assertNotIn(RedfishConnectionManager, ex.connection_library) + self.assertFalse( + ex._run_single_plugin( + REDFISH_PLUGIN, ex.plugin_config.plugins[REDFISH_PLUGIN], [], queue_callback=None + ) + ) + + def test_plugin_declaring_proxy_keeps_proxy_when_both_configured(self): + ex = _executor(BOTH_CONFIG) + plugin_cls = ex.plugin_registry.plugins[PROXY_PLUGIN] + self.assertIs(plugin_cls.CONNECTION_TYPE, RedfishSshProxyConnectionManager) + self.assertIs( + _resolve(ex, PROXY_PLUGIN), ex.connection_library[RedfishSshProxyConnectionManager] + ) + + def test_inventory_plugins_split_bmc_and_amc_when_both_configured(self): + ex = _executor(BOTH_CONFIG) + self.assertIs( + _resolve(ex, "RedfishInventoryPlugin"), + ex.connection_library[RedfishConnectionManager], + ) + self.assertIs( + _resolve(ex, "AmcRedfishInventoryPlugin"), + ex.connection_library[RedfishSshProxyConnectionManager], + ) + + def test_endpoint_plugins_split_bmc_and_amc_when_both_configured(self): + ex = _executor(BOTH_CONFIG) + self.assertIs( + _resolve(ex, "RedfishEndpointPlugin"), + ex.connection_library[RedfishConnectionManager], + ) + self.assertIs( + _resolve(ex, "AmcRedfishEndpointPlugin"), + ex.connection_library[RedfishSshProxyConnectionManager], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit/framework/test_regexanalyzer.py b/test/unit/framework/test_regexanalyzer.py index f07afb8c..df4e8371 100644 --- a/test/unit/framework/test_regexanalyzer.py +++ b/test/unit/framework/test_regexanalyzer.py @@ -24,6 +24,7 @@ # ############################################################################### import re +import signal from pydantic import BaseModel @@ -42,6 +43,8 @@ class DummyArgs(BaseModel): class TestRegexAnalyzer(RegexAnalyzer[DummyData, DummyArgs]): + __test__ = False # Tells pytest to ignore this class + DATA_MODEL = DummyData ERROR_REGEX = [ @@ -263,3 +266,20 @@ def test_check_all_regexes_skips_ignore_match_rules(system_info): assert len(events) == 1 assert "dummy error 3" in str(events[0].data["match_content"]) + + +def test_check_all_regexes_terminates_on_zero_width_match(system_info): + """A pattern able to match empty text must not spin forever in check_all_regexes.""" + analyzer = TestRegexAnalyzer(system_info=system_info) + zero_width = [ErrorRegex(regex=re.compile(r"x*"), message="Zero width")] + + def _on_timeout(signum, frame): + raise TimeoutError("check_all_regexes did not terminate") + + previous_handler = signal.signal(signal.SIGALRM, _on_timeout) + signal.setitimer(signal.ITIMER_REAL, 2.0) + try: + analyzer.check_all_regexes("abc", "src", zero_width) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) diff --git a/test/unit/framework/test_tablesummary.py b/test/unit/framework/test_tablesummary.py new file mode 100644 index 00000000..16da5bee --- /dev/null +++ b/test/unit/framework/test_tablesummary.py @@ -0,0 +1,95 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging + +from serviceability_dummy_data import DUMMY_AFID_A, DUMMY_UNIT_A + +from nodescraper.enums import ExecutionStatus +from nodescraper.models import DataPluginResult, PluginResult, TaskResult +from nodescraper.plugins.serviceability.se_models import ( + HubTriageResult, + ServiceabilityBlock, +) +from nodescraper.plugins.serviceability.serviceability_data import ( + ServiceabilityDataModel, +) +from nodescraper.resultcollators.tablesummary import TableSummary + + +def test_tablesummary_prints_connection_serviceability_then_plugin(caplog): + block = ServiceabilityBlock( + hub_top_results=[ + HubTriageResult( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + service_action_num=11018, + service_action_title="Check and Retry FW Bundle", + priority=1, + sa_severity=20, + tier_label="Secondary", + hub_sort_priority=1000, + ) + ], + hub_triage_results=[ + HubTriageResult( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + service_action_num=11018, + service_action_title="Check and Retry FW Bundle", + priority=1, + sa_severity=20, + tier_label="Secondary", + hub_sort_priority=1000, + ) + ], + ) + plugin_results = [ + PluginResult( + status=ExecutionStatus.OK, + source="Mi4xxServiceabilityPlugin", + message="Plugin tasks completed successfully", + result_data=DataPluginResult( + system_data=ServiceabilityDataModel(serviceability=block), + ), + ) + ] + connection_results = [ + TaskResult( + task="RedfishConnectionManager", + status=ExecutionStatus.OK, + message="task completed successfully", + ) + ] + + logger = logging.getLogger("test_tablesummary") + caplog.set_level(logging.INFO, logger="test_tablesummary") + TableSummary(logger=logger).collate_results(plugin_results, connection_results) + + output = caplog.text + connection_pos = output.index("| Connection") + serviceability_pos = output.index("Top Hub service action") + plugin_pos = output.index("| Plugin") + assert connection_pos < serviceability_pos < plugin_pos diff --git a/test/unit/framework/test_task.py b/test/unit/framework/test_task.py new file mode 100644 index 00000000..a4af9699 --- /dev/null +++ b/test/unit/framework/test_task.py @@ -0,0 +1,34 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +import pytest + +from nodescraper.interfaces.task import Task + + +def test_task_subclass_with_task_type_is_allowed(): + """Baseline: a subclass that declares TASK_TYPE is created without error.""" + + class GoodTask(Task): + TASK_TYPE = "GOOD_TASK" + + assert GoodTask.TASK_TYPE == "GOOD_TASK" + + +def test_task_subclass_with_none_task_type_raises(): + """Baseline: TASK_TYPE explicitly set to None is rejected.""" + + with pytest.raises(TypeError): + + class NoneTask(Task): + TASK_TYPE = None + + +def test_task_subclass_without_task_type_raises_type_error(): + """A subclass that never declares TASK_TYPE must raise the documented TypeError.""" + + with pytest.raises(TypeError): + + class MissingTaskType(Task): + pass diff --git a/test/unit/framework/test_type_utils.py b/test/unit/framework/test_type_utils.py index a6bb1012..bb63b171 100644 --- a/test/unit/framework/test_type_utils.py +++ b/test/unit/framework/test_type_utils.py @@ -33,6 +33,7 @@ class TestGenericBase(Generic[T]): + __test__ = False # Tells pytest to ignore this class def __init__(self, generic_type: T): self.generic_type = generic_type @@ -42,6 +43,8 @@ def test_func(self, arg: list[str], arg2: Union[bool, str], arg3: Optional[int] class TestGenericImpl(TestGenericBase[str]): + __test__ = False # Tells pytest to ignore this class + pass @@ -50,10 +53,14 @@ class WiringMixin: class TestMixinFirstImpl(WiringMixin, TestGenericBase[str]): + __test__ = False # Tells pytest to ignore this class + pass class TestModel(BaseModel): + __test__ = False # Tells pytest to ignore this class + str_attr: str int_attr: int list_attr: list[str] @@ -105,3 +112,55 @@ def test_model_types(): assert res["optional_attr"] == TypeData( type_classes=[TypeClass(type_class=str, inner_type=None)], required=False ) + + +def test_process_type_strips_annotated(): + """Baseline: a top level Annotated type is unwrapped to its underlying type.""" + from typing import Annotated + + assert TypeUtils.process_type(Annotated[int, "meta"]) == [ + TypeClass(type_class=int, inner_type=None) + ] + + +def test_process_type_strips_annotated_inside_optional(): + """An Annotated type nested in a Union must still resolve to its underlying type.""" + from typing import Annotated + + assert TypeUtils.process_type(Optional[Annotated[int, "meta"]]) == [ + TypeClass(type_class=int, inner_type=None) + ] + + +class _FilterModel(BaseModel): + host: str = "h" + port: int = 22 + + +def test_filter_to_model_fields_drops_keys_that_are_not_model_fields(): + data = {"host": "a", "port": 2, "other": "x"} + + assert TypeUtils.filter_to_model_fields(_FilterModel, data) == {"host": "a", "port": 2} + + +def test_filter_to_model_fields_accepts_a_model_instance(): + assert TypeUtils.filter_to_model_fields(_FilterModel(), {"port": 1, "nope": 2}) == {"port": 1} + + +def test_filter_to_model_fields_does_not_modify_its_input_and_handles_empty(): + data = {"host": "a", "other": "x"} + + TypeUtils.filter_to_model_fields(_FilterModel, data) + + assert data == {"host": "a", "other": "x"} + assert TypeUtils.filter_to_model_fields(_FilterModel, {}) == {} + + +def test_filter_to_model_fields_result_validates_against_a_model_that_forbids_extra(): + from pydantic import ConfigDict + + class Strict(BaseModel): + model_config = ConfigDict(extra="forbid") + host: str + + Strict.model_validate(TypeUtils.filter_to_model_fields(Strict, {"host": "a", "other": 1})) diff --git a/test/unit/framework/test_utils.py b/test/unit/framework/test_utils.py new file mode 100644 index 00000000..7dc9e28e --- /dev/null +++ b/test/unit/framework/test_utils.py @@ -0,0 +1,41 @@ +# Copyright (C) 2025 Advanced Micro Devices, Inc. + +from __future__ import annotations + +from typing import Literal, Optional, Union + +from nodescraper.utils import bytes_to_human_readable, find_annotation_in_container + + +class Target: + pass + + +def test_find_annotation_in_container_union(): + """Baseline: the target type is found inside a Union.""" + assert find_annotation_in_container(Union[int, str], int) == (int, [Union]) + + +def test_find_annotation_in_container_not_found(): + """Baseline: a missing target type returns None.""" + assert find_annotation_in_container(Union[int, str], Target) == (None, []) + + +def test_find_annotation_in_container_supports_literal(): + """Literal is documented as a supported container and must not raise.""" + assert find_annotation_in_container(Optional[Literal["a", "b"]], Target) == (None, []) + + +def test_find_annotation_in_container_literal_nested_in_generic(): + """A Literal nested inside another container must not raise.""" + assert find_annotation_in_container(dict[str, Literal["a", "b"]], Target) == (None, []) + + +def test_bytes_to_human_readable_terabytes(): + """Baseline: a terabyte scale value is rendered with the TB unit.""" + assert bytes_to_human_readable(2 << 40) == "2.0 TB" + + +def test_bytes_to_human_readable_petabytes(): + """PB is a documented unit, so petabyte scale values must not be reported in TB.""" + assert bytes_to_human_readable(2 << 50) == "2.0 PB" diff --git a/test/unit/plugin/fixtures/dmidecode_full.txt b/test/unit/plugin/fixtures/dmidecode_full.txt new file mode 100644 index 00000000..0fc8d89a --- /dev/null +++ b/test/unit/plugin/fixtures/dmidecode_full.txt @@ -0,0 +1,63 @@ +# dmidecode 3.3 +Getting SMBIOS data from sysfs. +SMBIOS 3.5.0 present. + +Handle 0x0036, DMI type 16, 23 bytes +Physical Memory Array + Location: System Board Or Motherboard + Use: System Memory + Size: 6 TB + Number Of Devices: 3 + +Handle 0x0041, DMI type 17, 92 bytes +Memory Device + Array Handle: 0x0036 + Total Width: 72 bits + Data Width: 64 bits + Size: 16384 MB + Form Factor: DIMM + Locator: DIMM_A1 + Bank Locator: NODE 0 + Type: DDR4 + Speed: 3200 MT/s + Manufacturer: Micron Technology + Serial Number: DEADBEEF + Part Number: 36ASF2G72PZ-3G2E1 + Rank: 2 + Configured Memory Speed: 2933 MT/s + Memory Operating Mode Capability: Volatile memory + Volatile Size: 8 GB + Cache Size: None + Logical Size: None + +Handle 0x0042, DMI type 17, 92 bytes +Memory Device + Array Handle: 0x0036 + Total Width: Unknown + Data Width: Unknown + Size: No Module Installed + Form Factor: Unknown + Locator: DIMM_A2 + Bank Locator: NODE 0 + Type: Unknown + Speed: Unknown + Manufacturer: Not Specified + Serial Number: Not Specified + Part Number: Not Specified + +Handle 0x0043, DMI type 17, 92 bytes +Memory Device + Array Handle: 0x0036 + Total Width: 80 bits + Data Width: 64 bits + Size: 64 GB + Form Factor: DIMM + Locator: DIMM_B1 + Bank Locator: NODE 1 + Type: DDR5 + Speed: 4800 MT/s + Manufacturer: To Be Filled By O.E.M. + Serial Number: 00000000 + Part Number: M393A8G40AB2-CWE + Rank: 2 + Configured Memory Speed: 4800 MT/s diff --git a/test/unit/plugin/fixtures/dmidecode_quiet.txt b/test/unit/plugin/fixtures/dmidecode_quiet.txt new file mode 100644 index 00000000..45ca540a --- /dev/null +++ b/test/unit/plugin/fixtures/dmidecode_quiet.txt @@ -0,0 +1,767 @@ +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_A + Bank Locator: _Node0_Channel0_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2432-50F82DCB + Asset Tag: CPU0_A_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_B + Bank Locator: _Node0_Channel1_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2432-50F83B85 + Asset Tag: CPU0_B_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GiB + Form Factor: DIMM + Set: None + Locator: CPU0_C + Bank Locator: _Node0_Channel2_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2432-50F83B92 + Asset Tag: CPU0_C_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_D + Bank Locator: _Node0_Channel3_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2432-50F83B8C + Asset Tag: CPU0_D_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_E + Bank Locator: _Node0_Channel4_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2432-50F833EE + Asset Tag: CPU0_E_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_F + Bank Locator: _Node0_Channel5_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5A2B + Asset Tag: CPU0_F_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_G + Bank Locator: _Node0_Channel6_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2432-50F833D3 + Asset Tag: CPU0_G_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_H + Bank Locator: _Node0_Channel7_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E59BD + Asset Tag: CPU0_H_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_I + Bank Locator: _Node0_Channel8_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5880 + Asset Tag: CPU0_I_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_J + Bank Locator: _Node0_Channel9_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5998 + Asset Tag: CPU0_J_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_K + Bank Locator: _Node0_Channel10_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5207 + Asset Tag: CPU0_K_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU0_L + Bank Locator: _Node0_Channel11_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5169 + Asset Tag: CPU0_L_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_A + Bank Locator: _Node1_Channel0_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E517A + Asset Tag: CPU1_A_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_B + Bank Locator: _Node1_Channel1_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5311 + Asset Tag: CPU1_B_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_C + Bank Locator: _Node1_Channel2_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5778 + Asset Tag: CPU1_C_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_D + Bank Locator: _Node1_Channel3_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5884 + Asset Tag: CPU1_D_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_E + Bank Locator: _Node1_Channel4_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5755 + Asset Tag: CPU1_E_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_F + Bank Locator: _Node1_Channel5_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E520A + Asset Tag: CPU1_F_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_G + Bank Locator: _Node1_Channel6_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5774 + Asset Tag: CPU1_G_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_H + Bank Locator: _Node1_Channel7_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E56E5 + Asset Tag: CPU1_H_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_I + Bank Locator: _Node1_Channel8_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5883 + Asset Tag: CPU1_I_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_J + Bank Locator: _Node1_Channel9_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E56E6 + Asset Tag: CPU1_J_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_K + Bank Locator: _Node1_Channel10_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E599C + Asset Tag: CPU1_K_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None + +Memory Device + Total Width: 80 bits + Data Width: 64 bits + Size: 96 GB + Form Factor: DIMM + Set: None + Locator: CPU1_L + Bank Locator: _Node1_Channel11_Dimm0 + Type: DDR5 + Type Detail: Synchronous Registered (Buffered) + Speed: 5600 MT/s + Manufacturer: Samsung + Serial Number: 2435-512E5D04 + Asset Tag: CPU1_L_AssetTag + Part Number: M321RYGA0PB0-CWMXJ + Rank: 2 + Configured Memory Speed: 5600 MT/s + Minimum Voltage: 1.1 V + Maximum Voltage: 1.1 V + Configured Voltage: 1.1 V + Memory Technology: DRAM + Memory Operating Mode Capability: Volatile memory + Firmware Version: 0000 + Module Manufacturer ID: Bank 1, Hex 0xCE + Module Product ID: Unknown + Memory Subsystem Controller Manufacturer ID: Unknown + Memory Subsystem Controller Product ID: Unknown + Non-Volatile Size: None + Volatile Size: 96 GB + Cache Size: None + Logical Size: None diff --git a/test/unit/plugin/fixtures/dmidecode_wmic.txt b/test/unit/plugin/fixtures/dmidecode_wmic.txt new file mode 100644 index 00000000..5cbfa899 --- /dev/null +++ b/test/unit/plugin/fixtures/dmidecode_wmic.txt @@ -0,0 +1,6 @@ + +Node,Attributes,BankLabel,Capacity,Caption,ConfiguredClockSpeed,ConfiguredVoltage,CreationClassName,DataWidth,Description,DeviceLocator,FormFactor,HotSwappable,InstallDate,InterleaveDataDepth,InterleavePosition,Manufacturer,MaxVoltage,MemoryType,MinVoltage,Model,Name,OtherIdentifyingInfo,PartNumber,PositionInRow,PoweredOn,Removable,Replaceable,SerialNumber,SKU,SMBIOSMemoryType,Speed,Status,Tag,TotalWidth,TypeDetail,Version +MKMGRAEPAUL01,2,P0 CHANNEL A,8589934592,Physical Memory,6400,500,Win32_PhysicalMemory,32,Physical Memory,DIMM 0,1,,,,,Micron Technology,500,0,500,,Physical Memory,,MT62F2G32D8DR-031 WT,,,,,00000000,,35,6400,,Physical Memory 0,32,16512, +MKMGRAEPAUL01,2,P0 CHANNEL B,8589934592,Physical Memory,6400,500,Win32_PhysicalMemory,32,Physical Memory,DIMM 0,1,,,,,Micron Technology,500,0,500,,Physical Memory,,MT62F2G32D8DR-031 WT,,,,,00000000,,35,6400,,Physical Memory 1,32,16512, +MKMGRAEPAUL01,2,P0 CHANNEL C,8589934592,Physical Memory,6400,500,Win32_PhysicalMemory,32,Physical Memory,DIMM 0,1,,,,,Micron Technology,500,0,500,,Physical Memory,,MT62F2G32D8DR-031 WT,,,,,00000000,,35,6400,,Physical Memory 2,32,16512, +MKMGRAEPAUL01,2,P0 CHANNEL D,8589934592,Physical Memory,6400,500,Win32_PhysicalMemory,32,Physical Memory,DIMM 0,1,,,,,Micron Technology,500,0,500,,Physical Memory,,MT62F2G32D8DR-031 WT,,,,,00000000,,35,6400,,Physical Memory 3,32,16512, diff --git a/test/unit/plugin/test_afid_events_bmc_schema.py b/test/unit/plugin/test_afid_events_bmc_schema.py index 8529577c..cdcec83b 100644 --- a/test/unit/plugin/test_afid_events_bmc_schema.py +++ b/test/unit/plugin/test_afid_events_bmc_schema.py @@ -24,6 +24,7 @@ # ############################################################################### """AFID / serviceable unit extraction for OpenBMC-style LogEntry payloads.""" + from __future__ import annotations from serviceability_dummy_data import ( @@ -41,8 +42,12 @@ from nodescraper.plugins.serviceability.afid_events import ( _afid_event_from_rf_member, + amd_field_identifiers_with_units, build_afid_events_from_data, ) +from nodescraper.plugins.serviceability.se_runner import ( + drop_afids_without_serviceable_units, +) from nodescraper.plugins.serviceability.serviceability_data import ( ServiceabilityDataModel, ) @@ -80,3 +85,102 @@ def test_build_afid_events_from_data_includes_openbmc_entries(): by_afid_oam = {(e.afid, e.serviceable_unit) for e in events} assert (DUMMY_AFID_BELOW_RF, DUMMY_UNIT_A) in by_afid_oam assert (DUMMY_AFID_FATAL_HBM, DUMMY_UNIT_C) in by_afid_oam + + +def test_amd_field_identifiers_split_and_drop_members_without_a_location(): + located = { + "Id": "located", + "Oem": { + "AMD": { + "AMDFieldIdentifiers": [ + { + "AFID": DUMMY_AFID_A, + "ServiceableUnits": [{"@odata.id": f"/redfish/v1/Chassis/{DUMMY_UNIT_B}"}], + } + ] + } + }, + } + missing = { + "@odata.id": "/redfish/v1/Systems/dummy/LogServices/EventLog/Entries/9", + "Id": "missing", + "Oem": {"AMD": {"AMDFieldIdentifiers": [{"AFID": 11045, "Description": "update"}]}}, + } + mixed = { + "Id": "mixed", + "Oem": { + "AMD": { + "AMDFieldIdentifiers": [ + {"AFID": 11045}, + { + "AFID": DUMMY_AFID_A, + "ServiceableUnits": [{"@odata.id": f"/redfish/v1/Chassis/{DUMMY_UNIT_A}"}], + }, + ] + } + }, + } + plain = {"Id": "plain", "Message": "no oem"} + usable, unusable = amd_field_identifiers_with_units(missing["Oem"]) + assert usable == [] + assert unusable == [{"AFID": 11045, "Description": "update"}] + assert amd_field_identifiers_with_units(plain) is None + kept = drop_afids_without_serviceable_units([located, missing, mixed, plain]) + kept_ids = [member["Id"] for member in kept] + assert kept_ids == ["located", "mixed", "plain"] + mixed_kept = next(member for member in kept if member["Id"] == "mixed") + assert mixed_kept["Oem"]["AMD"]["AMDFieldIdentifiers"] == [ + { + "AFID": DUMMY_AFID_A, + "ServiceableUnits": [{"@odata.id": f"/redfish/v1/Chassis/{DUMMY_UNIT_A}"}], + } + ] + + +def test_ok_severity_without_serviceable_units_is_silent(caplog): + member = { + "Id": "ok-update", + "@odata.id": "/redfish/v1/Systems/dummy/LogServices/EventLog/Entries/1", + "Severity": "OK", + "Oem": {"AMD": {"AMDFieldIdentifiers": [{"AFID": 11045}]}}, + } + with caplog.at_level("WARNING"): + kept = drop_afids_without_serviceable_units([member]) + assert kept == [] + assert "11045" not in caplog.text + + +def test_non_ok_severity_without_serviceable_units_warns(caplog): + member = { + "Id": "fault", + "@odata.id": "/redfish/v1/Systems/dummy/LogServices/EventLog/Entries/2", + "Severity": "Critical", + "Oem": {"AMD": {"AMDFieldIdentifiers": [{"AFID": 11045}]}}, + } + with caplog.at_level("WARNING"): + kept = drop_afids_without_serviceable_units([member]) + assert kept == [] + assert "Waiving AFID 11045" in caplog.text + + +def test_serviceable_unit_from_oem_amd_field_identifiers(): + ev = _afid_event_from_rf_member( + { + "Created": DUMMY_TIMESTAMP, + "Oem": { + "AMD": { + "AMDFieldIdentifiers": [ + { + "AFID": DUMMY_AFID_A, + "ServiceableUnits": [ + {"@odata.id": f"/redfish/v1/Chassis/{DUMMY_UNIT_B}"} + ], + } + ] + } + }, + } + ) + assert ev is not None + assert ev.afid == DUMMY_AFID_A + assert ev.serviceable_unit == DUMMY_UNIT_B diff --git a/test/unit/plugin/test_amc_redfish_diag.py b/test/unit/plugin/test_amc_redfish_diag.py new file mode 100644 index 00000000..9b00685d --- /dev/null +++ b/test/unit/plugin/test_amc_redfish_diag.py @@ -0,0 +1,448 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import patch + +import pytest + +from nodescraper.connection.redfish import ( + RedfishGetResult, + RedfishSshProxyConnectionManager, +) +from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.pluginregistry import PluginRegistry +from nodescraper.plugins.ooband.amc_redfish_diag import ( + AmcDiagCollectionSpec, + AmcRedfishDiagCollector, + AmcRedfishDiagCollectorArgs, + AmcRedfishDiagPlugin, +) +from nodescraper.taskresulthooks.filesystemloghook import FileSystemLogHook + + +@pytest.fixture +def amc_collector(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + return AmcRedfishDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + ) + + +def test_amc_redfish_diag_plugin_registers(): + assert AmcRedfishDiagPlugin.is_valid() + assert AmcRedfishDiagPlugin.CONNECTION_TYPE is RedfishSshProxyConnectionManager + assert "AmcRedfishDiagPlugin" in PluginRegistry().plugins + assert "RedfishSshProxyConnectionManager" in PluginRegistry().connection_managers + + +def test_amc_redfish_diag_plugin_is_collection_only(): + assert AmcRedfishDiagPlugin.ANALYZER is None + assert AmcRedfishDiagPlugin.ANALYZER_ARGS is None + + +def test_amc_collector_no_jobs(amc_collector): + result, data = amc_collector.collect_data(args=AmcRedfishDiagCollectorArgs(collections=[])) + assert result.status == ExecutionStatus.NOT_RAN + assert data is None + + +def _get_side_effect(path): + p = str(path) + if p.endswith("/LogServices"): + if "Managers" in p: + oid = "/redfish/v1/Managers/dummy-amc/LogServices/Dump" + else: + oid = "/redfish/v1/Systems/dummy-system/LogServices/DiagLogs" + return RedfishGetResult( + path=p, + success=True, + data={"Members": [{"@odata.id": oid}]}, + status_code=200, + ) + if p.endswith("/Dump") or p.endswith("/DiagLogs"): + return RedfishGetResult( + path=p, + success=True, + data={"Actions": {"#LogService.CollectDiagnosticData": {}}}, + status_code=200, + ) + return RedfishGetResult(path=p, success=False, error="unexpected", status_code=404) + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_runs_manager_and_system_jobs(mock_collect, amc_collector): + mock_collect.return_value = (b"archive", {"Id": "1"}, None) + amc_collector.connection.run_get.side_effect = _get_side_effect + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + system_ids=["dummy-system"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + AmcDiagCollectionSpec( + root="Systems", + diagnostic_data_type="OEM", + oem_data_type="AllLogs", + ), + ], + ) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert "Managers:Manager" in data.results + assert "Systems:OEM:AllLogs" in data.results + assert mock_collect.call_count == 2 + first_kwargs = mock_collect.call_args_list[0].kwargs + assert first_kwargs["diagnostic_data_type"] == "Manager" + second_kwargs = mock_collect.call_args_list[1].kwargs + assert second_kwargs["diagnostic_data_type"] == "OEM" + assert second_kwargs["oem_diagnostic_type"] == "AllLogs" + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_missing_log_service(mock_collect, amc_collector): + amc_collector.connection.run_get.return_value = RedfishGetResult( + path="/redfish/v1/Managers/dummy-amc/LogServices", + success=True, + data={"Members": []}, + status_code=200, + ) + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + system_ids=["dummy-system"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + ) + ) + assert result.status == ExecutionStatus.ERROR + assert data is not None + assert data.results["Managers:Manager"].success is False + mock_collect.assert_not_called() + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_output_dir_is_diag_logs( + mock_collect, system_info, redfish_conn_mock, tmp_path +): + mock_collect.return_value = (b"archive", {"Id": "1"}, None) + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _get_side_effect + collector = AmcRedfishDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + log_path=str(tmp_path), + ) + collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + ) + ) + output_dir = mock_collect.call_args.kwargs["output_dir"] + assert output_dir == (tmp_path / "diag_logs").resolve() + assert not output_dir.exists() + + +def _both_collections(): + return [ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + AmcDiagCollectionSpec(root="Systems", diagnostic_data_type="OEM", oem_data_type="AllLogs"), + ] + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_partial_failure_is_warning(mock_collect, amc_collector): + def side_effect(conn, log_service_path, diagnostic_data_type, **kwargs): + if diagnostic_data_type == "Manager": + return (None, None, "LogEntry GET failed") + return (b"archive", {"Id": "34"}, None) + + mock_collect.side_effect = side_effect + amc_collector.connection.run_get.side_effect = _get_side_effect + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + system_ids=["dummy-system"], + collections=_both_collections(), + ) + ) + assert result.status == ExecutionStatus.WARNING + assert data.results["Managers:Manager"].success is False + assert data.results["Systems:OEM:AllLogs"].success is True + warnings = [e for e in result.events if e.priority == EventPriority.WARNING] + assert len(warnings) == 1 + assert warnings[0].description == "AMC diag Managers:Manager: error" + assert warnings[0].data["response_body"] == "LogEntry GET failed" + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_all_failed_is_error_with_event_per_failure(mock_collect, amc_collector): + mock_collect.return_value = (None, None, "LogEntry GET failed") + amc_collector.connection.run_get.side_effect = _get_side_effect + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + system_ids=["dummy-system"], + collections=_both_collections(), + ) + ) + assert result.status == ExecutionStatus.ERROR + assert all(not r.success for r in data.results.values()) + warnings = [e for e in result.events if e.priority == EventPriority.WARNING] + assert len(warnings) == 2 + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_redfish_body_stays_on_event_not_description(mock_collect, amc_collector): + body = '{"error":{"code":"Base.1.21.ResourceInUse",' '"message":"The resource is in use."}}' + full = f"Unexpected status 503 for CollectDiagnosticData: {body}" + mock_collect.return_value = (None, None, full) + amc_collector.connection.run_get.side_effect = _get_side_effect + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + ) + ) + warning = result.events[0] + assert warning.description == "AMC diag Managers:Manager: error" + assert warning.data["response_body"] == full + assert data.results["Managers:Manager"].error == full + assert "{" not in result.message + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_amc_collector_logs_event_for_ssh_failure(mock_collect, amc_collector): + mock_collect.return_value = (None, None, "Timeout opening channel.") + amc_collector.connection.run_get.side_effect = _get_side_effect + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + ) + ) + warning = result.events[0] + assert warning.description == "AMC diag Managers:Manager: error" + assert warning.data["response_body"] == "Timeout opening channel." + assert data.results["Managers:Manager"].error == "Timeout opening channel." + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +def test_failed_collection_skips_diag_logs_and_writes_events( + mock_collect, system_info, redfish_conn_mock, tmp_path +): + full = ( + "Unexpected status 503 for CollectDiagnosticData: " + '{"error":{"code":"Base.1.21.ResourceInUse","message":"The resource is in use."}}' + ) + mock_collect.return_value = (None, None, full) + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _get_side_effect + collector = AmcRedfishDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + log_path=str(tmp_path), + task_result_hooks=[FileSystemLogHook(log_base_path=str(tmp_path), include_task_path=False)], + ) + result, _data = collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + ) + ) + assert result.status == ExecutionStatus.ERROR + assert not (tmp_path / "diag_logs").exists() + event_log = (tmp_path / "events.json").read_text(encoding="utf-8") + assert "ResourceInUse" in event_log + assert "The resource is in use." in event_log + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector._parse_clear_endpoint") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.clear_redfish_logs") +def test_amc_clear_logs_called_after_success(mock_clear, mock_parse, mock_collect, amc_collector): + from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + ) + + mock_collect.return_value = (b"archive", {"Id": "1"}, None) + amc_collector.connection.run_get.side_effect = _get_side_effect + endpoint = ClearLogEndpoint( + path="/redfish/v1/Managers/dummy-amc/LogServices/Dump/Actions/LogService.ClearLog" + ) + mock_parse.return_value = endpoint + mock_clear.return_value = [ClearLogResult(path=endpoint.path, success=True, status_code=200)] + + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + system_ids=["dummy-system"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.OK + mock_parse.assert_called_once() + mock_clear.assert_called_once() + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector._parse_clear_endpoint") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.clear_redfish_logs") +def test_amc_clear_logs_not_called_when_all_failed( + mock_clear, mock_parse, mock_collect, amc_collector +): + mock_collect.return_value = (None, None, "Task timeout") + amc_collector.connection.run_get.side_effect = _get_side_effect + + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.ERROR + mock_parse.assert_not_called() + mock_clear.assert_not_called() + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector._parse_clear_endpoint") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.clear_redfish_logs") +def test_amc_clear_log_failure_downgrades_ok_to_warning( + mock_clear, mock_parse, mock_collect, amc_collector +): + from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + ) + + mock_collect.return_value = (b"archive", {"Id": "1"}, None) + amc_collector.connection.run_get.side_effect = _get_side_effect + endpoint = ClearLogEndpoint(path="/some/ClearLog") + mock_parse.return_value = endpoint + mock_clear.return_value = [ + ClearLogResult(path=endpoint.path, success=False, status_code=503, error="Server Error") + ] + + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.WARNING + assert "log store(s) failed to clear" in result.message + warning_events = [e for e in result.events if e.priority == EventPriority.WARNING] + assert any("Failed to clear" in e.description for e in warning_events) + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector._parse_clear_endpoint") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.clear_redfish_logs") +def test_amc_clear_logs_skipped_when_passive( + mock_clear, mock_parse, mock_collect, system_info, redfish_conn_mock +): + from nodescraper.enums import SystemInteractionLevel + + mock_collect.return_value = (b"archive", {"Id": "1"}, None) + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _get_side_effect + collector = AmcRedfishDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + system_interaction_level=SystemInteractionLevel.PASSIVE, + ) + result, data = collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + collections=[ + AmcDiagCollectionSpec(root="Managers", diagnostic_data_type="Manager"), + ], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.OK + mock_parse.assert_not_called() + mock_clear.assert_not_called() + + +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector._parse_clear_endpoint") +@patch("nodescraper.plugins.ooband.amc_redfish_diag.amc_diag_collector.clear_redfish_logs") +def test_amc_clear_only_clears_successful_log_services( + mock_clear, mock_parse, mock_collect, amc_collector +): + """Only log services whose collection succeeded are cleared; failed ones are skipped.""" + from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + ) + + def collect_side(conn, log_service_path, diagnostic_data_type, **kwargs): + if diagnostic_data_type == "Manager": + return (None, None, "Task timeout") + return (b"archive", {"Id": "1"}, None) + + mock_collect.side_effect = collect_side + amc_collector.connection.run_get.side_effect = _get_side_effect + endpoint = ClearLogEndpoint(path="/some/Systems/ClearLog") + mock_parse.return_value = endpoint + mock_clear.return_value = [ClearLogResult(path=endpoint.path, success=True, status_code=200)] + + result, data = amc_collector.collect_data( + args=AmcRedfishDiagCollectorArgs( + manager_ids=["dummy-amc"], + system_ids=["dummy-system"], + collections=_both_collections(), + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.WARNING + assert data.results["Managers:Manager"].success is False + assert data.results["Systems:OEM:AllLogs"].success is True + # _parse_clear_endpoint called once (for the Systems log service that succeeded only) + mock_parse.assert_called_once() + mock_clear.assert_called_once() diff --git a/test/unit/plugin/test_amdsmi_analyzer.py b/test/unit/plugin/test_amdsmi_analyzer.py index 560a9ba4..25c0a90f 100644 --- a/test/unit/plugin/test_amdsmi_analyzer.py +++ b/test/unit/plugin/test_amdsmi_analyzer.py @@ -28,7 +28,7 @@ import pytest -from nodescraper.enums import EventPriority +from nodescraper.enums import EventPriority, ExecutionStatus from nodescraper.plugins.inband.amdsmi.amdsmi_analyzer import ( AmdSmiAnalyzer, _gpu_mismatch_description, @@ -40,8 +40,12 @@ AmdSmiStatic, AmdSmiVersion, EccState, + Fabric, + FabricInfo, Fw, FwListItem, + LinkStatusTable, + MetricClockData, MetricEccTotals, MetricPcie, Partition, @@ -62,9 +66,13 @@ StaticVram, ValueUnit, XgmiLinkMetrics, + XgmiLinks, XgmiMetrics, ) -from nodescraper.plugins.inband.amdsmi.analyzer_args import AmdSmiAnalyzerArgs +from nodescraper.plugins.inband.amdsmi.analyzer_args import ( + AmdSmiAnalyzerArgs, + PowerConfig, +) @pytest.fixture @@ -314,6 +322,80 @@ def test_check_expected_max_power_ppt0(mock_analyzer): assert "GPU max power mismatch" in analyzer.result.events[0].description +def test_check_power_cap_consistency_success(mock_analyzer): + """Matching GPU power caps pass consistency validation.""" + analyzer = mock_analyzer + static_data = [ + create_static_gpu(0, max_power=550.0), + create_static_gpu(1, max_power=550.0), + ] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=False), + ) + + assert not analyzer.result.events + + +def test_check_power_cap_consistency_mismatch(mock_analyzer): + """Different GPU power caps generate an error.""" + analyzer = mock_analyzer + static_data = [ + create_static_gpu(0, max_power=550.0), + create_static_gpu(1, max_power=450.0), + ] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=False), + ) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].category == "PLATFORM" + assert analyzer.result.events[0].priority == EventPriority.ERROR + assert "GPU power cap mismatch" in analyzer.result.events[0].description + assert "GPU 1=450.0 W" in analyzer.result.events[0].description + assert analyzer.result.events[0].data["power_caps"] == {0: 550.0, 1: 450.0} + assert analyzer.result.events[0].data["mismatched_power_caps"] == {1: 450.0} + assert analyzer.result.events[0].data["expected_power_cap"] == 550.0 + + +def test_check_power_cap_consistency_multiple_mismatches(mock_analyzer): + """Every differing power cap is reported in a single event.""" + analyzer = mock_analyzer + static_data = [create_static_gpu(gpu, max_power=float(500 + gpu * 10)) for gpu in range(8)] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=False), + ) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.ERROR + assert event.data["gpus"] == [1, 2, 3, 4, 5, 6, 7] + assert event.data["expected_power_cap"] == 500.0 + assert "GPU 1=510.0 W" in event.description + assert "GPU 7=570.0 W" in event.description + + +def test_check_power_cap_consistency_allowed(mock_analyzer): + """Power-cap mismatches are skipped when explicitly allowed.""" + analyzer = mock_analyzer + static_data = [ + create_static_gpu(0, max_power=550.0), + create_static_gpu(1, max_power=450.0), + ] + + analyzer.check_power_cap_consistency( + static_data, + PowerConfig(power_cap_mismatch_allowed=True), + ) + + assert not analyzer.result.events + + def test_check_expected_driver_version_success(mock_analyzer): """Test check_expected_driver_version passes when all GPUs have correct driver.""" analyzer = mock_analyzer @@ -751,6 +833,96 @@ def test_check_expected_xgmi_link_speed_missing_bit_rate(mock_analyzer): assert "XGMI link speed not available" in analyzer.result.events[0].description +def test_check_xgmi_or_peer_links_status_accepts_up_and_self_links(mock_analyzer): + """Healthy links generate an informational event and self-links are ignored.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.UP, LinkStatusTable.SELF], + ), + XgmiLinks( + gpu=1, + bdf="0000:02:00.0", + link_status=[LinkStatusTable.UP], + ), + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.INFO + assert "All XGMI/peer GPU links are working fine" in analyzer.result.events[0].description + + +def test_check_xgmi_or_peer_links_status_requires_healthy_links(mock_analyzer): + """Self-only links are ignored and do not count as all links working.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.SELF], + ) + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert not any( + "All XGMI/peer GPU links are working fine" in event.description + for event in analyzer.result.events + ) + + +def test_check_xgmi_or_peer_links_status_reports_down_and_degraded_links(mock_analyzer): + """Down and degraded links generate IO warnings.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.DOWN, LinkStatusTable.DISABLED], + ) + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert len(analyzer.result.events) == 2 + assert all(event.category == "IO" for event in analyzer.result.events) + assert all(event.priority == EventPriority.WARNING for event in analyzer.result.events) + assert analyzer.result.events[0].data["link_error_count"] == 1 + assert analyzer.result.events[1].data["degraded_links"][0]["status"] == "X" + + +def test_check_xgmi_or_peer_links_status_reports_degraded_links(mock_analyzer): + """Disabled/degraded links generate warnings.""" + analyzer = mock_analyzer + xgmi_links = [ + XgmiLinks( + gpu=0, + bdf="0000:01:00.0", + link_status=[LinkStatusTable.DISABLED], + ) + ] + + analyzer.check_xgmi_or_peer_links_status(xgmi_links) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + + +def test_check_xgmi_or_peer_links_status_reports_missing_data(mock_analyzer): + """Configured link validation reports when XGMI data is unavailable.""" + analyzer = mock_analyzer + + analyzer.check_xgmi_or_peer_links_status(None) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + assert "XGMI/peer link data is not available" in analyzer.result.events[0].description + + def test_analyze_data_full_workflow(mock_analyzer): """Test full analyze_data workflow with various checks.""" analyzer = mock_analyzer @@ -882,10 +1054,24 @@ def _minimal_amdsmi_metric( ecc: Optional[MetricEccTotals] = None, ecc_blocks: Optional[dict] = None, power_management: Optional[str] = None, + mem_usage: Optional[dict] = None, ) -> AmdSmiMetric: """Build minimal AmdSmiMetric for PCIe/ECC tests with all required fields present.""" pcie_dict = pcie.model_dump() if pcie is not None else {k: None for k in _PCIE_KEYS} ecc_dict = ecc.model_dump() if ecc is not None else {k: None for k in _ECC_TOTALS_KEYS} + mem_usage_dict = { + "total_vram": None, + "used_vram": None, + "free_vram": None, + "total_visible_vram": None, + "used_visible_vram": None, + "free_visible_vram": None, + "total_gtt": None, + "used_gtt": None, + "free_gtt": None, + } + if mem_usage is not None: + mem_usage_dict.update(mem_usage) return AmdSmiMetric.model_validate( { "gpu": gpu, @@ -917,22 +1103,50 @@ def _minimal_amdsmi_metric( "perf_level": None, "xgmi_err": None, "energy": None, - "mem_usage": { - "total_vram": None, - "used_vram": None, - "free_vram": None, - "total_visible_vram": None, - "used_visible_vram": None, - "free_visible_vram": None, - "total_gtt": None, - "used_gtt": None, - "free_gtt": None, - }, + "mem_usage": mem_usage_dict, "throttle": {}, } ) +def test_metric_accepts_legacy_and_per_block_clocks(): + """Older clock objects and per-block clocks validate together.""" + raw = _minimal_amdsmi_metric().model_dump() + raw["clock"] = { + "GFX_0": { + "clk": {"value": 132, "unit": "MHz"}, + "min_clk": {"value": 500, "unit": "MHz"}, + "max_clk": {"value": 2100, "unit": "MHz"}, + "clk_locked": 0, + "deep_sleep": 0, + }, + "uclk_aid": { + "aid_0": {"value": 1900, "unit": "MHz"}, + "aid_1": {"value": 1900, "unit": "MHz"}, + }, + "socclks_mid": { + "mid_0": {"value": 1350, "unit": "MHz"}, + "mid_1": "N/A", + }, + } + raw["pcie"].pop("lc_perf_other_end_recovery", None) + raw["pcie"]["lc_perf_other_end_recovery_count"] = 4 + + metric = AmdSmiMetric.model_validate(raw) + + assert isinstance(metric.clock["GFX_0"], MetricClockData) + assert metric.clock["GFX_0"].min_clk.value == 500 + assert metric.clock["uclk_aid"]["aid_0"]["value"] == 1900 + assert metric.clock["socclks_mid"]["mid_1"] == "N/A" + assert metric.pcie.lc_perf_other_end_recovery == 4 + + legacy = _minimal_amdsmi_metric() + legacy_raw = legacy.model_dump() + legacy_raw["pcie"]["lc_perf_other_end_recovery"] = 2 + legacy_raw["pcie"]["lc_perf_other_end_recovery_count"] = 9 + assert AmdSmiMetric.model_validate(legacy_raw).pcie.lc_perf_other_end_recovery == 2 + + def test_gpu_mismatch_description_helpers(): """Per-GPU mismatch helpers include GPU id, expected, and actual in output.""" desc, details = _gpu_mismatch_description( @@ -1002,10 +1216,94 @@ def test_analyze_data_expected_power_management(mock_analyzer): assert not any("power_management mismatch" in e.description for e in result.events) +def test_check_gpu_memory_meets_minimum(mock_analyzer): + """GPU VRAM availability at the configured minimum passes.""" + analyzer = mock_analyzer + metrics = [ + _minimal_amdsmi_metric( + 0, + mem_usage={ + "total_vram": {"value": 100, "unit": "B"}, + "free_vram": {"value": 95, "unit": "B"}, + }, + ) + ] + + analyzer.check_gpu_memory(metrics, 95) + + assert not analyzer.result.events + + +def test_check_gpu_memory_below_minimum_logs_warning(mock_analyzer): + """GPU VRAM availability below the configured minimum logs a warning.""" + analyzer = mock_analyzer + metrics = [ + _minimal_amdsmi_metric( + 1, + mem_usage={ + "total_vram": {"value": 100, "unit": "B"}, + "free_vram": {"value": 90, "unit": "B"}, + }, + ) + ] + + analyzer.check_gpu_memory(metrics, 95) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.WARNING + assert "GPU 1 free VRAM is 90.00%" in event.description + assert event.data["available_percent"] == 90.0 + assert event.data["minimum_available_percent"] == 95 + + +def test_check_gpu_memory_no_data(mock_analyzer): + """check_gpu_memory warns and returns when metric data is missing.""" + analyzer = mock_analyzer + + analyzer.check_gpu_memory(None, 95) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + assert "No AMD SMI metric data available" in analyzer.result.events[0].description + + +def test_analyze_data_gpu_memory_no_metric_data(mock_analyzer): + """analyze_data warns when gpu_memory is set but metric data is missing.""" + analyzer = mock_analyzer + data = AmdSmiDataModel( + version=None, + static=None, + process=None, + firmware=None, + partition=None, + gpu_list=None, + metric=None, + ) + args = AmdSmiAnalyzerArgs.model_validate({"gpu_memory": {"minimum_available_percent": 95}}) + + result = analyzer.analyze_data(data, args) + + assert any( + event.priority == EventPriority.WARNING + and "No AMD SMI metric data available" in event.description + for event in result.events + ) + + def test_amdsmi_analyzer_args_rejects_unknown_fields(): """Plugin config must only use declared AmdSmiAnalyzerArgs fields.""" from pydantic import ValidationError + args = AmdSmiAnalyzerArgs.model_validate({"power": {"power_cap_mismatch_allowed": False}}) + assert args.power is not None + assert args.power.power_cap_mismatch_allowed is False + args = AmdSmiAnalyzerArgs.model_validate({"gpu_memory": {"minimum_available_percent": 95}}) + assert args.gpu_memory is not None + assert args.gpu_memory.minimum_available_percent == 95 + args = AmdSmiAnalyzerArgs.model_validate({"check_xgmi_or_peer_links_status": True}) + assert args.check_xgmi_or_peer_links_status is True + with pytest.raises(ValidationError): AmdSmiAnalyzerArgs.model_validate( {"expected_power_management": "DISABLED", "not_a_field": 1} @@ -1162,3 +1460,220 @@ def test_check_amdsmi_metric_ecc_blocks(mock_analyzer): assert any("GFX" in d and "correctable" in d for d in desc) assert any("MMHUB" in d for d in desc) assert any("HDP" in d for d in desc) + + +def create_fabric( + gpu: int = 0, + fabric_type: str = "UALOE", + accel_state: str = "ACTIVE", + ppod_id: str = "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", +) -> Fabric: + """Helper function to create a healthy mock Fabric object for testing.""" + return Fabric( + gpu=gpu, + bdf=f"{gpu + 1:04d}:01:00.0", + fabric_info=FabricInfo( + bdf=f"{gpu + 1:04d}:01:00.1", + version=4294967295, + accelerator_id=7 - gpu, + fabric_type=fabric_type, + bandwidth=ValueUnit(value=400000, unit="Mb/s"), + latency=ValueUnit(value=150, unit="ns"), + ppod_id=ppod_id, + ppod_size=72, + vpod_id=1, + vpod_size=8, + local_accelerators="1, 2, 3, 4, 5, 6, 7, 8", + local_active_accelerators=["1, 2, 3, 4, 5, 6, 7, 8"], + addr_mode="DIRECT", + accel_state=accel_state, + ), + ) + + +def test_check_fabric_success(mock_analyzer): + """Healthy fabric data on all GPUs generates no events.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_fabric(gpu) for gpu in range(4)]) + + assert len(analyzer.result.events) == 0 + + +def test_check_fabric_no_data(mock_analyzer): + """Missing fabric data logs a warning rather than an error.""" + analyzer = mock_analyzer + + analyzer.check_fabric([]) + + assert len(analyzer.result.events) == 1 + assert analyzer.result.events[0].priority == EventPriority.WARNING + assert "Fabric data is not available" in analyzer.result.events[0].description + + +def test_check_fabric_inactive_output(mock_analyzer): + """Real amd-smi fabric output from an inactive fabric reports every mismatch.""" + analyzer = mock_analyzer + + fabric_data = [ + Fabric.model_validate( + { + "gpu": gpu, + "bdf": f"{gpu + 1:04d}:01:00.0", + "fabric_info": { + "bdf": f"{gpu + 1:04d}:01:00.1", + "version": 4294967295, + "accelerator_id": 7 - gpu, + "fabric_type": "UALOE", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", + "ppod_size": 72, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNKNOWN", + }, + } + ) + for gpu in range(4) + ] + + analyzer.check_fabric(fabric_data) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.ERROR + assert event.category == "NETWORK" + mismatched_fields = {m["field"] for m in event.data["mismatches"]} + assert mismatched_fields == { + "accel_state", + "local_accelerators", + "local_active_accelerators", + "vpod_size", + } + assert {m["gpu"] for m in event.data["mismatches"]} == {0, 1, 2, 3} + assert "Fabric data mismatch on 4 GPU(s)" in event.description + + +def test_check_fabric_unexpected_fabric_type(mock_analyzer): + """A fabric_type other than the expected one is reported.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_fabric(0, fabric_type="XGMI")]) + + assert len(analyzer.result.events) == 1 + mismatches = analyzer.result.events[0].data["mismatches"] + assert mismatches == [{"gpu": 0, "field": "fabric_type", "expected": "UALOE", "actual": "XGMI"}] + + +def test_check_fabric_custom_expected_values(mock_analyzer): + """Expected accel_state/fabric_type are configurable and case insensitive.""" + analyzer = mock_analyzer + + analyzer.check_fabric( + [create_fabric(0, fabric_type="xgmi", accel_state="up")], + expected_accel_state="UP", + expected_fabric_type="XGMI", + ) + + assert len(analyzer.result.events) == 0 + + +def test_check_fabric_missing_fabric_info(mock_analyzer): + """A GPU with no fabric_info reports all fields as N/A.""" + analyzer = mock_analyzer + + analyzer.check_fabric([Fabric(gpu=0, bdf="0001:01:00.0", fabric_info=None)]) + + assert len(analyzer.result.events) == 1 + mismatches = analyzer.result.events[0].data["mismatches"] + assert len(mismatches) == 7 + assert all(m["actual"] == "N/A" and m["gpu"] == 0 for m in mismatches) + + +def test_check_fabric_zero_ppod_id(mock_analyzer): + """An all-zero ppod_id is treated as a mismatch.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_fabric(0, ppod_id="00000000-0000-0000-0000-000000000000")]) + + assert len(analyzer.result.events) == 1 + mismatches = analyzer.result.events[0].data["mismatches"] + assert [m["field"] for m in mismatches] == ["ppod_id"] + assert mismatches[0]["expected"] == "non-zero" + + +def create_unconfigured_fabric(gpu: int) -> Fabric: + """Helper building amd-smi fabric output for a GPU whose fabric is not configured.""" + return Fabric.model_validate( + { + "gpu": gpu, + "bdf": f"0000:{gpu:02x}:00.0", + "fabric_info": { + "bdf": f"0000:{gpu:02x}:00.1", + "version": 4294967295, + "accelerator_id": gpu, + "fabric_type": "UNKNOWN", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "00000000-0000-0000-0000-000000000000", + "ppod_size": 0, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNCONFIGURED", + }, + } + ) + + +def test_check_fabric_unconfigured_many_gpus(mock_analyzer): + """8 GPUs failing all 7 fabric fields log one bounded ERROR event with full detail in data.""" + analyzer = mock_analyzer + + analyzer.check_fabric([create_unconfigured_fabric(gpu) for gpu in range(8)]) + + assert len(analyzer.result.events) == 1 + event = analyzer.result.events[0] + assert event.priority == EventPriority.ERROR + assert event.category == "NETWORK" + assert len(event.data["mismatches"]) == 8 + assert {m["gpu"] for m in event.data["mismatches"]} == set(range(8)) + assert {m["field"] for m in event.data["mismatches"]} == {"accel_state"} + assert len(event.description.encode("utf-8")) < 2048 + assert event.description.startswith("Fabric data mismatch on 8 GPU(s)") + + +def test_analyze_data_unconfigured_fabric_completes(mock_analyzer): + """An oversized fabric mismatch does not abort analyze_data or skip later checks.""" + analyzer = mock_analyzer + + data = AmdSmiDataModel( + version=None, + static=None, + process=None, + firmware=None, + partition=None, + gpu_list=None, + fabric=[create_unconfigured_fabric(gpu) for gpu in range(8)], + xgmi_link=[XgmiLinks(gpu=0, bdf="0000:00:00.0", link_status=[LinkStatusTable.UP])], + ) + + result = analyzer.analyze_data(data, AmdSmiAnalyzerArgs(check_xgmi_or_peer_links_status=True)) + + assert result.status != ExecutionStatus.EXECUTION_FAILURE + assert not any(e.priority == EventPriority.CRITICAL for e in result.events) + assert not any(e.description == "Validation error during analysis" for e in result.events) + error_events = [e for e in result.events if e.priority == EventPriority.ERROR] + assert len(error_events) == 1 + assert error_events[0].description.startswith("Fabric data mismatch") + assert any( + e.priority == EventPriority.INFO + and e.description == "All XGMI/peer GPU links are working fine" + for e in result.events + ) diff --git a/test/unit/plugin/test_amdsmi_collector.py b/test/unit/plugin/test_amdsmi_collector.py index 74b56266..9857880a 100644 --- a/test/unit/plugin/test_amdsmi_collector.py +++ b/test/unit/plugin/test_amdsmi_collector.py @@ -31,8 +31,13 @@ import pytest +from nodescraper.enums.eventpriority import EventPriority +from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel -from nodescraper.plugins.inband.amdsmi.amdsmi_collector import AmdSmiCollector +from nodescraper.plugins.inband.amdsmi.amdsmi_collector import ( + AmdSmiCollector, + _parse_afid_output, +) from nodescraper.plugins.inband.amdsmi.amdsmidata import ( AmdSmiDataModel, ) @@ -266,6 +271,38 @@ def test_check_amdsmi_installed(collector): assert collector._check_amdsmi_installed() is True +def test_check_amdsmi_installed_path_hit_leaves_exe_unchanged(collector, monkeypatch): + """Test that a PATH hit does not trigger the fallback search""" + mock_run_sut_cmd = MagicMock(return_value=make_cmd_result("/usr/bin/amd-smi")) + monkeypatch.setattr(collector, "_run_sut_cmd", mock_run_sut_cmd) + + assert collector._check_amdsmi_installed() is True + assert collector.AMD_SMI_EXE == "amd-smi" + mock_run_sut_cmd.assert_called_once_with("which amd-smi") + + +def test_check_amdsmi_installed_falls_back_to_rocm_path(conn_mock, system_info, monkeypatch): + """Test that a failed PATH lookup falls back to the known install paths""" + mock_run_sut_cmd = MagicMock( + side_effect=[ + make_cmd_result("", "no amd-smi in /usr/bin", 1), + make_cmd_result("/opt/rocm/bin/amd-smi"), + ] + ) + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + + assert c._check_amdsmi_installed() is True + assert c.AMD_SMI_EXE == "/opt/rocm/bin/amd-smi" + for path in AmdSmiCollector.AMD_SMI_FALLBACK_PATHS: + assert path in mock_run_sut_cmd.call_args.args[0] + + def test_check_amdsmi_not_installed(conn_mock, system_info, monkeypatch): """Test when amd-smi is not installed""" @@ -491,6 +528,46 @@ def mock_run_sut_cmd(cmd: str) -> MagicMock: assert stat[0].vbios is None +def test_static_numa_node_na(conn_mock, system_info, monkeypatch): + """N/A numa node is kept as a string, the same way affinity is.""" + + def mock_run_sut_cmd(cmd: str) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "static -g all --json" in cmd: + return make_cmd_result( + make_json_response( + [ + { + "gpu": 0, + "bus": {"bdf": "0000:0b:00.0"}, + "numa": {"node": "N/A", "affinity": "N/A"}, + }, + { + "gpu": 1, + "bus": {"bdf": "0000:0c:00.0"}, + "numa": {"node": 3, "affinity": 0}, + }, + ] + ) + ) + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + + stat = c.get_static() + assert stat is not None and len(stat) == 2 + assert stat[0].numa.node == "N/A" + assert stat[0].numa.affinity == "N/A" + assert stat[1].numa.node == 3 + assert stat[1].numa.affinity == 0 + + def test_json_parse_error(conn_mock, system_info, monkeypatch): """Test handling of malformed JSON""" @@ -520,7 +597,7 @@ def test_command_error(conn_mock, system_info, monkeypatch): def mock_cmd_error(cmd: str) -> MagicMock: if "which amd-smi" in cmd: return make_cmd_result("/usr/bin/amd-smi") - return make_cmd_result("", "Command failed", 1) + return make_cmd_result("Command output", "Command failed", 1) c = AmdSmiCollector( system_info=system_info, @@ -534,6 +611,11 @@ def mock_cmd_error(cmd: str) -> MagicMock: assert data.version is None assert data.gpu_list == [] assert len(result.events) > 0 + command_error = next( + event for event in result.events if event.description == "Error running amd-smi command" + ) + assert command_error.data["stdout"] == "Command output" + assert command_error.data["stderr"] == "Command failed" def test_multi_json_parsing(conn_mock, system_info, monkeypatch): @@ -594,6 +676,33 @@ def mock_single_json(cmd: str) -> MagicMock: assert result[0]["tool"] == "amdsmi" +def test_run_amd_smi_dict_appends_json_once(conn_mock, system_info, monkeypatch): + """Every JSON command receives one --json from the runner.""" + seen = [] + + def mock_run_sut_cmd(cmd: str) -> MagicMock: + seen.append(cmd) + return make_cmd_result(make_json_response({"ok": True})) + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + + assert c._run_amd_smi_dict(c.CMD_LIST) == {"ok": True} + assert c._run_amd_smi_dict(c.CMD_STATIC) == {"ok": True} + assert c._run_amd_smi_dict(c.CMD_TOPOLOGY) == {"ok": True} + assert c._run_amd_smi_dict("list --json") == {"ok": True} + assert seen == [ + "amd-smi list --json", + "amd-smi static -g all --json", + "amd-smi topology --json", + "amd-smi list --json", + ] + + def test_get_cper_afid_success(conn_mock, system_info, monkeypatch): """Test successful AFID retrieval from CPER file""" @@ -637,6 +746,69 @@ def mock_run_sut_cmd(cmd: str) -> MagicMock: afid = c._get_cper_afid("/path/to/test.cper") assert afid is None + assert any(event.priority == EventPriority.ERROR for event in c.result.events) + + +def test_get_cper_afid_table(conn_mock, system_info, monkeypatch): + """Test AFID retrieval when amd-smi prints a file_name and list of afids table""" + + def mock_run_sut_cmd(cmd: str) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "ras --afid --cper-file" in cmd: + return make_cmd_result("file_name list of afids\ncorrected-1.cper 29\n") + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + + afid = c._get_cper_afid("/path/to/corrected-1.cper") + c.result.finalize() + + assert afid == 29 + assert c.result.status == ExecutionStatus.OK + assert not any(event.priority >= EventPriority.ERROR for event in c.result.events) + + +def test_get_cper_afid_absent(conn_mock, system_info, monkeypatch): + """Test a dash or N/A AFID is recorded as none and does not fail the task""" + + def mock_run_sut_cmd(cmd: str) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "ras --afid --cper-file" in cmd: + if "boot.cper" in cmd: + return make_cmd_result("file_name list of afids\nboot.cper N/A\n") + return make_cmd_result("-\n") + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + + assert c._get_cper_afid("/path/to/boot.cper") is None + assert c._get_cper_afid("/path/to/other.cper") is None + c.result.finalize() + + assert c.result.status == ExecutionStatus.OK + assert not any(event.priority >= EventPriority.ERROR for event in c.result.events) + + +def test_parse_afid_output(): + """Parse a bare AFID, an AFID table, and an absent marker.""" + assert _parse_afid_output("12345\n") == 12345 + assert _parse_afid_output("file_name list of afids\ncorrected-1.cper 29\n") == 29 + assert _parse_afid_output("file_name list of afids\nboot.cper N/A\n") is None + assert _parse_afid_output("-\n") is None + with pytest.raises(ValueError): + _parse_afid_output("not_a_number\n") def test_get_cper_afid_command_failure(conn_mock, system_info, monkeypatch): @@ -832,3 +1004,625 @@ def mock_read_sut_file(path: str, encoding=None, strip=False, log_artifact=False assert data.cper_afids["auto1.cper"] == 11111 assert "/home/user/custom.cper" in data.cper_afids assert data.cper_afids["/home/user/custom.cper"] == 99999 + + +def fabric_json_entry(gpu: int) -> dict[str, Any]: + """Build one `amd-smi fabric --json` entry as emitted by the tool.""" + return { + "gpu": gpu, + "fabric": { + "gpu": gpu, + "bdf": f"{gpu + 1:04d}:01:00.0", + "fabric_info": { + "bdf": f"{gpu + 1:04d}:01:00.1", + "version": 4294967295, + "accelerator_id": 7 - gpu, + "fabric_type": "UALOE", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", + "ppod_size": 72, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": ["0, 0, 0, 0, 0, 0, 0, 0"], + "addr_mode": "UNKNOWN", + "accel_state": "UNKNOWN", + }, + "fabric_telemetry": "N/A", + }, + } + + +def make_fabric_collector(conn_mock, system_info, monkeypatch, fabric_payload) -> AmdSmiCollector: + """Create a collector whose only mocked amd-smi command is `fabric`.""" + + def mock_run_sut_cmd(cmd: str, sudo: bool = False) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "fabric --json" in cmd: + if fabric_payload is None: + return make_cmd_result("", "fabric not supported", 1) + return make_cmd_result(make_json_response(fabric_payload)) + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + return c + + +def test_get_fabric(conn_mock, system_info, monkeypatch): + """Test fabric parsing from the nested {'gpu': n, 'fabric': {...}} output""" + payload = [fabric_json_entry(0), fabric_json_entry(1)] + c = make_fabric_collector(conn_mock, system_info, monkeypatch, payload) + + fabric = c.get_fabric() + + assert len(fabric) == 2 + assert [f.gpu for f in fabric] == [0, 1] + assert fabric[0].bdf == "0001:01:00.0" + info = fabric[0].fabric_info + assert info is not None + assert info.fabric_type == "UALOE" + assert info.accelerator_id == 7 + assert info.ppod_id == "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb" + assert info.ppod_size == 72 + assert info.bandwidth is not None and info.bandwidth.unit == "Mb/s" + assert info.latency is not None and info.latency.unit == "ns" + + +def test_get_fabric_drops_fabric_telemetry(conn_mock, system_info, monkeypatch): + """Test that the large fabric_telemetry payload is not collected""" + entry = fabric_json_entry(0) + entry["fabric"]["fabric_telemetry"] = {"links": [{"id": i} for i in range(64)]} + c = make_fabric_collector(conn_mock, system_info, monkeypatch, [entry]) + + fabric = c.get_fabric() + + assert len(fabric) == 1 + assert "fabric_telemetry" not in fabric[0].model_dump() + + +def test_get_fabric_gpu_data_wrapper(conn_mock, system_info, monkeypatch): + """Test fabric parsing when output is wrapped in a gpu_data key""" + payload = {"gpu_data": [fabric_json_entry(0), fabric_json_entry(1), fabric_json_entry(2)]} + c = make_fabric_collector(conn_mock, system_info, monkeypatch, payload) + + fabric = c.get_fabric() + + assert [f.gpu for f in fabric] == [0, 1, 2] + + +def test_get_fabric_flat_entry(conn_mock, system_info, monkeypatch): + """Test fabric parsing when entries are not nested under a fabric key""" + payload = [fabric_json_entry(0)["fabric"]] + c = make_fabric_collector(conn_mock, system_info, monkeypatch, payload) + + fabric = c.get_fabric() + + assert len(fabric) == 1 + assert fabric[0].gpu == 0 + assert fabric[0].fabric_info is not None + + +def test_get_fabric_command_failure(conn_mock, system_info, monkeypatch): + """Test fabric returns an empty list when amd-smi fabric fails""" + c = make_fabric_collector(conn_mock, system_info, monkeypatch, None) + + assert c.get_fabric() == [] + + +def test_get_fabric_na_values(conn_mock, system_info, monkeypatch): + """Test fabric handles N/A values reported by amd-smi""" + entry = fabric_json_entry(0) + entry["fabric"]["bdf"] = "N/A" + entry["fabric"]["fabric_info"] = "N/A" + c = make_fabric_collector(conn_mock, system_info, monkeypatch, [entry]) + + fabric = c.get_fabric() + + assert len(fabric) == 1 + assert fabric[0].bdf is None + assert fabric[0].fabric_info is None + + +# NODE + + +def node_json_entry(include_gtt: bool = True) -> dict[str, Any]: + """Build one dummy `amd-smi node --json` entry as emitted by the tool.""" + node: dict[str, Any] = { + "power_management": {"limit": "N/A", "status": "N/A", "threshold": "N/A"}, + } + if include_gtt: + node["gtt"] = {"size_gb": 100.0, "size_pages": 1000} + return {"node": node} + + +def make_node_collector(conn_mock, system_info, monkeypatch, node_payload) -> AmdSmiCollector: + """Create a collector whose only mocked amd-smi command is `node`.""" + + def mock_run_sut_cmd(cmd: str, sudo: bool = False) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "node --json" in cmd: + if node_payload is None: + return make_cmd_result("", "node not supported", 1) + return make_cmd_result(make_json_response(node_payload)) + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + return c + + +def test_get_node(conn_mock, system_info, monkeypatch): + """Test node parsing from the {"node": {...}} array shape.""" + payload = [node_json_entry()] + c = make_node_collector(conn_mock, system_info, monkeypatch, payload) + + node = c.get_node() + + assert len(node) == 1 + assert node[0].node.gtt.size_gb == 100.0 + assert node[0].node.gtt.size_pages == 1000 + assert node[0].node.power_management.limit is None + assert node[0].node.power_management.status is None + assert node[0].node.power_management.threshold is None + + +def test_get_node_missing_gtt(conn_mock, system_info, monkeypatch): + """gtt may be entirely absent from a node entry.""" + payload = [node_json_entry(include_gtt=False)] + c = make_node_collector(conn_mock, system_info, monkeypatch, payload) + + node = c.get_node() + + assert len(node) == 1 + assert node[0].node.gtt is None + + +def test_get_node_command_failure(conn_mock, system_info, monkeypatch): + """Test get_node returns an empty list when the command fails.""" + c = make_node_collector(conn_mock, system_info, monkeypatch, None) + + assert c.get_node() == [] + + +def test_get_node_real_hardware_shape(conn_mock, system_info, monkeypatch): + """Real amd-smi reports power_management.limit/threshold as value+unit dicts + (not bare numbers/"N/A" strings), and includes a base_board.temperature block.""" + payload = [ + { + "node": { + "power_management": { + "limit": {"value": 0, "unit": "W"}, + "status": "DISABLED", + "threshold": "N/A", + }, + "base_board": { + "temperature": { + "UBB_FPGA": {"value": -1, "unit": "°C"}, + "IBC": {"value": -1, "unit": "°C"}, + } + }, + "gtt": {"size_gb": 256.000, "size_pages": 65900000}, + } + } + ] + c = make_node_collector(conn_mock, system_info, monkeypatch, payload) + + node = c.get_node() + + assert len(node) == 1 + pm = node[0].node.power_management + assert pm.limit.value == 0 + assert pm.limit.unit == "W" + assert pm.status == "DISABLED" + assert pm.threshold is None + + temps = node[0].node.base_board.temperature + assert temps["UBB_FPGA"].value == -1 + assert temps["UBB_FPGA"].unit == "°C" + assert temps["IBC"].value == -1 + + +# CPU / CORE METRICS + + +def cpu_metric_json_entry(cpu: int = 0) -> dict[str, Any]: + """Build one dummy `amd-smi metric --cpu all --json` entry as emitted by the tool.""" + return { + "cpu": cpu, + "power_metrics": { + "socket power": "100.000 W", + "socket power limit": "200.000 W", + "socket max power limit": "200.000 W", + }, + "prochot": {"prochot_status": 0}, + "freq_metrics": { + "fclkmemclk": {"fclk": "1000 MHz", "mclk": "2000 MHz"}, + "cclkfreqlimit": "2000 MHz", + "soc_current_active_freq_limit": {"freq": "2000 MHz", "freq_src": "['Example']"}, + "soc_freq_range": {"max_socket_freq": "2000 MHz", "min_socket_freq": "500 MHz"}, + }, + "c0_residency": {"residency": "1 %"}, + "svi_telemetry_all_rails": {"power": "1000 mW"}, + "pwr_eff_mode": {"mode": "0"}, + "metric_version": {"version": 1}, + "metrics_table": {"cpu_family": 1, "cpu_model": 1, "response": "N/A"}, + "socket_energy": {"response": "1000.0 J"}, + "ddr_bandwidth": { + "response": { + "ddr_bw_max_bw": "100 Gbps", + "ddr_bw_utilized_bw": "0 Gbps", + "ddr_bw_utilized_pct": "0 %", + } + }, + "cpu_temp": {"response": "N/A"}, + "xgmi_pstate_range": {"min_pstate": "N/A", "max_pstate": "N/A"}, + "railisofreq_policy": {"value": 0}, + "dfcstate_ctrl": {"value": 1}, + "pc6_enable": {"value": "N/A"}, + "cc6_enable": {"value": "N/A"}, + "tdelta": {"value": "N/A"}, + "enabled_commands": { + "READ_ENABLED_COMMANDS_BITMASK0": "N/A", + "READ_ENABLED_COMMANDS_BITMASK1": "N/A", + "READ_ENABLED_COMMANDS_BITMASK2": "N/A", + "WRITE_ENABLED_COMMANDS_BITMASK0": "N/A", + "WRITE_ENABLED_COMMANDS_BITMASK1": "N/A", + "WRITE_ENABLED_COMMANDS_BITMASK2": "N/A", + }, + "sdps_limit": {"value": "N/A"}, + } + + +def core_metric_json_entry(core: int = 0) -> dict[str, Any]: + """Build a `amd-smi metric --core all --json` entry as emitted by the tool.""" + return { + "core": core, + "boost_limit": {"value": 2000}, + "curr_active_freq_core_limit": {"value": "2000 MHz"}, + "core_energy": {"value": "N/A"}, + "ccd_power": {"value": "N/A"}, + "floor_limit": {"value": "N/A"}, + "eff_floor_limit": {"value": "N/A"}, + } + + +def make_cpu_core_collector( + conn_mock, system_info, monkeypatch, cpu_payload=None, core_payload=None +) -> AmdSmiCollector: + """Create a collector with metric --cpu/--core all commands.""" + + def mock_run_sut_cmd(cmd: str, sudo: bool = False) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + if "metric --cpu all --json" in cmd: + if cpu_payload is None: + return make_cmd_result("", "cpu metrics not supported", 1) + return make_cmd_result(make_json_response(cpu_payload)) + if "metric --core all --json" in cmd: + if core_payload is None: + return make_cmd_result("", "core metrics not supported", 1) + return make_cmd_result(make_json_response(core_payload)) + return make_cmd_result("") + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + return c + + +def test_get_cpu_metric(conn_mock, system_info, monkeypatch): + """Test CPU metric parsing from the amd-smi metric --cpu all --json shape.""" + payload = {"cpu_data": [cpu_metric_json_entry()]} + c = make_cpu_core_collector(conn_mock, system_info, monkeypatch, cpu_payload=payload) + + cpu_metric = c.get_cpu_metric() + + assert len(cpu_metric) == 1 + entry = cpu_metric[0] + assert entry.cpu == 0 + assert entry.power_metrics.socket_power.value == 100.0 + assert entry.power_metrics.socket_power_limit.unit == "W" + assert entry.prochot.prochot_status == 0 + assert entry.freq_metrics.fclkmemclk.fclk.value == 1000 + assert entry.ddr_bandwidth.response.ddr_bw_max_bw.value == 100 + assert entry.cpu_temp.response is None + assert entry.sdps_limit is None + assert entry.xgmi_pstate_range.min_pstate is None + + +def test_get_cpu_metric_command_failure(conn_mock, system_info, monkeypatch): + """Test get_cpu_metric returns an empty list when the command fails.""" + c = make_cpu_core_collector(conn_mock, system_info, monkeypatch, cpu_payload=None) + + assert c.get_cpu_metric() == [] + + +def test_get_core_metric(conn_mock, system_info, monkeypatch): + """Test core metric parsing from the amd-smi metric --core all --json shape.""" + payload = {"core_data": [core_metric_json_entry(), core_metric_json_entry(core=1)]} + c = make_cpu_core_collector(conn_mock, system_info, monkeypatch, core_payload=payload) + + core_metric = c.get_core_metric() + + assert [entry.core for entry in core_metric] == [0, 1] + assert core_metric[0].boost_limit.value == 2000 + assert core_metric[0].curr_active_freq_core_limit.unit == "MHz" + assert core_metric[0].core_energy is None + assert core_metric[0].ccd_power is None + assert core_metric[0].floor_limit is None + + +def test_get_core_metric_command_failure(conn_mock, system_info, monkeypatch): + """Test get_core_metric returns an empty list when the command fails.""" + c = make_cpu_core_collector(conn_mock, system_info, monkeypatch, core_payload=None) + + assert c.get_core_metric() == [] + + +# --------------------------------------------------------------------------- +# CPU metric flags that require an explicit argument (not covered by +# ``metric --cpu all``). +# --------------------------------------------------------------------------- + + +def make_cpu_arg_collector( + conn_mock, system_info, monkeypatch, responses: dict[str, Any] +) -> tuple[AmdSmiCollector, list[str]]: + """Create a collector driven by an exact command -> JSON-payload map.""" + calls: list[str] = [] + + def mock_run_sut_cmd(cmd: str) -> MagicMock: + if "which amd-smi" in cmd: + return make_cmd_result("/usr/bin/amd-smi") + assert cmd.startswith("amd-smi ") and cmd.endswith(" --json") + key = cmd[len("amd-smi ") : -len(" --json")] + calls.append(key) + if key not in responses: + return make_cmd_result("", "not captured in test", 1) + return make_cmd_result(make_json_response(responses[key])) + + c = AmdSmiCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + monkeypatch.setattr(c, "_run_sut_cmd", mock_run_sut_cmd) + return c, calls + + +def _dimm_payload(group_key: str, response: Any) -> dict: + """Build a ``{"cpu_data": [...]}`` payload for a DIMM-flag response group.""" + return {"cpu_data": [{"cpu": 0, group_key: {"response": response}}]} + + +def test_get_cpu_dimm_pow_consumption_stops_after_na_grace(conn_mock, system_info, monkeypatch): + """DIMM 0-1 populated, 2-3 N/A (real capture shape): collect the 2 real + entries and stop after CPU_SEQUENTIAL_NA_GRACE consecutive misses, never + probing address 4+.""" + responses = { + "metric --cpu-dimm-pow-consumption 0": _dimm_payload( + "dimm_pow_consumption", + { + "dimm_power_consumed": "3688 mW", + "dimm_power_update_rate": "0 ms", + "dimm_dimm_addr": 0, + }, + ), + "metric --cpu-dimm-pow-consumption 1": _dimm_payload( + "dimm_pow_consumption", + { + "dimm_power_consumed": "3562 mW", + "dimm_power_update_rate": "256 ms", + "dimm_dimm_addr": 0, + }, + ), + "metric --cpu-dimm-pow-consumption 2": _dimm_payload("dimm_pow_consumption", "N/A"), + "metric --cpu-dimm-pow-consumption 3": _dimm_payload("dimm_pow_consumption", "N/A"), + } + c, calls = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_dimm_pow_consumption() + + assert [e.dimm_addr for e in entries] == [0, 1] + assert entries[0].dimm_power_consumed.value == 3688 + assert entries[0].dimm_power_consumed.unit == "mW" + assert entries[1].dimm_power_update_rate.value == 256 + assert calls == [ + "metric --cpu-dimm-pow-consumption 0", + "metric --cpu-dimm-pow-consumption 1", + "metric --cpu-dimm-pow-consumption 2", + "metric --cpu-dimm-pow-consumption 3", + ] + + +def test_get_cpu_dimm_pow_consumption_command_failure_stops_immediately( + conn_mock, system_info, monkeypatch +): + """A hard command failure at address 0 stops enumeration immediately.""" + c, calls = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses={}) + + assert c.get_cpu_dimm_pow_consumption() == [] + assert calls == ["metric --cpu-dimm-pow-consumption 0"] + + +def test_get_cpu_dimm_thermal_sensor_parses_spaced_unit_temperature( + conn_mock, system_info, monkeypatch +): + """``"0.0 Degrees C"`` must parse into value=0.0/unit="Degrees C" (the + shared ValueUnit regex doesn't allow a space inside the unit, so this + field needs ``coerce_spaced_unit_value_unit_input``).""" + responses = { + "metric --cpu-dimm-thermal-sensor 0": _dimm_payload( + "dimm_thermal_sensor", + { + "dimm_thermal_sensor_value": 0, + "dimm_thermal_update_rate": "0 ms", + "dimm_thermal_dimm_addr": 0, + "dimm_thermal_temperature": "0.0 Degrees C", + }, + ), + "metric --cpu-dimm-thermal-sensor 1": _dimm_payload("dimm_thermal_sensor", "N/A"), + "metric --cpu-dimm-thermal-sensor 2": _dimm_payload("dimm_thermal_sensor", "N/A"), + } + c, _ = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_dimm_thermal_sensor() + + assert len(entries) == 1 + assert entries[0].dimm_thermal_temperature.value == 0.0 + assert entries[0].dimm_thermal_temperature.unit == "Degrees C" + assert entries[0].dimm_thermal_sensor_value == 0 + + +def test_get_cpu_dimm_temp_range_rate(conn_mock, system_info, monkeypatch): + """Plain-int fields (no units) parse directly; N/A collapses the group.""" + responses = { + "metric --cpu-dimm-temp-range-rate 0": _dimm_payload( + "dimm_temp_range_rate", {"dimm_temperature_range": 0, "dimm_refresh_rate": 0} + ), + "metric --cpu-dimm-temp-range-rate 1": _dimm_payload("dimm_temp_range_rate", "N/A"), + "metric --cpu-dimm-temp-range-rate 2": _dimm_payload("dimm_temp_range_rate", "N/A"), + } + c, _ = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_dimm_temp_range_rate() + + assert [e.dimm_addr for e in entries] == [0] + assert entries[0].dimm_temperature_range == 0 + assert entries[0].dimm_refresh_rate == 0 + + +def test_get_cpu_lclk_dpm_level_stops_after_na_grace(conn_mock, system_info, monkeypatch): + """NBIO 0-3 populated, stop after 2 consecutive N/A.""" + dpm_range = {"nbio_max_dpm_level": 1, "nbio_min_dpm_level": 0} + responses = { + f"metric --cpu-lclk-dpm-level {i}": { + "cpu_data": [{"cpu": 0, "socket_dpm": {"dpml_level_range": dpm_range}}] + } + for i in range(4) + } + responses["metric --cpu-lclk-dpm-level 4"] = { + "cpu_data": [{"cpu": 0, "socket_dpm": {"dpml_level_range": "N/A"}}] + } + responses["metric --cpu-lclk-dpm-level 5"] = { + "cpu_data": [{"cpu": 0, "socket_dpm": {"dpml_level_range": "N/A"}}] + } + c, calls = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_lclk_dpm_level() + + assert [e.nbio_id for e in entries] == [0, 1, 2, 3] + assert entries[0].nbio_max_dpm_level == 1 + assert entries[0].nbio_min_dpm_level == 0 + assert calls == [f"metric --cpu-lclk-dpm-level {i}" for i in range(6)] + + +def test_get_cpu_svi3_vr_controller_temp_probes_full_fixed_candidate_set( + conn_mock, system_info, monkeypatch +): + """SVI3's candidate set (TYPE 0, TYPE 1 x rail 0-4) is small and fully + documented by amd-smi --help, so every combination is always probed and + kept, including all-N/A.""" + na_group = {"RAIL_SELECTION": "N/A", "RAIL_INDEX": "N/A", "TEMPERATURE": "N/A"} + responses = { + "metric --cpu-svi3-vr-controller-temp 0": { + "cpu_data": [{"cpu": 0, "svi3_vr_controller_temp": na_group}] + }, + **{ + f"metric --cpu-svi3-vr-controller-temp 1 {rail}": { + "cpu_data": [{"cpu": 0, "svi3_vr_controller_temp": na_group}] + } + for rail in range(5) + }, + } + c, calls = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_svi3_vr_controller_temp() + + assert [(e.rail_type, e.rail_index) for e in entries] == [ + (0, None), + (1, 0), + (1, 1), + (1, 2), + (1, 3), + (1, 4), + ] + assert all(e.temperature is None for e in entries) + assert calls == [ + "metric --cpu-svi3-vr-controller-temp 0", + "metric --cpu-svi3-vr-controller-temp 1 0", + "metric --cpu-svi3-vr-controller-temp 1 1", + "metric --cpu-svi3-vr-controller-temp 1 2", + "metric --cpu-svi3-vr-controller-temp 1 3", + "metric --cpu-svi3-vr-controller-temp 1 4", + ] + + +def test_get_cpu_xgmi_bandwidth_keeps_na_links_and_probes_every_bw( + conn_mock, system_info, monkeypatch +): + """Every candidate link is kept (N/A included, no filtering) and + every XGMI_BW value is probed for every link (no shared-link-id + optimization across BW values).""" + + def payload(band_width): + return {"cpu_data": [{"cpu": 0, "xgmi_bandwidth": {"band_width": band_width}}]} + + real_links = {"G0": "37 Mbps", "G1": "37 Mbps", "G2": "92444 Mbps"} + responses = {} + for bw in (1, 2, 4): + for link_id in AmdSmiCollector.CPU_LINK_ID_CANDIDATES: + responses[f"metric --cpu-xgmi-bandwidth {bw} {link_id}"] = payload( + real_links.get(link_id, "N/A") + ) + c, calls = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_xgmi_bandwidth() + + assert len(entries) == 3 * len(AmdSmiCollector.CPU_LINK_ID_CANDIDATES) + assert len(calls) == len(entries) + by_bw_link = {(e.xgmi_bw, e.link_id): e.band_width for e in entries} + assert by_bw_link[(1, "G2")].value == 92444 + assert by_bw_link[(1, "G2")].unit == "Mbps" + assert by_bw_link[(1, "P2")] is None + assert by_bw_link[(4, "G7")] is None + + +def test_get_cpu_io_bandwidth_independent_from_xgmi_validity(conn_mock, system_info, monkeypatch): + """IO bandwidth's valid-link subset does not have to match xgmi-bandwidth's, + it must be probed and kept independently.""" + + def payload(band_width): + return {"cpu_data": [{"cpu": 0, "io_bandwidth": {"band_width": band_width}}]} + + real_links = {"P2": "0 Mbps", "G0": "0 Mbps", "G1": "1 Mbps", "G2": "0 Mbps"} + responses = { + f"metric --cpu-io-bandwidth 1 {link_id}": payload(real_links.get(link_id, "N/A")) + for link_id in AmdSmiCollector.CPU_LINK_ID_CANDIDATES + } + c, calls = make_cpu_arg_collector(conn_mock, system_info, monkeypatch, responses) + + entries = c.get_cpu_io_bandwidth() + + assert len(entries) == len(AmdSmiCollector.CPU_LINK_ID_CANDIDATES) + assert len(calls) == len(entries) + by_link = {e.link_id: e.band_width for e in entries} + assert by_link["P2"].value == 0 + assert by_link["G1"].value == 1 + assert by_link["G3"] is None diff --git a/test/unit/plugin/test_amdsmi_data.py b/test/unit/plugin/test_amdsmi_data.py index f6c4f750..4a2bf4fb 100644 --- a/test/unit/plugin/test_amdsmi_data.py +++ b/test/unit/plugin/test_amdsmi_data.py @@ -32,7 +32,12 @@ from nodescraper.plugins.inband.amdsmi.amdsmidata import ( AmdSmiDataModel, AmdSmiMetric, + Fabric, + FabricInfo, + MetricBaseBoardTemperature, MetricClockData, + MetricClockRails, + MetricGpuBoardTemperature, MetricPcie, MetricPower, StaticClockData, @@ -489,15 +494,18 @@ def test_metric_usage_string_na(): def test_metric_gpuboard_baseboard_optional(): - """Store gpuboard/baseboard sections from ROCm 7.1+ metric output.""" + """Store gpuboard/baseboard sections from ROCm 7.1+ metric output as typed models.""" metric = AmdSmiMetric.model_validate( dummy_metric_dict( gpuboard=DUMMY_METRIC_GPUBOARD, baseboard=DUMMY_METRIC_BASEBOARD, ) ) - assert metric.gpuboard == DUMMY_METRIC_GPUBOARD - assert metric.baseboard == DUMMY_METRIC_BASEBOARD + assert metric.gpuboard.temperature.NODE_RETIMER_X.value == 43 + assert metric.gpuboard.temperature.NODE_OAM_X_IBC.value == 53 + assert metric.gpuboard.temperature.VDDCR_VDD0 is None + assert metric.baseboard.temperature.UBB_FRONT.value == 55 + assert metric.baseboard.temperature.UBB_FPGA.value == 78 def test_metric_gpuboard_gpu_board_alias(): @@ -510,8 +518,62 @@ def test_metric_gpuboard_gpu_board_alias(): } ) ) - assert metric.gpuboard == DUMMY_METRIC_GPUBOARD - assert metric.baseboard == DUMMY_METRIC_BASEBOARD + assert metric.gpuboard.temperature.NODE_RETIMER_X.value == 43 + assert metric.baseboard.temperature.UBB_FRONT.value == 55 + + +def test_metric_gpuboard_all_rails_present(): + """Every known gpuboard rail is parsed when present, numeric or N/A.""" + sample_payload = { + name: (0 if i % 2 == 0 else "N/A") + for i, name in enumerate(MetricGpuBoardTemperature.RAIL_NAMES) + } + board = MetricGpuBoardTemperature.model_validate(sample_payload) + for i, name in enumerate(MetricGpuBoardTemperature.RAIL_NAMES): + rail = getattr(board, name) + if i % 2 == 0: + assert rail is not None and rail.value == 0 + else: + assert rail is None + + +def test_metric_gpuboard_rail_renamed_or_dropped_is_none_not_absorbed(): + """A rail missing/renamed from the payload surfaces as None on its named field.""" + board = MetricGpuBoardTemperature.model_validate({"NODE_RETIMER_X": {"value": 5, "unit": "C"}}) + assert board.NODE_RETIMER_X.value == 5 + assert board.VDDCR_VDD0 is None + assert board.VDD_085_HBM is None + + +def test_metric_gpuboard_unknown_rail_tolerated(): + """An unrecognized rail (future FW addition) does not fail validation.""" + board = MetricGpuBoardTemperature.model_validate( + {"NODE_RETIMER_X": {"value": 1, "unit": "C"}, "SOME_NEW_RAIL": {"value": 2, "unit": "C"}} + ) + assert board.NODE_RETIMER_X.value == 1 + + +def test_metric_baseboard_rail_na_and_zero(): + """baseboard rails independently accept N/A and 0 without erroring.""" + board = MetricBaseBoardTemperature.model_validate({"UBB_FRONT": "N/A", "UBB_FPGA": 0}) + assert board.UBB_FRONT is None + assert board.UBB_FPGA.value == 0 + + +def test_metric_gpuboard_rail_case_insensitive_match(): + """Current amd-smi builds emit lowercase rail names (e.g. node_retimer_x).""" + board = MetricGpuBoardTemperature.model_validate({"node_retimer_x": {"value": 43, "unit": "C"}}) + assert board.NODE_RETIMER_X is not None + assert board.NODE_RETIMER_X.value == 43 + assert board.model_extra == {} + + +def test_metric_baseboard_rail_case_insensitive_match(): + """Current amd-smi builds emit lowercase rail names (e.g. ubb_front).""" + board = MetricBaseBoardTemperature.model_validate({"ubb_front": {"value": 55, "unit": "C"}}) + assert board.UBB_FRONT is not None + assert board.UBB_FRONT.value == 55 + assert board.model_extra == {} def test_metric_clock_per_aid_na_maps(): @@ -542,3 +604,180 @@ def test_metric_clock_per_aid_na_maps(): assert metric.clock["uclk_aid"]["AID_0"] == "N/A" assert isinstance(metric.clock["GFX_0"], MetricClockData) assert metric.pcie.lc_perf_other_end_recovery_count == 0 + + assert metric.clock_rails.UCLK_AID == {"AID_0": None, "AID_1": None} + assert metric.clock_rails.SOCCLKS_MID == {"MID_0": None, "MID_1": None} + + +def test_metric_clock_rails_enumerates_known_rails_and_catches_drop(): + """clock_rails exposes every known rail by name; a dropped one reads None.""" + metric = AmdSmiMetric.model_validate(dummy_metric_dict()) + assert isinstance(metric.clock_rails.GFX_0, MetricClockData) + assert metric.clock_rails.GFX_0.clk.value == 132 + assert metric.clock_rails.MEM_0 is None + assert metric.clock_rails.VCLK_0 is None + assert metric.clock_rails.FCLK_0 is None + assert metric.clock_rails.UCLK_AID is None + assert metric.clock_rails.SOCCLKS_MID is None + + +def test_metric_clock_rails_case_insensitive_match(): + """Current amd-smi builds emit lowercase rail names (e.g. gfx_0, mem_0).""" + rails = MetricClockRails.model_validate( + { + "gfx_0": {"clk": {"value": 2353, "unit": "MHz"}}, + "mem_0": {"clk": {"value": 1200, "unit": "MHz"}}, + "uclk_aid": {"AID_0": "N/A", "AID_1": {"value": 2000, "unit": "MHz"}}, + } + ) + assert rails.GFX_0 is not None + assert rails.GFX_0.clk.value == 2353 + assert rails.UCLK_AID["AID_0"] is None + assert rails.UCLK_AID["AID_1"].value == 2000 + assert rails.UCLK_AID["AID_1"].unit == "MHz" + assert rails.MEM_0 is not None + assert rails.MEM_0.clk.value == 1200 + assert rails.model_extra == {} + + +# FABRIC + + +def dummy_fabric_entry(gpu: int = 0, **overrides: Any) -> dict[str, Any]: + """Build one ``amd-smi fabric --json`` entry as emitted by the tool.""" + fabric_info: dict[str, Any] = { + "bdf": f"{gpu + 1:04d}:01:00.1", + "version": 4294967295, + "accelerator_id": 7 - gpu, + "fabric_type": "UALOE", + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": {"value": 0, "unit": "ns"}, + "ppod_id": "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb", + "ppod_size": 72, + "vpod_id": 0, + "vpod_size": 0, + "local_accelerators": "0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0", + "local_active_accelerators": [ + "0, 0, 0, 0, 0, 0, 0, 0", + "0, 0, 0, 0, 0, 0, 0, 0", + "0, 0, 0, 0, 0, 0, 0, 0", + "0, 0, 0, 0, 0, 0, 0, 0", + ], + "addr_mode": "UNKNOWN", + "accel_state": "UNKNOWN", + } + fabric_info.update(overrides) + return { + "gpu": gpu, + "fabric": { + "gpu": gpu, + "bdf": f"{gpu + 1:04d}:01:00.0", + "fabric_info": fabric_info, + "fabric_telemetry": "N/A", + }, + } + + +def test_fabric_model_from_amd_smi_output(): + """Fabric parses the nested ``fabric`` payload from amd-smi fabric --json.""" + fabric = Fabric.model_validate(dummy_fabric_entry(gpu=0)["fabric"]) + + assert fabric.gpu == 0 + assert fabric.bdf == "0001:01:00.0" + info = fabric.fabric_info + assert info is not None + assert info.bdf == "0001:01:00.1" + assert info.version == 4294967295 + assert info.accelerator_id == 7 + assert info.fabric_type == "UALOE" + assert info.ppod_id == "4c8fab1c-fb8b-42aa-92ae-ea17ca953bdb" + assert info.ppod_size == 72 + assert info.vpod_id == 0 + assert info.vpod_size == 0 + assert info.addr_mode == "UNKNOWN" + assert info.accel_state == "UNKNOWN" + assert len(info.local_active_accelerators) == 4 + + +def test_fabric_ignores_fabric_telemetry(): + """fabric_telemetry is not modelled and must not cause a validation error.""" + entry = dummy_fabric_entry(gpu=1)["fabric"] + entry["fabric_telemetry"] = {"huge": [1, 2, 3]} + + fabric = Fabric.model_validate(entry) + + assert fabric.gpu == 1 + assert not hasattr(fabric, "fabric_telemetry") + + +def test_fabric_info_value_unit_coercion(): + """bandwidth/latency accept dicts, bare numbers and number+unit strings.""" + info = FabricInfo.model_validate( + { + "bandwidth": {"value": 0, "unit": "Mb/s"}, + "latency": "150 ns", + } + ) + + assert info.bandwidth == ValueUnit(value=0, unit="Mb/s") + assert info.latency is not None + assert info.latency.value == 150 + assert info.latency.unit == "ns" + + +def test_fabric_na_fields_map_to_none(): + """'N/A' values from amd-smi are normalized to None.""" + fabric = Fabric.model_validate( + { + "gpu": 2, + "bdf": "N/A", + "fabric_info": { + "bdf": "N/A", + "fabric_type": "N/A", + "ppod_id": "N/A", + "ppod_size": "N/A", + "accel_state": "N/A", + "bandwidth": "N/A", + "latency": None, + }, + } + ) + + assert fabric.bdf is None + info = fabric.fabric_info + assert info is not None + assert info.bdf is None + assert info.fabric_type is None + assert info.ppod_id is None + assert info.ppod_size is None + assert info.accel_state is None + assert info.bandwidth is None + assert info.latency is None + + +def test_fabric_info_na_maps_to_none(): + """A whole fabric_info block reported as 'N/A' becomes None.""" + fabric = Fabric.model_validate({"gpu": 3, "bdf": "0004:01:00.0", "fabric_info": "N/A"}) + + assert fabric.fabric_info is None + + +def test_data_model_get_fabric(): + """AmdSmiDataModel.get_fabric returns the entry for the requested gpu.""" + data = AmdSmiDataModel( + fabric=[ + Fabric.model_validate(dummy_fabric_entry(gpu=0)["fabric"]), + Fabric.model_validate(dummy_fabric_entry(gpu=1)["fabric"]), + ] + ) + + fabric = data.get_fabric(1) + assert fabric is not None + assert fabric.gpu == 1 + assert fabric.bdf == "0002:01:00.0" + assert data.get_fabric(9) is None + + +def test_data_model_fabric_defaults_to_empty_list(): + """fabric defaults to an empty list when amd-smi fabric is unavailable.""" + assert AmdSmiDataModel().fabric == [] diff --git a/test/unit/plugin/test_amdsmi_flavor_collector.py b/test/unit/plugin/test_amdsmi_flavor_collector.py new file mode 100644 index 00000000..49c2013a --- /dev/null +++ b/test/unit/plugin/test_amdsmi_flavor_collector.py @@ -0,0 +1,213 @@ +from unittest.mock import MagicMock + +import pytest +from pydantic import BaseModel + +from nodescraper.enums import ExecutionStatus, OSFamily +from nodescraper.enums.systeminteraction import SystemInteractionLevel +from nodescraper.plugins.inband.amdsmi.amdsmi_flavor_collector import ( + AmdSmiFlavorCollector, +) +from nodescraper.plugins.inband.amdsmi.amdsmidata_host import HostDriverAmdSmiVersion + + +class _Tiny(BaseModel): + gpu: int + + +@pytest.fixture +def collector(system_info, conn_mock): + return AmdSmiFlavorCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + + +def _cmd(exit_code=0, stdout="", stderr=""): + return MagicMock(exit_code=exit_code, stdout=stdout, stderr=stderr, command="x") + + +# ---- driver-flavor detection -------------------------------------------------- + + +def test_loaded_driver_host_linux(collector): + """gim in lsmod -> host driver flavor.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="gim 12345 0\namdxcp 1 0")) + assert collector._get_loaded_gpu_driver() == "gim" + assert collector.is_host_driver is True + assert collector.is_guest_driver is False + + +def test_loaded_driver_guest_linux(collector): + """amdgpu + hypervisor flag -> guest-VF flavor.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(side_effect=[_cmd(stdout="amdgpu 999 0"), _cmd(stdout="1")]) + assert collector.is_host_driver is False + assert collector.is_guest_driver is True + + +def test_loaded_driver_baremetal_linux(collector): + """amdgpu with no hypervisor flag -> bare-metal (neither host nor guest).""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(side_effect=[_cmd(stdout="amdgpu 999 0"), _cmd(stdout="0")]) + assert collector.is_host_driver is False + assert collector.is_guest_driver is False + + +def test_loaded_driver_esxi_host(collector): + """ESXi reads modules via vmkload_mod; amdgpuv -> host flavor, never virtualized.""" + collector.system_info.os_family = OSFamily.ESXI + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="vmklinux\namdgpuv\nvmkapi")) + assert collector._get_loaded_gpu_driver() == "amdgpuv" + assert collector.is_host_driver is True + assert collector._is_virtualized() is False + + +def test_loaded_driver_none_means_not_loaded(collector): + """No known AMD GPU module -> None -> driver not loaded.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="ext4\nnvme")) + assert collector._get_loaded_gpu_driver() is None + assert collector._check_gpu_driver_loaded() is False + + +def test_loaded_driver_cached(collector): + """The driver lookup is cached after the first probe.""" + collector.system_info.os_family = OSFamily.LINUX + run = MagicMock(return_value=_cmd(stdout="gim 1 0")) + collector._run_sut_cmd = run + collector._get_loaded_gpu_driver() + collector._get_loaded_gpu_driver() + assert run.call_count == 1 + + +# ---- _build_flavor_model ------------------------------------------------------ + + +def test_build_flavor_model_dict(collector): + out = collector._build_flavor_model(_Tiny, {"gpu": 3}) + assert isinstance(out, _Tiny) and out.gpu == 3 + + +def test_build_flavor_model_none_logs_error(collector): + assert collector._build_flavor_model(_Tiny, None) is None + assert any(e.priority.name == "ERROR" for e in collector.result.events) + + +def test_build_flavor_model_keep_key_drops_missing(collector): + """Rows without keep_key are dropped silently (non-GPU enumeration rows).""" + out = collector._build_flavor_model( + _Tiny, [{"gpu": 0}, {"other": 1}, {"gpu": 2}], keep_key="gpu" + ) + assert [m.gpu for m in out] == [0, 2] + + +def test_build_flavor_model_skips_unparseable_row(collector): + """A present-but-invalid row is skipped and a warning is logged; valid rows kept.""" + out = collector._build_flavor_model(_Tiny, [{"gpu": 0}, {"gpu": "NA"}]) + assert [m.gpu for m in out] == [0] + assert any(e.priority.name == "WARNING" for e in collector.result.events) + + +def test_build_flavor_model_return_first(collector): + out = collector._build_flavor_model(_Tiny, [{"gpu": 7}, {"gpu": 8}], return_first=True) + assert isinstance(out, _Tiny) and out.gpu == 7 + + +# ---- flavor-aware command dispatch ------------------------------------------- + + +def test_detect_commands_host_uses_help_flag(collector): + collector._loaded_gpu_driver_cache = "gim" # force host + collector._run_amd_smi = MagicMock(return_value=" static Foo\n metric Bar\n") + cmds = collector.detect_amdsmi_commands() + collector._run_amd_smi.assert_called_once_with("help") + assert {"static", "metric"} <= cmds + + +def test_detect_commands_guest_uses_dash_h(collector): + collector._loaded_gpu_driver_cache = "amdgpu" # not host + collector._run_amd_smi = MagicMock(return_value=" list Foo\n") + collector.detect_amdsmi_commands() + collector._run_amd_smi.assert_called_once_with("-h") + + +def test_get_static_host_vs_nonhost_subcommand(collector): + collector.amd_smi_commands = {"static"} + collector._run_amd_smi_dict = MagicMock(return_value=[{"gpu": 0}, {"no_gpu": 1}]) + collector._loaded_gpu_driver_cache = "gim" # host + assert collector.get_static() == [{"gpu": 0}] + assert collector._run_amd_smi_dict.call_args.args[0] == "static" + + collector._run_amd_smi_dict.reset_mock() + collector._loaded_gpu_driver_cache = "amdgpu" # non-host + collector.get_static() + assert collector._run_amd_smi_dict.call_args.args[0] == "static -g all" + + +def test_get_static_unsupported_returns_none(collector): + collector.amd_smi_commands = set() + assert collector.get_static() is None + + +def test_get_metric_unwraps_gpu_data(collector): + collector.amd_smi_commands = {"metric"} + collector._loaded_gpu_driver_cache = "gim" + collector._run_amd_smi_dict = MagicMock(return_value={"gpu_data": [{"gpu": 0}, {"x": 1}]}) + assert collector.get_metric() == [{"gpu": 0}] + + +def test_get_partition_host_parses_text_table(collector): + collector.amd_smi_commands = {"partition"} + collector._loaded_gpu_driver_cache = "gim" # host: no --json, parse text + table = "GPU memory accelerator_type accelerator_profile_index partition_id\n0 NPS1 SPX 0 0\n" + collector._run_amd_smi = MagicMock(return_value=table) + out = collector.get_partition() + assert out == { + "current_partition": [ + { + "gpu_id": 0, + "memory": "NPS1", + "accelerator_type": "SPX", + "accelerator_profile_index": "0", + "partition_id": "0", + } + ] + } + + +def test_get_xgmi_flags_host_vs_guest(collector): + collector.amd_smi_commands = {"xgmi"} + collector._loaded_gpu_driver_cache = "gim" # host + collector._run_amd_smi_dict = MagicMock(return_value=[]) + collector.get_xgmi_data_metric() + flags = [c.args[0] for c in collector._run_amd_smi_dict.call_args_list] + assert "xgmi --metric" in flags and "xgmi --link-status" in flags + + collector._run_amd_smi_dict.reset_mock() + collector._loaded_gpu_driver_cache = "amdgpu" # guest/bare-metal + collector.get_xgmi_data_metric() + flags = [c.args[0] for c in collector._run_amd_smi_dict.call_args_list] + assert "xgmi -m" in flags and "xgmi -l" in flags + + +def test_version_model_selected_by_flavor(collector): + collector._loaded_gpu_driver_cache = "gim" # host + collector._run_amd_smi_dict = MagicMock( + return_value={"version": "1.0", "amdsmi_library_version": "37.0.5"} + ) + out = collector._get_amdsmi_version() + assert isinstance(out, HostDriverAmdSmiVersion) + + +# ---- collect_data guard ------------------------------------------------------- + + +def test_collect_data_no_driver_not_ran(collector): + collector._check_amdsmi_installed = MagicMock(return_value=True) + collector._run_sut_cmd = MagicMock(return_value=_cmd(stdout="ext4")) # no driver + result, data = collector.collect_data() + assert result.status == ExecutionStatus.NOT_RAN + assert data is None diff --git a/test/unit/plugin/test_analyzer_args_build_from_model.py b/test/unit/plugin/test_analyzer_args_build_from_model.py index e6eb7485..5702a481 100644 --- a/test/unit/plugin/test_analyzer_args_build_from_model.py +++ b/test/unit/plugin/test_analyzer_args_build_from_model.py @@ -185,11 +185,10 @@ def test_sysctl_analyzer_args_build_from_model(): def test_process_analyzer_args_build_from_model(): """Test ProcessAnalyzerArgs.build_from_model includes all fields""" - datamodel = ProcessDataModel(kfd_process=5, cpu_usage=15.5) + datamodel = ProcessDataModel(cpu_usage=15.5) args = ProcessAnalyzerArgs.build_from_model(datamodel) assert isinstance(args, ProcessAnalyzerArgs) - assert args.max_kfd_processes == 5 assert args.max_cpu_usage == 15.5 diff --git a/test/unit/plugin/test_bios_collector.py b/test/unit/plugin/test_bios_collector.py index 48dda07a..a2455e1c 100644 --- a/test/unit/plugin/test_bios_collector.py +++ b/test/unit/plugin/test_bios_collector.py @@ -78,6 +78,22 @@ def test_task_body_linux(system_info, bios_collector): assert data == exp_data +def test_task_body_esxi(system_info, bios_collector): + """ESXi: BIOS version is parsed from the smbiosDump 'Version:' line.""" + system_info.os_family = OSFamily.ESXI + + bios_collector._run_sut_cmd = MagicMock( + return_value=MagicMock( + exit_code=0, + stdout=' Version: "1.8"', + ) + ) + + res, data = bios_collector.collect_data() + assert res.status == ExecutionStatus.OK + assert data == BiosDataModel(bios_version="1.8") + + def test_task_body_error(system_info, bios_collector): """Test the _task_body method when an error occurs.""" system_info.os_family = OSFamily.LINUX diff --git a/test/unit/plugin/test_device_enumeration_collector.py b/test/unit/plugin/test_device_enumeration_collector.py index 50335f1f..3577e402 100644 --- a/test/unit/plugin/test_device_enumeration_collector.py +++ b/test/unit/plugin/test_device_enumeration_collector.py @@ -27,9 +27,13 @@ import pytest +from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.device_enumeration.collector_args import ( + DeviceEnumerationCollectorArgs, +) from nodescraper.plugins.inband.device_enumeration.device_enumeration_collector import ( DeviceEnumerationCollector, ) @@ -170,3 +174,101 @@ def test_collect_error(system_info, device_enumeration_collector): result, data = device_enumeration_collector.collect_data() assert result.status == ExecutionStatus.EXECUTION_FAILURE assert data is None + + +def test_collect_esxi(system_info, device_enumeration_collector): + """ESXi counts CPUs via esxcli and GPUs/VFs by device ID from collector args.""" + system_info.os_family = OSFamily.ESXI + + device_enumeration_collector._run_sut_cmd = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="2\n", stderr="", command="cpu"), + MagicMock(exit_code=0, stdout="8\n", stderr="", command="gpu"), + MagicMock(exit_code=0, stdout="0\n", stderr="", command="vf"), + ] + ) + + args = DeviceEnumerationCollectorArgs(devid_ep=0x75A3, devid_ep_vf=0x75B3) + result, data = device_enumeration_collector.collect_data(args) + + assert result.status == ExecutionStatus.OK + assert data == DeviceEnumerationDataModel(cpu_count=2, gpu_count=8, vf_count=0) + # GPU count command must carry the PF device id (hex, unpadded) into the grep. + gpu_cmd = device_enumeration_collector._run_sut_cmd.call_args_list[1].args[0] + assert "0x0*75a3" in gpu_cmd + vf_cmd = device_enumeration_collector._run_sut_cmd.call_args_list[2].args[0] + assert "0x0*75b3" in vf_cmd + + +def test_collect_esxi_no_devid(system_info, device_enumeration_collector): + """Without a devid_ep the ESXi run warns and does not match GPUs by device id.""" + system_info.os_family = OSFamily.ESXI + + device_enumeration_collector._run_sut_cmd = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="2\n", stderr="", command="cpu"), + MagicMock(exit_code=0, stdout="0\n", stderr="", command="gpu"), + MagicMock(exit_code=0, stdout="0\n", stderr="", command="vf"), + ] + ) + + result, data = device_enumeration_collector.collect_data(DeviceEnumerationCollectorArgs()) + + assert result.status == ExecutionStatus.OK + assert data == DeviceEnumerationDataModel(cpu_count=2, gpu_count=0, vf_count=0) + assert any("devid_ep not set" in e.description for e in result.events) + # The unset device id must not be formatted as hex into the grep (no match). + gpu_cmd = device_enumeration_collector._run_sut_cmd.call_args_list[1].args[0] + assert "__unset__" in gpu_cmd + + +def test_collect_esxi_none_args_defaults(system_info, device_enumeration_collector): + """collect_data(None) on ESXi falls back to default args (devid_ep unset).""" + system_info.os_family = OSFamily.ESXI + device_enumeration_collector._run_sut_cmd = MagicMock( + side_effect=[ + MagicMock(exit_code=0, stdout="2", stderr="", command="cpu"), + MagicMock(exit_code=0, stdout="0", stderr="", command="gpu"), + MagicMock(exit_code=0, stdout="0", stderr="", command="vf"), + ] + ) + result, data = device_enumeration_collector.collect_data(None) + assert result.status == ExecutionStatus.OK + assert data.cpu_count == 2 + + +def test_esxi_device_count_formats_device_id(device_enumeration_collector): + """_esxi_device_count renders the id as bare lowercase hex; None -> unmatched sentinel.""" + device_enumeration_collector._run_sut_cmd = MagicMock( + return_value=MagicMock(exit_code=0, stdout="1", stderr="", command="x") + ) + device_enumeration_collector._esxi_device_count(0x744C) + assert "0x0*744c" in device_enumeration_collector._run_sut_cmd.call_args.args[0] + + device_enumeration_collector._run_sut_cmd.reset_mock() + device_enumeration_collector._esxi_device_count(None) + assert "__unset__" in device_enumeration_collector._run_sut_cmd.call_args.args[0] + + +def test_parse_count_valid(device_enumeration_collector): + """A clean numeric stdout parses to int (with surrounding whitespace stripped).""" + res = MagicMock(exit_code=0, stdout=" 8 \n", stderr="", command="c") + assert device_enumeration_collector._parse_count(res, "count") == 8 + + +def test_parse_count_non_numeric(device_enumeration_collector): + """Non-numeric stdout returns None and warns instead of raising ValueError.""" + res = MagicMock(exit_code=0, stdout="N/A", stderr="", command="c") + assert device_enumeration_collector._parse_count(res, "count") is None + assert any( + e.priority == EventPriority.WARNING for e in device_enumeration_collector.result.events + ) + + +def test_parse_count_bad_exit(device_enumeration_collector): + """A non-zero exit returns None and warns.""" + res = MagicMock(exit_code=1, stdout="", stderr="boom", command="c") + assert device_enumeration_collector._parse_count(res, "count") is None + assert any( + e.priority == EventPriority.WARNING for e in device_enumeration_collector.result.events + ) diff --git a/test/unit/plugin/test_dimms_collector.py b/test/unit/plugin/test_dimms_collector.py index eeaa15ff..7e9f6073 100644 --- a/test/unit/plugin/test_dimms_collector.py +++ b/test/unit/plugin/test_dimms_collector.py @@ -23,20 +23,40 @@ # SOFTWARE. # ############################################################################### +from pathlib import Path from unittest.mock import MagicMock import pytest +from nodescraper.connection.inband.inband import CommandArtifact from nodescraper.enums.eventcategory import EventCategory from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.dimm.collector_args import DimmCollectorArgs from nodescraper.plugins.inband.dimm.dimm_collector import DimmCollector from nodescraper.plugins.inband.dimm.dimmdata import DimmDataModel +FIXTURES = Path(__file__).parent / "fixtures" + +# `dmidecode -q --type 17` from a dual socket host, 24 slots of Samsung DDR5. +# Being quiet, it carries no handle lines naming the DMI type, so a record is +# only identifiable by its section title and closed by a blank line. +QUIET_DUMP = (FIXTURES / "dmidecode_quiet.txt").read_text() + +# An unfiltered dump, where every record opens with a handle line naming its +# type. It covers the awkward records a real fleet throws up: a type 16 array +# whose size is not a module at all, a module sized in megabytes that nests a +# further volatile size, an empty slot, and placeholder manufacturer text. +FULL_DUMP = (FIXTURES / "dmidecode_full.txt").read_text() + +# `wmic memorychip get /format:csv` output from a host with soldered LPDDR5. +WMIC_DUMP = (FIXTURES / "dmidecode_wmic.txt").read_text() + @pytest.fixture def collector(system_info, conn_mock): + system_info.os_family = OSFamily.LINUX return DimmCollector( system_info=system_info, system_interaction_level=SystemInteractionLevel.PASSIVE, @@ -44,69 +64,264 @@ def collector(system_info, conn_mock): ) -def test_run_windows(system_info, conn_mock): +@pytest.fixture +def windows_collector(system_info, conn_mock): system_info.os_family = OSFamily.WINDOWS - collector = DimmCollector( + return DimmCollector( system_info=system_info, system_interaction_level=SystemInteractionLevel.PASSIVE, connection=conn_mock, ) + +def cmd_result(exit_code=0, stdout="", stderr="", command="dmidecode"): + """Build the CommandArtifact that _run_sut_cmd returns.""" + return CommandArtifact(exit_code=exit_code, stdout=stdout, stderr=stderr, command=command) + + +def descriptions(result): + """List the description of every event logged against a result.""" + return [event.description for event in result.events] + + +def linux_run(collector, targeted=QUIET_DUMP, full=FULL_DUMP): + """Collect with the full dump and the targeted dump each stubbed out.""" collector._run_sut_cmd = MagicMock( - return_value=MagicMock( - exit_code=0, - stdout="8589934592\n8589934592\n17179869184\n", - ) + side_effect=[cmd_result(stdout=full), cmd_result(stdout=targeted)] ) + return collector.collect_data() + + +def test_run_linux(collector): + result, data = linux_run(collector) - result, data = collector.collect_data() - assert data == DimmDataModel(dimms="32768.00GB @ 2 x 8192.00GB 1 x 16384.00GB ") assert result.status == ExecutionStatus.OK + assert data.dimm_count == 24 + assert ( + data.total_size_bytes == 24 * 96 * 1024**3 + ), f"Expected {24 * 96 * 1024**3} but got {data.total_size_bytes}" + assert data.population == {"96.0 GB": 24} + assert str(data) == "2.25 TB @ 24 x 96.0 GB" + assert result.message == "DIMM: 2.25 TB @ 24 x 96.0 GB" -def test_run_linux(collector, system_info): - system_info.os_family = OSFamily.LINUX +def test_run_linux_decodes_every_field(collector): + _, data = linux_run(collector) + dimm = data.dimms[0] + + assert dimm.size_bytes == 96 * 1024**3 + assert dimm.size == "96 GB" + assert dimm.locator == "CPU0_A" + assert dimm.bank_locator == "_Node0_Channel0_Dimm0" + assert dimm.manufacturer == "Samsung" + # dmidecode pads the part number out to its full field width. + assert dimm.part_number == "M321RYGA0PB0-CWMXJ" + assert dimm.serial_number == "2432-50F82DCB" + assert dimm.memory_type == "DDR5" + assert dimm.form_factor == "DIMM" + assert dimm.speed_mts == 5600 + assert dimm.configured_speed_mts == 5600 + assert dimm.rank == 2 + assert dimm.data_width_bits == 64 + assert dimm.total_width_bits == 80 + assert str(dimm) == "CPU0_A: 96.0 GB DDR5 5600MT/s Samsung" + + +def test_run_linux_covers_every_slot(collector): + _, data = linux_run(collector) + + assert [dimm.locator for dimm in data.dimms] == [ + f"CPU{cpu}_{channel}" for cpu in (0, 1) for channel in "ABCDEFGHIJKL" + ] + +def test_run_linux_keeps_full_dump_as_artifact(collector): + result, _ = linux_run(collector) + + artifacts = [a for a in result.artifacts if a.filename == "dmidecode.txt"] + assert len(artifacts) == 1 + assert artifacts[0].contents == FULL_DUMP + + +def test_run_linux_parses_unfiltered_dump(collector): + # The targeted dump fails, so the records come from the full dump instead. collector._run_sut_cmd = MagicMock( side_effect=[ - MagicMock( - exit_code=0, - stdout="Full dmidecode output...", - ), - MagicMock( - exit_code=0, - stdout="Size: 64 GB\nSize: 64 GB\nSize: 128 GB\n", - ), + cmd_result(stdout=FULL_DUMP), + cmd_result(exit_code=1, stderr="invalid option -- 'q'"), ] ) - result, data = collector.collect_data() + _, data = collector.collect_data() - assert result.status == ExecutionStatus.OK - assert data == DimmDataModel(dimms="256GB @ 2 x 64GB 1 x 128GB") + assert data.dimm_count == 3 + assert [dimm.locator for dimm in data.dimms] == ["DIMM_A1", "DIMM_A2", "DIMM_B1"] + assert str(data) == "80.0 GB @ 1 x 16.0 GB, 1 x 64.0 GB, 1 x Unknown Size" -def test_run_linux_error(collector, system_info): - system_info.os_family = OSFamily.LINUX +def test_run_linux_drops_placeholder_text(collector): + collector._run_sut_cmd = MagicMock( + side_effect=[ + cmd_result(stdout=FULL_DUMP), + cmd_result(exit_code=1, stderr="invalid option -- 'q'"), + ] + ) + + _, data = collector.collect_data() + dimm = data.dimms[1] + + assert dimm.manufacturer is None + # A serial of all zeroes is a real value rather than a placeholder. + assert dimm.serial_number is None + assert str(dimm) == "DIMM_A2: Unknown Size" + + +def test_run_linux_warns_when_full_dump_fails(collector): + collector._run_sut_cmd = MagicMock( + side_effect=[ + cmd_result(exit_code=1, stderr="command not found"), + cmd_result(stdout=QUIET_DUMP), + ] + ) + + result, data = collector.collect_data() + + # The inventory still comes back, the missing artifact is only a warning. + assert data.dimm_count == 24 + assert "Could not collect full dmidecode output" in descriptions(result) + assert not [a for a in result.artifacts if a.filename == "dmidecode.txt"] + +def test_run_linux_retries_without_sudo(collector): collector._run_sut_cmd = MagicMock( side_effect=[ - MagicMock( - exit_code=1, - stderr="Error occurred", - command="dmidecode", - ), - MagicMock( - exit_code=1, - stderr="Error occurred", - command="sh -c 'dmidecode -t 17 | ...'", - ), + cmd_result(exit_code=127, stderr="sudo: command not found"), + cmd_result(stdout=FULL_DUMP), + cmd_result(exit_code=127, stderr="sudo: command not found"), + cmd_result(stdout=QUIET_DUMP), ] ) result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data.dimm_count == 24 + assert [call.kwargs["sudo"] for call in collector._run_sut_cmd.call_args_list] == [ + True, + False, + True, + False, + ] + + +def test_run_linux_error(collector): + collector._run_sut_cmd = MagicMock( + return_value=cmd_result(exit_code=1, stderr="Error occurred") + ) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.ERROR + assert data is None + assert result.message.startswith("DIMM info not found") + assert descriptions(result) == [ + "Error checking dimms", + "Could not collect full dmidecode output", + "Error checking dimms", + "DIMM info not found", + ] + assert result.events[0].category == EventCategory.OS.value + + +def test_run_linux_no_modules_installed(collector): + empty = "Memory Device\n\tSize: No Module Installed\n\tLocator: DIMM_A1\n" + + result, data = linux_run(collector, targeted=empty) + + assert result.status == ExecutionStatus.OK + + +def test_skip_sudo(collector): + collector._run_sut_cmd = MagicMock() + + result, data = collector.collect_data(DimmCollectorArgs(skip_sudo=True)) + + assert result.status == ExecutionStatus.NOT_RAN + assert data is None + collector._run_sut_cmd.assert_not_called() + + +def test_run_windows(windows_collector): + windows_collector._run_sut_cmd = MagicMock(return_value=cmd_result(stdout=WMIC_DUMP)) + + result, data = windows_collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data.dimm_count == 4 + assert data.total_size_bytes == 4 * 8 * 1024**3 + assert data.population == {"8.0 GB": 4} + assert str(data) == "32.0 GB @ 4 x 8.0 GB" + windows_collector._run_sut_cmd.assert_called_once_with(DimmCollector.CMD_WINDOWS) + + +def test_run_windows_decodes_every_field(windows_collector): + windows_collector._run_sut_cmd = MagicMock(return_value=cmd_result(stdout=WMIC_DUMP)) + + _, data = windows_collector.collect_data() + dimm = data.dimms[0] + + # Capacity arrives as a bare byte count rather than a "8 GB" style size. + assert dimm.size_bytes == 8 * 1024**3 + assert dimm.size == "8589934592" + assert dimm.locator == "DIMM 0" + assert dimm.bank_locator == "P0 CHANNEL A" + assert dimm.manufacturer == "Micron Technology" + assert dimm.part_number == "MT62F2G32D8DR-031 WT" + assert dimm.serial_number == "00000000" + # SMBIOSMemoryType 35 and FormFactor 1 are raw enum codes on Windows, and + # the two enumerations do not share a numbering. + assert dimm.memory_type == "LPDDR5" + assert dimm.form_factor == "Other" + assert dimm.speed_mts == 6400 + assert dimm.configured_speed_mts == 6400 + assert dimm.rank is None + assert dimm.data_width_bits == 32 + assert dimm.total_width_bits == 32 + assert str(dimm) == "DIMM 0: 8.0 GB LPDDR5 6400MT/s Micron Technology" + + +def test_run_windows_covers_every_slot(windows_collector): + windows_collector._run_sut_cmd = MagicMock(return_value=cmd_result(stdout=WMIC_DUMP)) + + result, data = windows_collector.collect_data() + + # Every module reports the same device locator, so only the bank tells the + # four soldered channels apart. + assert [dimm.bank_locator for dimm in data.dimms] == [ + "P0 CHANNEL A", + "P0 CHANNEL B", + "P0 CHANNEL C", + "P0 CHANNEL D", + ] + assert len([a for a in result.artifacts if a.filename == "memorychip.csv"]) == 1 + + +def test_run_windows_error(windows_collector): + windows_collector._run_sut_cmd = MagicMock( + return_value=cmd_result(exit_code=1, stderr="Invalid query", command="wmic") + ) + + result, data = windows_collector.collect_data() + assert result.status == ExecutionStatus.ERROR assert data is None - assert result.events[1].category == EventCategory.OS.value - assert result.events[1].description == "Error checking dimms" + assert descriptions(result) == ["Error checking dimms", "DIMM info not found"] + + +def test_empty_model(): + data = DimmDataModel() + + assert data.dimm_count == 0 + assert data.total_size_bytes == 0 + assert data.population == {} + assert str(data) == "0GB" diff --git a/test/unit/plugin/test_dmesg_analyzer.py b/test/unit/plugin/test_dmesg_analyzer.py index 784b5453..d0baa4e6 100644 --- a/test/unit/plugin/test_dmesg_analyzer.py +++ b/test/unit/plugin/test_dmesg_analyzer.py @@ -31,6 +31,7 @@ from nodescraper.enums.eventcategory import EventCategory from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus +from nodescraper.models.systeminfo import OSFamily from nodescraper.plugins.inband.dmesg.analyzer_args import DmesgAnalyzerArgs from nodescraper.plugins.inband.dmesg.dmesg_analyzer import DmesgAnalyzer from nodescraper.plugins.inband.dmesg.dmesgdata import DmesgData @@ -1435,3 +1436,75 @@ def test_mce_match_content_is_single_status_line(system_info): assert "CPU:29" in match_content assert "CPU:8" not in match_content assert "\n" not in match_content + + +# --- ESXi ------------------------------------------------------------------ + + +def test_esxi_analyzer_uses_esxi_timestamp_pattern(system_info): + """On ESXi the analyzer swaps in the vmkernel.log (dot-ms/Z) timestamp pattern.""" + system_info.os_family = OSFamily.ESXI + analyzer = DmesgAnalyzer(system_info=system_info) + assert analyzer._is_esxi() is True + assert analyzer.TIMESTAMP_PATTERN is DmesgAnalyzer.ESXI_TIMESTAMP_PATTERN + + system_info.os_family = OSFamily.LINUX + linux_analyzer = DmesgAnalyzer(system_info=system_info) + assert linux_analyzer._is_esxi() is False + assert linux_analyzer.TIMESTAMP_PATTERN is not DmesgAnalyzer.ESXI_TIMESTAMP_PATTERN + + +def test_filter_dmesg_handles_esxi_and_linux_timestamps(): + """filter_dmesg accepts both ESXi dot-ms/Z and Linux comma-form timestamps.""" + esxi_log = ( + "2026-08-20T08:00:00.100Z -INFO vmkernel - line A\n" + "2026-08-20T09:00:00.100Z -INFO vmkernel - line B\n" + "2026-08-20T10:00:00.100Z -INFO vmkernel - line C\n" + ) + start = datetime.datetime.fromisoformat("2026-08-20T08:30:00+00:00") + end = datetime.datetime.fromisoformat("2026-08-20T09:30:00+00:00") + filtered = DmesgAnalyzer.filter_dmesg(esxi_log, start, end) + assert filtered.strip() == "2026-08-20T09:00:00.100Z -INFO vmkernel - line B" + + # Linux comma-form still filters (no regression). + linux_log = "2024-10-01T07:00:00,000000-05:00 log1\n" "2024-10-01T09:00:00,000000-05:00 log2\n" + l_start = datetime.datetime.fromisoformat("2024-10-01T08:00:00-05:00") + assert "log2" in DmesgAnalyzer.filter_dmesg(linux_log, l_start) + assert "log1" not in DmesgAnalyzer.filter_dmesg(linux_log, l_start) + + +def test_esxi_ras_regex_phrasing(system_info): + """ESXi mxGPU RAS phrasing ('... detected in Block.') is flagged as RAS.""" + system_info.os_family = OSFamily.ESXI + analyzer = DmesgAnalyzer(system_info=system_info) + data = DmesgData( + dmesg_content=( + "2026-08-20T09:35:58.380Z -ALERT vmkernel - " + "3 new uncorrectable hardware errors detected in MMHUB Block.\n" + "2026-08-20T09:35:59.000Z -ALERT vmkernel - GPU detected ECC Fatal Error.\n" + ), + skip_log_file=True, + ) + res = analyzer.analyze_data(data) + by_desc = {e.description: e for e in res.events} + assert "RAS Uncorrectable Error" in by_desc + assert "RAS ECC Fatal Error" in by_desc + assert by_desc["RAS Uncorrectable Error"].category == EventCategory.RAS.value + + +def test_esxi_unknown_error_uses_driver_body_severity(system_info): + """On ESXi the unknown-error signal is the driver-body severity (gim/amdgpuv), + not the unreliable vmkernel -ALERT/-INFO token.""" + system_info.os_family = OSFamily.ESXI + analyzer = DmesgAnalyzer(system_info=system_info) + data = DmesgData( + dmesg_content=( + "2026-08-20T09:35:58.380Z -INFO vmkernel - amdgpuv error [0:65:0]: unexpected thing\n" + "2026-08-20T09:35:59.000Z -ALERT vmkernel - benign boot notice, no driver marker\n" + ), + skip_log_file=True, + ) + res = analyzer.analyze_data(data) + unknown = [e for e in res.events if e.description == "Unknown dmesg error"] + assert len(unknown) == 1 + assert "unexpected thing" in str(unknown[0].data["match_content"]) diff --git a/test/unit/plugin/test_dmesg_collector.py b/test/unit/plugin/test_dmesg_collector.py index 4202c0f9..4dbccfbd 100644 --- a/test/unit/plugin/test_dmesg_collector.py +++ b/test/unit/plugin/test_dmesg_collector.py @@ -101,6 +101,32 @@ def test_dmesg_collection(system_info, conn_mock): assert data.dmesg_content == dmesg +def test_dmesg_collection_esxi(system_info, conn_mock): + """ESXi has no dmesg ring buffer; the kernel log is read from vmkernel.log.""" + system_info.os_family = OSFamily.ESXI + collector = DmesgCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.INTERACTIVE, + connection=conn_mock, + ) + + vmkernel = ( + "2026-08-20T09:35:58.380Z -INFO vmkernel - boot line\n" + "2026-08-20T09:36:00.000Z -WARNING vmkernel - a warning\n" + ) + conn_mock.run_command.return_value = CommandArtifact( + exit_code=0, + stdout=vmkernel, + stderr="", + command="cat /var/log/vmkernel.log", + ) + + res, data = collector.collect_data() + assert res.status == ExecutionStatus.OK + assert data is not None + assert data.dmesg_content == vmkernel + + def test_bad_exit_code(conn_mock, system_info): conn_mock.run_command.return_value = CommandArtifact( diff --git a/test/unit/plugin/test_fabrics_collector.py b/test/unit/plugin/test_fabrics_collector.py index add8bb50..9ecd42cb 100644 --- a/test/unit/plugin/test_fabrics_collector.py +++ b/test/unit/plugin/test_fabrics_collector.py @@ -174,6 +174,67 @@ def test_parse_ibv_devinfo_basic(collector): assert device.hw_ver == "0x0" +IBSTAT_MULTI_INDENTED_OUTPUT = """CA 'bnxt_re0' + CA type: AB1234 + Number of ports: 1 + Firmware version: 1.2.3 + Port 1: + State: Active + Physical state: LinkUp + Rate: 400 + Port GUID: 0x0000000000000001 + Link layer: Ethernet +CA 'bnxt_re1' + CA type: AB1234 + Number of ports: 1 + Firmware version: 1.2.3 + Port 1: + State: Down + Physical state: Disabled + Rate: 400 + Port GUID: 0x0000000000000002 + Link layer: Ethernet""" + +IBV_DEVINFO_MULTI_OUTPUT = """hca_id: bnxt_re0 + transport: InfiniBand (0) + fw_ver: 1.2.3 + port: 1 + state: PORT_ACTIVE (4) + active_mtu: 4096 (5) + link_layer: Ethernet + +hca_id: bnxt_re1 + transport: InfiniBand (0) + fw_ver: 1.2.3 + port: 1 + state: PORT_DOWN (1) + active_mtu: 1024 (3) + link_layer: Ethernet""" + + +def test_parse_ibstat_indented_ports_multiple_devices(collector): + """Indented 'Port N:' lines are parsed and each device keeps its own port.""" + devices = collector._parse_ibstat(IBSTAT_MULTI_INDENTED_OUTPUT) + + assert [device.ca_name for device in devices] == ["bnxt_re0", "bnxt_re1"] + assert devices[0].ports[1]["State"] == "Active" + assert devices[0].ports[1]["Rate"] == "400" + assert devices[0].ports[1]["Link layer"] == "Ethernet" + assert devices[0].ports[1]["Port GUID"] == "0x0000000000000001" + assert devices[1].ports[1]["State"] == "Down" + + +def test_parse_ibv_devinfo_multiple_devices_keep_ports(collector): + """Each ibv_devinfo device keeps its port when another hca_id follows.""" + devices = collector._parse_ibv_devinfo(IBV_DEVINFO_MULTI_OUTPUT) + + assert [device.device for device in devices] == ["bnxt_re0", "bnxt_re1"] + assert devices[0].ports[1]["state"] == "PORT_ACTIVE (4)" + assert devices[0].ports[1]["active_mtu"] == "4096 (5)" + assert devices[1].ports[1]["state"] == "PORT_DOWN (1)" + assert devices[1].ports[1]["link_layer"] == "Ethernet" + + def test_parse_ibv_devinfo_port(collector): """Test parsing ibv_devinfo port information""" devices = collector._parse_ibv_devinfo(IBV_DEVINFO_OUTPUT) @@ -366,3 +427,49 @@ def run_sut_cmd_side_effect(cmd, *args, **kwargs): assert result.status == ExecutionStatus.NOT_RAN assert data is None + + +def test_collect_data_logs_fabric_rate_and_mtu(collector): + """ibstat and ibv_devinfo events carry fabric, rate, state, and active MTU.""" + + def run_sut_cmd_side_effect(cmd, *args, **kwargs): + if cmd == "ibstat": + return MagicMock(exit_code=0, stdout=IBSTAT_OUTPUT, command=cmd) + if cmd == "ibv_devinfo": + return MagicMock(exit_code=0, stdout=IBV_DEVINFO_OUTPUT, command=cmd) + return MagicMock(exit_code=1, stdout="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + ibstat_event = next( + event + for event in result.events + if event.description == "Collected 1 IB devices from ibstat" + ) + assert ibstat_event.data["ports"] == [ + { + "device": "mlx5_0", + "port": 1, + "link_layer": "MockBand", + "rate": "200", + "state": "Active", + } + ] + ibv_event = next( + event + for event in result.events + if event.description == "Collected 1 IB devices from ibv_devinfo" + ) + assert ibv_event.data["devices"][0]["transport"] == "MockBand (0)" + assert ibv_event.data["devices"][0]["ports"] == [ + { + "port": 1, + "state": "PORT_ACTIVE (4)", + "link_layer": "MockBand", + "active_mtu": "4096 (5)", + } + ] diff --git a/test/unit/plugin/test_journal_collector.py b/test/unit/plugin/test_journal_collector.py index 4865baa2..0efe9b5d 100644 --- a/test/unit/plugin/test_journal_collector.py +++ b/test/unit/plugin/test_journal_collector.py @@ -24,7 +24,7 @@ # ############################################################################### import types -from datetime import datetime +from datetime import datetime, timezone from nodescraper.enums import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel @@ -75,6 +75,46 @@ def run_map(cmd, **kwargs): assert data.journal_log == '{"MESSAGE":"hello"}\n' +def test_journal_entry_timestamps_are_utc_aware(): + entry = JournalJsonEntry( + **{ + "_SOURCE_REALTIME_TIMESTAMP": "0", + "__REALTIME_TIMESTAMP": "2025-02-23T21:03:46.685975", + } + ) + + assert entry.SOURCE_REALTIME_TIMESTAMP == datetime(1970, 1, 1, tzinfo=timezone.utc) + assert entry.REALTIME_TIMESTAMP == datetime(2025, 2, 23, 21, 3, 46, 685975, tzinfo=timezone.utc) + + +def test_journal_entry_decodes_non_utf8_message_byte_array(): + entry = JournalJsonEntry(MESSAGE=[104, 195, 169, 255]) + + assert entry.MESSAGE == "hé�" + + +def test_journal_entry_joins_repeated_message_values(): + strings = JournalJsonEntry(MESSAGE=["first", "second"]) + mixed = JournalJsonEntry(MESSAGE=[[104, 105], "x"]) + with_null = JournalJsonEntry(MESSAGE=["first line", [104, 105], None]) + + assert strings.MESSAGE == "first second" + assert mixed.MESSAGE == "hi x" + assert with_null.MESSAGE == "first line hi " + + +def test_journal_entry_converts_offset_and_datetime_inputs_to_utc(): + offset = JournalJsonEntry(**{"__REALTIME_TIMESTAMP": "2025-02-23T21:03:46+05:00"}) + assert offset.REALTIME_TIMESTAMP == datetime(2025, 2, 23, 16, 3, 46, tzinfo=timezone.utc) + + naive = JournalJsonEntry(**{"__REALTIME_TIMESTAMP": datetime(2025, 2, 23, 21, 3, 46)}) + assert naive.REALTIME_TIMESTAMP == datetime(2025, 2, 23, 21, 3, 46, tzinfo=timezone.utc) + + aware = datetime(2025, 2, 23, 16, 3, 46, tzinfo=timezone.utc) + from_aware = JournalJsonEntry(**{"__REALTIME_TIMESTAMP": aware}) + assert from_aware.REALTIME_TIMESTAMP == aware + + def test_journal_filter(): """Test filtering journal entries based on timestamp range.""" journal_data = [ @@ -142,8 +182,8 @@ def test_journal_filter(): journal_content_json = [JournalJsonEntry(**entry) for entry in journal_data] - start_range = datetime.fromisoformat("2025-02-23T21:03:46.686006") - end_range = datetime.fromisoformat("2025-02-23T21:03:46.686020") + start_range = datetime.fromisoformat("2025-02-23T21:03:46.686006").replace(tzinfo=timezone.utc) + end_range = datetime.fromisoformat("2025-02-23T21:03:46.686020").replace(tzinfo=timezone.utc) filtered_journal = JournalAnalyzer.filter_journal(journal_content_json, start_range, end_range) assert filtered_journal == [JournalJsonEntry(**entry) for entry in journal_data[1:3]] @@ -154,6 +194,11 @@ def test_journal_filter(): filtered_journal = JournalAnalyzer.filter_journal(journal_content_json, None, end_range) assert filtered_journal == [JournalJsonEntry(**entry) for entry in journal_data[:3]] + naive_start = datetime.fromisoformat("2025-02-23T21:03:46.686006") + naive_end = datetime.fromisoformat("2025-02-23T21:03:46.686020") + filtered_journal = JournalAnalyzer.filter_journal(journal_content_json, naive_start, naive_end) + assert filtered_journal == [JournalJsonEntry(**entry) for entry in journal_data[1:3]] + def test_check_priority(system_info): """Test checking priority of journal entries.""" @@ -230,3 +275,18 @@ def test_check_priority(system_info): assert res.status == ExecutionStatus.ERROR assert len(res.events) == 3 + + +def test_grouped_missing_timestamp_uses_utc_epoch(system_info): + journal_data = JournalData( + journal_log="", + journal_content_json=[ + JournalJsonEntry(MESSAGE="no time", PRIORITY=3, REALTIME_TIMESTAMP=None) + ], + ) + analyzer = JournalAnalyzer(system_info=system_info) + res = analyzer.analyze_data(data=journal_data, args=JournalAnalyzerArgs(check_priority=5)) + + assert res.status == ExecutionStatus.ERROR + assert res.events[0].data["first_occurrence"] == datetime(1970, 1, 1, tzinfo=timezone.utc) + assert res.events[0].data["last_occurrence"] == datetime(1970, 1, 1, tzinfo=timezone.utc) diff --git a/test/unit/plugin/test_kernel_collector.py b/test/unit/plugin/test_kernel_collector.py index 3b370783..fa1420e9 100644 --- a/test/unit/plugin/test_kernel_collector.py +++ b/test/unit/plugin/test_kernel_collector.py @@ -88,6 +88,26 @@ def test_run_linux(collector, conn_mock): assert result.status == ExecutionStatus.OK +def test_run_esxi(collector, conn_mock): + """ESXi reuses the `uname -a` path (release in the same field); numa_balancing + has no ESXi equivalent and its command fails gracefully -> None.""" + collector.system_info.os_family = OSFamily.ESXI + uname = "VMkernel host 9.1.0 #1 SMP Release build-25166133 Jan 14 2026 x86_64" + conn_mock.run_command.side_effect = [ + CommandArtifact(exit_code=0, stdout=uname, stderr="", command="sh -c 'uname -a'"), + CommandArtifact( + exit_code=1, + stdout="", + stderr="not found", + command="sh -c 'cat /proc/sys/kernel/numa_balancing'", + ), + ] + + result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == KernelDataModel(kernel_info=uname, kernel_version="9.1.0", numa_balancing=None) + + def test_run_error(collector, conn_mock): collector.system_info.os_family = OSFamily.LINUX conn_mock.run_command.return_value = CommandArtifact( diff --git a/test/unit/plugin/test_memory_analyzer.py b/test/unit/plugin/test_memory_analyzer.py index 0c7c3f03..38297320 100644 --- a/test/unit/plugin/test_memory_analyzer.py +++ b/test/unit/plugin/test_memory_analyzer.py @@ -60,6 +60,85 @@ def test_config_provided(analyzer, model_obj): assert result.status == ExecutionStatus.OK +def test_minimum_free_memory_percent_and_ratio_pass(analyzer): + model = MemoryDataModel( + mem_total="100Gi", + mem_free="20Gi", + mem_available="20Gi", + ) + + result = analyzer.analyze_data( + model, + MemoryAnalyzerArgs( + minimum_free_memory_percent=10, + ratio=0.9, + memory_threshold="100Gi", + ), + ) + + assert result.status == ExecutionStatus.OK + assert "20.00%" in result.message + assert "ratio=0.90" in result.message + + +def test_minimum_free_memory_percent_passes_ratio_fails(analyzer): + model = MemoryDataModel( + mem_total="100Gi", + mem_free="20Gi", + mem_available="20Gi", + ) + + result = analyzer.analyze_data( + model, + MemoryAnalyzerArgs( + minimum_free_memory_percent=10, + ratio=0.5, + memory_threshold="100Gi", + ), + ) + + assert result.status == ExecutionStatus.ERROR + assert "Memory usage exceeded max allowed!" in result.message + + +def test_ratio_passes_minimum_free_memory_percent_fails(analyzer): + model = MemoryDataModel( + mem_total="100Gi", + mem_free="90Gi", + mem_available="90Gi", + ) + + result = analyzer.analyze_data( + model, + MemoryAnalyzerArgs( + minimum_free_memory_percent=95, + ratio=0.5, + memory_threshold="100Gi", + ), + ) + + assert result.status == ExecutionStatus.ERROR + assert "Minimum free memory percentage not met" in result.message + assert len(result.events) == 1 + + +def test_minimum_free_memory_percent_fails(analyzer): + model = MemoryDataModel( + mem_total="100Gi", + mem_free="5Gi", + mem_available="5Gi", + ) + + result = analyzer.analyze_data( + model, + MemoryAnalyzerArgs(minimum_free_memory_percent=10), + ) + + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert result.events[0].data["available_percent"] == pytest.approx(5.0) + + def test_linux_low_free_high_available_passes(analyzer): """MemFree alone looks tight; MemAvailable reflects reclaimable cache.""" model = MemoryDataModel( diff --git a/test/unit/plugin/test_mi4xx_plugin.py b/test/unit/plugin/test_mi4xx_plugin.py index 5b654e3f..5d5edd81 100644 --- a/test/unit/plugin/test_mi4xx_plugin.py +++ b/test/unit/plugin/test_mi4xx_plugin.py @@ -23,6 +23,7 @@ # SOFTWARE. # ############################################################################### +import json from types import SimpleNamespace from unittest.mock import patch @@ -87,6 +88,7 @@ def test_mi4xx_analyzer_analyze_data_wrapped_once(): class _FakeHub: name = "ExampleHub" + version = "0.2.0" def analyze(self, events, afid_sag_path): assert afid_sag_path @@ -183,7 +185,7 @@ def test_mi4xx_serviceability_plugin_wiring(): def test_mi4xx_analyzer_args_defaults_to_afse(): args = Mi4xxServiceabilityAnalyzerArgs() assert args.hub_entry_point == "afse" - assert args.hub_display_name == "AFSE" + assert args.hub_display_name == "Hub" assert args.resolved_hub_entry_point() == "afse" assert args.skip_hub is False @@ -318,7 +320,8 @@ def test_serviceability_hub_analyzer_runs_entry_point_hub(system_info, tmp_path) Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)), ) assert task.status == ExecutionStatus.OK - assert "afse" in task.message.lower() + assert task.message.startswith("Hub:") + assert DUMMY_HUB_VERSION_ENTRY in task.message def test_mi4xx_analyzer_appends_afid_sag_metadata_artifact(system_info, tmp_path): @@ -347,6 +350,38 @@ def test_mi4xx_analyzer_appends_afid_sag_metadata_artifact(system_info, tmp_path assert any(isinstance(artifact, AfidSagMetadataArtifact) for artifact in task.artifacts) +def test_mi4xx_analyzer_writes_top_action_summary_json(system_info, tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + data = ServiceabilityDataModel( + rf_events=[ + { + "Afid": DUMMY_AFID_A, + "ServiceableUnit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + } + ] + ) + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + analyzer = MI4XXAnalyzer(system_info=system_info) + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ): + task = analyzer.analyze_data( + data, + Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)), + ) + log_dir = tmp_path / "mi4xx_analyzer" + task.log_result(str(log_dir)) + + summary = json.loads((log_dir / "summary.json").read_text(encoding="utf-8")) + assert summary["system_name"] == system_info.name + assert summary["afid_event_count"] == 1 + assert [action["afid"] for action in summary["top_service_actions"]] == [DUMMY_AFID_A] + assert summary["top_service_actions"][0]["rank"] == 1 + + def test_mi4xx_plugin_analyzes_offline_data_without_collection(system_info, tmp_path): sag = tmp_path / "afid_sag.json" sag.write_text("{}", encoding="utf-8") @@ -377,6 +412,56 @@ def test_mi4xx_plugin_analyzes_offline_data_without_collection(system_info, tmp_ assert result.result_data.analysis_result.status == ExecutionStatus.OK +def test_mi4xx_plugin_missing_sag_stops_before_collection(system_info, tmp_path): + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + plugin = Mi4xxServiceabilityPlugin(system_info=system_info) + with ( + patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ), + patch.object(MI4XXCollector, "collect_data") as collect_data, + ): + result = plugin.run( + collection=True, + analysis=True, + analysis_args=Mi4xxServiceabilityAnalyzerArgs( + afid_sag_path=str(tmp_path / "missing_sag.json") + ), + ) + collect_data.assert_not_called() + assert result.status == ExecutionStatus.ERROR + assert "AFID_SAG preflight failed: AFID SAG file not found" in result.message + assert result.result_data.collection_result.status == ExecutionStatus.NOT_RAN + + +def test_mi4xx_plugin_sag_rejected_by_afse_stops_before_collection(system_info, tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + + class _RejectingHub(_FakeHub): + def validate(self, afid_sag_path): + return {"status": "error", "error": {"message": "schema mismatch"}} + + fake_ep = SimpleNamespace(name="afse", load=lambda: _RejectingHub) + plugin = Mi4xxServiceabilityPlugin(system_info=system_info) + with ( + patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ), + patch.object(MI4XXCollector, "collect_data") as collect_data, + ): + result = plugin.run( + collection=True, + analysis=True, + analysis_args=Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)), + ) + collect_data.assert_not_called() + assert result.status == ExecutionStatus.ERROR + assert "schema mismatch" in result.message + + def test_load_hub_from_entry_point_missing_raises(): with patch( "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", diff --git a/test/unit/plugin/test_network_analyzer.py b/test/unit/plugin/test_network_analyzer.py index 9796a173..ab1f66c8 100644 --- a/test/unit/plugin/test_network_analyzer.py +++ b/test/unit/plugin/test_network_analyzer.py @@ -26,6 +26,7 @@ import pytest from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.plugins.inband.network.analyzer_args import NetworkAnalyzerArgs from nodescraper.plugins.inband.network.ethtool_vendor import ( EthtoolStatistics, Thor2EthtoolStatistics, @@ -77,6 +78,89 @@ def test_empty_ethtool_info(network_analyzer): assert result.message == "No network devices found" +def test_ethtool_firmware_exact_match(network_analyzer): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="bnxt_en", + bus_info="0000:01:00.0", + firmware_version="238.1.169.0", + ) + } + ) + args = NetworkAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = network_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +@pytest.mark.parametrize( + "firmware_version", + ["1.117.5-a-77", "28.35.1012", "238.1.168.0"], +) +def test_ethtool_firmware_regex_matches_supported_formats(network_analyzer, firmware_version): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="bnxt_en", + firmware_version=firmware_version, + ) + } + ) + args = NetworkAnalyzerArgs( + expected_nic_firmware=(r"^[0-9]+(?:\.[0-9]+){2,}(?:-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$") + ) + + result = network_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_ethtool_firmware_mismatch_is_reported(network_analyzer): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="bnxt_en", + firmware_version="238.1.168.0", + ) + } + ) + args = NetworkAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = network_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.WARNING + assert any("firmware" in event.description.lower() for event in result.events) + + +def test_ethtool_unsupported_driver_is_non_blocking(network_analyzer): + model = NetworkDataModel( + ethtool_info={ + "eth0": EthtoolInfo( + interface="eth0", + raw_output="", + driver="i40e", + firmware_version="1.3534.0", + ) + } + ) + result = network_analyzer.analyze_data( + model, NetworkAnalyzerArgs(expected_nic_firmware="238.1.168.0") + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + def test_rdma_ethtool_vendor_error_only(network_analyzer): """Vendor ethtool: error-tier counter raises ERROR.""" stat = EthtoolStatistics( diff --git a/test/unit/plugin/test_network_collector.py b/test/unit/plugin/test_network_collector.py index 909e1319..c7de9a7a 100644 --- a/test/unit/plugin/test_network_collector.py +++ b/test/unit/plugin/test_network_collector.py @@ -27,6 +27,7 @@ import pytest +from nodescraper.enums import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily @@ -331,6 +332,38 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert "Network data collected successfully" in result.message +def test_collect_data_logs_interface_mtu_and_speed(collector, conn_mock): + """Observed interface state, MTU, and ethtool speed are logged. Loopback is omitted.""" + collector.system_info.os_family = OSFamily.LINUX + + def run_sut_cmd_side_effect(cmd, **kwargs): + if cmd == "ip addr show": + return MagicMock(exit_code=0, stdout=IP_ADDR_OUTPUT, command=cmd) + if cmd == "ethtool eth0": + return MagicMock(exit_code=0, stdout=ETHTOOL_OUTPUT, command=cmd) + return MagicMock(exit_code=1, stdout="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + interface_event = next( + event for event in result.events if event.description == "Collected 2 network interfaces" + ) + eth0 = next(iface for iface in interface_event.data["interfaces"] if iface["name"] == "eth0") + assert eth0["state"] == "UP" + assert eth0["mtu"] == 5678 + speed_event = next( + event + for event in result.events + if event.description == "Collected ethtool info for interface: eth0" + ) + assert speed_event.data["interface"] == "eth0" + assert speed_event.data["speed"] == "1000mockMb/s" + + def test_collect_data_addr_failure(collector, conn_mock): """Test collection when ip addr command fails""" collector.system_info.os_family = OSFamily.LINUX @@ -464,6 +497,22 @@ def test_parse_ethtool_basic(collector): assert ethtool_info.raw_output == ETHTOOL_OUTPUT +def test_parse_ethtool_driver_info_normalizes_package_firmware(collector): + """Keep the device firmware version when ethtool reports package metadata too.""" + info = EthtoolInfo(interface="ethmock123", raw_output="") + + collector._parse_ethtool_driver_info( + info, + "driver: bnxt_en\n" + "firmware-version: 238.1.168.0/pkg 238.1.169.0\n" + "bus-info: 0000:01:00.0\n", + ) + + assert info.driver == "bnxt_en" + assert info.firmware_version == "238.1.168.0" + assert info.bus_info == "0000:01:00.0" + + def test_parse_ethtool_supported_link_modes(collector): """Test parsing supported link modes from ethtool output""" ethtool_info = collector._parse_ethtool("ethmock123", ETHTOOL_OUTPUT) @@ -736,6 +785,60 @@ def run_sut_cmd_side_effect(cmd, **kwargs): assert data.ethtool_statistics[0].vendor_statistics is not None +def test_collect_ethtool_info_skips_loopback_driver_command(collector): + """Loopback interfaces skip only the ethtool -i driver command.""" + loopback = NetworkInterface(name="lo", flags=["LOOPBACK"]) + collector._run_sut_cmd = MagicMock( + return_value=MagicMock(exit_code=0, stdout="", command="ethtool lo") + ) + + data, skipped = collector._collect_ethtool_info([loopback]) + + assert "lo" in data + assert skipped == set() + collector._run_sut_cmd.assert_called_once() + assert collector._run_sut_cmd.call_args.args[0] == "ethtool lo" + + +def test_collect_data_logs_ethtool_driver_failure(collector, conn_mock): + """A failed ethtool -i command emits a warning with command details.""" + collector.system_info.os_family = OSFamily.LINUX + + def run_sut_cmd_side_effect(cmd, **kwargs): + if "addr show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ADDR_OUTPUT, command=cmd) + if "route show" in cmd: + return MagicMock(exit_code=0, stdout=IP_ROUTE_OUTPUT, command=cmd) + if "rule show" in cmd: + return MagicMock(exit_code=0, stdout=IP_RULE_OUTPUT, command=cmd) + if "neighbor show" in cmd: + return MagicMock(exit_code=0, stdout=IP_NEIGHBOR_OUTPUT, command=cmd) + if "ethtool -i" in cmd and "eth0" in cmd: + return MagicMock( + exit_code=1, + stdout="", + stderr="driver information unavailable", + command=cmd, + ) + if "ethtool" in cmd or "lldpcli" in cmd or "lldpctl" in cmd: + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert data is not None + event = next( + event + for event in result.events + if "ethtool -i driver info for interface: eth0" in event.description + ) + assert event.priority == EventPriority.WARNING + assert event.data["exit_code"] == 1 + assert event.data["stderr"] == "driver information unavailable" + + def test_collect_data_skips_non_vendor_netdev(collector, conn_mock): """ethtool -S is not run for netdevs whose driver is not a known vendor.""" collector.system_info.os_family = OSFamily.LINUX diff --git a/test/unit/plugin/test_nic_analyzer.py b/test/unit/plugin/test_nic_analyzer.py new file mode 100644 index 00000000..3beaa12f --- /dev/null +++ b/test/unit/plugin/test_nic_analyzer.py @@ -0,0 +1,97 @@ +############################################################################### +# +# MIT License +# +############################################################################### +from nodescraper.enums import ExecutionStatus +from nodescraper.plugins.inband.nic.analyzer_args import NicAnalyzerArgs +from nodescraper.plugins.inband.nic.nic_analyzer import NicAnalyzer +from nodescraper.plugins.inband.nic.nic_data import ( + NicCliDevice, + NicDataModel, + PensandoNicVersionFirmware, +) + + +def test_broadcom_firmware_exact_match(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + broadcom_nic_firmware={0: "238.1.169.0"}, + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware="238.1.169.0"), + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_broadcom_firmware_regex_matches_supported_formats(system_info): + pattern = r"^[0-9]+(?:\.[0-9]+){2,}(?:-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$" + for firmware_version in ("1.117.5-a-77", "28.35.1012", "238.1.168.0"): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + broadcom_nic_firmware={0: firmware_version}, + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware=pattern), + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_broadcom_firmware_mismatch_is_reported(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + broadcom_nic_firmware={0: "238.1.168.0"}, + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware="238.1.169.0"), + ) + + assert result.status == ExecutionStatus.WARNING + assert any("firmware" in event.description.lower() for event in result.events) + + +def test_missing_broadcom_firmware_is_reported(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + broadcom_nic_devices=[NicCliDevice(device_num=0, model="Thor2")], + ) + + result = analyzer.analyze_data( + data, + NicAnalyzerArgs(expected_nic_firmware="238.1.169.0"), + ) + + assert result.status == ExecutionStatus.WARNING + assert any("unavailable" in event.data["reason"] for event in result.events) + + +def test_pensando_firmware_is_validated_by_nic_analyzer(system_info): + analyzer = NicAnalyzer(system_info) + data = NicDataModel( + pensando_nic_version_firmware=[ + PensandoNicVersionFirmware( + nic_id="card0", + pcie_bdf="0000:01:00.0", + firmware_a="238.1.169.0", + ) + ] + ) + args = NicAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = analyzer.analyze_data(data, args) + + assert result.status == ExecutionStatus.OK + assert not result.events diff --git a/test/unit/plugin/test_niccli_collector.py b/test/unit/plugin/test_niccli_collector.py index 46a2fcff..90161728 100644 --- a/test/unit/plugin/test_niccli_collector.py +++ b/test/unit/plugin/test_niccli_collector.py @@ -1,450 +1,450 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2025 Advanced Micro Devices, Inc. -# -############################################################################### -from unittest.mock import MagicMock - -import pytest - -from nodescraper.enums.executionstatus import ExecutionStatus -from nodescraper.enums.systeminteraction import SystemInteractionLevel -from nodescraper.models.systeminfo import OSFamily -from nodescraper.plugins.inband.nic.nic_collector import ( - NicCollector, - _parse_bcmcli_device_list, - _parse_bcmcli_qos, - _parse_niccli_version, -) -from nodescraper.plugins.inband.nic.nic_data import ( - NicCliDevice, - NicCliQos, - NicDataModel, - PensandoNicCard, -) - - -@pytest.fixture -def collector(system_info, conn_mock): - return NicCollector( - system_info=system_info, - system_interaction_level=SystemInteractionLevel.PASSIVE, - connection=conn_mock, - ) - - -NICCLI_LISTDEV_OUTPUT = """1) Broadcom BCM57608 1x400G QSFP-DD PCIe Ethernet NIC (Adp#1 Port#1) - Device Interface : abcd1p1 - MAC Address : 81:82:83:84:85:88 - PCI Address : 0000:22:00.0 -""" - -NICCLI_QOS_OUTPUT = """IEEE 8021QAZ ETS Configuration TLV: - PRIO_MAP: 0:0 1:0 2:0 3:1 4:0 5:0 6:0 7:2 - TC Bandwidth: 50% 50% 0% - TSA_MAP: 0:ets 1:ets 2:strict -IEEE 8021QAZ PFC TLV: - PFC enabled: 3 -IEEE 8021QAZ APP TLV: - APP#0: - Priority: 7 - Sel: 5 - DSCP: 48 - - APP#1: - Priority: 3 - Sel: 5 - DSCP: 26 - - APP#2: - Priority: 3 - Sel: 3 - UDP or DCCP: 4791 - -TC Rate Limit: 100% 100% 100% 0% 0% 0% 0% 0% -""" - - -def test_parse_niccli_listdev_device(collector): - """Test parsing Broadcom NIC device from niccli --list_devices output.""" - devices = collector._parse_niccli_listdev(NICCLI_LISTDEV_OUTPUT) - - assert len(devices) == 1 - device1 = devices[0] - assert device1.device_num == 1 - assert device1.model == "Broadcom BCM57608 1x400G QSFP-DD PCIe Ethernet NIC" - assert device1.adapter_port == "Adp#1 Port#1" - assert device1.interface_name == "abcd1p1" - assert device1.mac_address == "81:82:83:84:85:88" - assert device1.pci_address == "0000:22:00.0" - - -def test_parse_niccli_listdev_empty_output(collector): - """Test parsing empty niccli --list_devices output.""" - devices = collector._parse_niccli_listdev("") - assert len(devices) == 0 - - -def test_parse_niccli_listdev_malformed_output(collector): - """Test parsing malformed niccli --list_devices output gracefully.""" - malformed = """some random text -not a valid device line -123 invalid format -""" - devices = collector._parse_niccli_listdev(malformed) - assert isinstance(devices, list) - - -def test_parse_niccli_qos_complete(collector): - """Test parsing complete Broadcom NIC QoS output with all fields.""" - qos = collector._parse_niccli_qos(1, NICCLI_QOS_OUTPUT) - - assert qos.device_num == 1 - assert qos.raw_output == NICCLI_QOS_OUTPUT - assert len(qos.prio_map) == 8 - assert qos.prio_map[0] == 0 - assert qos.prio_map[3] == 1 - assert qos.prio_map[7] == 2 - assert len(qos.tc_bandwidth) == 3 - assert qos.tc_bandwidth[0] == 50 - assert qos.tc_bandwidth[1] == 50 - assert qos.tc_bandwidth[2] == 0 - assert len(qos.tsa_map) == 3 - assert qos.tsa_map[0] == "ets" - assert qos.tsa_map[2] == "strict" - assert qos.pfc_enabled == 3 - assert len(qos.app_entries) == 3 - assert qos.app_entries[0].priority == 7 - assert qos.app_entries[0].sel == 5 - assert qos.app_entries[0].dscp == 48 - assert qos.app_entries[2].protocol == "UDP or DCCP" - assert qos.app_entries[2].port == 4791 - assert len(qos.tc_rate_limit) == 8 - assert qos.tc_rate_limit[0] == 100 - - -def test_parse_niccli_qos_empty_output(collector): - """Test parsing empty QoS output.""" - qos = collector._parse_niccli_qos(1, "") - assert qos.device_num == 1 - assert qos.raw_output == "" - assert len(qos.prio_map) == 0 - assert len(qos.tc_bandwidth) == 0 - assert len(qos.tsa_map) == 0 - assert qos.pfc_enabled is None - assert len(qos.app_entries) == 0 - assert len(qos.tc_rate_limit) == 0 - - -def test_parse_niccli_qos_multiple_app_protocols(collector): - """Test parsing QoS with APP entries having different protocols.""" - qos_multi_protocol = """IEEE 8021QAZ ETS Configuration TLV: - PRIO_MAP: 0:0 - TC Bandwidth: 100% - TSA_MAP: 0:ets -IEEE 8021QAZ PFC TLV: - PFC enabled: 0 -IEEE 8021QAZ APP TLV: - APP#0: - Priority: 5 - Sel: 3 - TCP: 8080 - - APP#1: - Priority: 6 - Sel: 3 - UDP: 9000 - -TC Rate Limit: 100% -""" - qos = collector._parse_niccli_qos(3, qos_multi_protocol) - assert len(qos.app_entries) == 2 - assert qos.app_entries[0].priority == 5 - assert qos.app_entries[0].sel == 3 - assert qos.app_entries[0].protocol == "TCP" - assert qos.app_entries[0].port == 8080 - assert qos.app_entries[1].priority == 6 - assert qos.app_entries[1].protocol == "UDP" - assert qos.app_entries[1].port == 9000 - - -def test_parse_niccli_qos_malformed_values(collector): - """Test parsing QoS output with malformed values gracefully.""" - malformed = """IEEE 8021QAZ ETS Configuration TLV: - PRIO_MAP: 0:invalid 1:1 bad:data - TC Bandwidth: 50% invalid 50% - TSA_MAP: 0:ets bad:value 1:strict -IEEE 8021QAZ PFC TLV: - PFC enabled: not_a_number -TC Rate Limit: 100% bad% 100% -""" - qos = collector._parse_niccli_qos(1, malformed) - assert qos.device_num == 1 - assert 1 in qos.prio_map - assert qos.prio_map[1] == 1 - assert 50 in qos.tc_bandwidth - assert qos.tsa_map.get(0) == "ets" - assert qos.tsa_map.get(1) == "strict" - assert qos.pfc_enabled is None - - -def test_nic_data_model_with_broadcom_nic(collector): - """Test creating NicDataModel with Broadcom NIC data.""" - device = NicCliDevice( - device_num=1, - model="Broadcom BCM57608 1x400G QSFP-DD PCIe Ethernet NIC", - adapter_port="Adp#1 Port#1", - interface_name="benic1p1", - mac_address="8C:84:74:37:C3:70", - pci_address="0000:06:00.0", - ) - qos = NicCliQos( - device_num=1, - raw_output="test output", - prio_map={0: 0, 1: 1}, - tc_bandwidth=[50, 50], - tsa_map={0: "ets", 1: "strict"}, - pfc_enabled=3, - tc_rate_limit=[100, 100], - ) - data = NicDataModel( - broadcom_nic_devices=[device], - broadcom_nic_qos={1: qos}, - ) - assert len(data.broadcom_nic_devices) == 1 - assert len(data.broadcom_nic_qos) == 1 - assert data.broadcom_nic_devices[0].device_num == 1 - assert data.broadcom_nic_devices[0].interface_name == "benic1p1" - assert data.broadcom_nic_qos[1].device_num == 1 - assert data.broadcom_nic_qos[1].pfc_enabled == 3 - - -def test_nic_data_model_with_pensando_nic(collector): - """Test creating NicDataModel with Pensando NIC data.""" - card1 = PensandoNicCard( - id="42424650-4c32-3533-3330-323934000000", - pcie_bdf="0000:06:00.0", - asic="salina", - fw_partition="A", - serial_number="FPL25330294", - ) - card2 = PensandoNicCard( - id="42424650-4c32-3533-3731-304535000000", - pcie_bdf="0000:16:00.0", - asic="salina", - fw_partition="A", - serial_number="FPL253710E5", - ) - data = NicDataModel( - pensando_nic_cards=[card1, card2], - ) - assert len(data.pensando_nic_cards) == 2 - assert data.pensando_nic_cards[0].id == "42424650-4c32-3533-3330-323934000000" - assert data.pensando_nic_cards[0].pcie_bdf == "0000:06:00.0" - assert data.pensando_nic_cards[0].asic == "salina" - assert data.pensando_nic_cards[1].serial_number == "FPL253710E5" - - -def test_collect_data_not_ran_when_no_nic_hardware(collector, conn_mock): - """Skip collection when discovery finds no Broadcom or Pensando NICs.""" - collector.system_info.os_family = OSFamily.LINUX - collector._run_sut_cmd = MagicMock( - return_value=MagicMock(exit_code=1, stdout="", stderr="not found", command="") - ) - - result, data = collector.collect_data() - - assert result.status == ExecutionStatus.NOT_RAN - assert data is None - assert "skipped" in result.message.lower() - assert collector._run_sut_cmd.call_count <= 6 - - -def test_collect_data_skips_nicctl_commands_when_no_pensando_cards(collector, conn_mock): - """Do not run nicctl global/legacy commands when nicctl show card finds no cards.""" - collector.system_info.os_family = OSFamily.LINUX - commands_run: list[str] = [] - - def run_sut_cmd_side_effect(cmd, **kwargs): - commands_run.append(cmd) - if "niccli" in cmd and ("--list" in cmd or "--list_devices" in cmd or "--listdev" in cmd): - return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) - if cmd.strip() == "nicctl show card": - return MagicMock(exit_code=1, stdout="", stderr="no card", command=cmd) - return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) - - collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) - - result, data = collector.collect_data() - - assert result.status == ExecutionStatus.OK - assert data is not None - assert not any(c.startswith("nicctl show card flash") for c in commands_run) - assert not any(c == "nicctl --version" for c in commands_run) - - -def test_collect_data_success(collector, conn_mock): - """Test successful collection of niccli/nicctl data.""" - collector.system_info.os_family = OSFamily.LINUX - - def run_sut_cmd_side_effect(cmd, **kwargs): - if "niccli" in cmd and ("--list" in cmd or "--list_devices" in cmd): - return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) - if cmd.strip() == "nicctl show card": - return MagicMock( - exit_code=0, - stdout="1111111-4c32-3533-3330-12345000000 0000:06:00.0\n", - stderr="", - command=cmd, - ) - if "nicctl" in cmd or "niccli" in cmd: - return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) - return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) - - collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) - - result, data = collector.collect_data() - - assert result.status == ExecutionStatus.OK - assert data is not None - assert isinstance(data, NicDataModel) - assert len(data.results) >= 1 - - -BCMCLI_DEVICE_LIST_OUTPUT = """\ -PCI Address Device Type Firmware Version --------- ----------- ---------------- -0000:03:00.0 BCM957608 1.140.0 -0000:04:00.0 BCM957608 1.140.0 -""" - -BCMCLI_QOS_OUTPUT = """\ -ETS Configuration ------------------ -TC TX_BW RX_BW TSA RateLimit Priority -0 50 50 ETS 0 0 1 2 -1 50 50 ETS 0 3 4 -2 0 0 strict 0 5 6 7 -PFC configuration ------------------ -priority 0 1 2 3 4 5 6 7 -enabled 0 0 0 1 0 0 0 0 -APP TLV Configuration ---------------------- -Index Selector Priority DSCP/Protocol -0 5 7 48 -""" - - -def test_parse_niccli_version(): - assert _parse_niccli_version("niccli v234") == 234 - assert _parse_niccli_version("") is None - - -def test_parse_bcmcli_device_list(): - """PCI addresses parsed from table; empty input returns empty list.""" - ids = _parse_bcmcli_device_list(BCMCLI_DEVICE_LIST_OUTPUT) - assert ids == ["0000:03:00.0", "0000:04:00.0"] - assert _parse_bcmcli_device_list("") == [] - - -def test_parse_bcmcli_qos(): - """ETS rows, PFC bitmask, and APP entries all parsed correctly.""" - qos = _parse_bcmcli_qos("0000:03:00.0", BCMCLI_QOS_OUTPUT) - assert qos.prio_map[0] == 0 and qos.prio_map[3] == 1 and qos.prio_map[5] == 2 - assert qos.tc_bandwidth == [50, 50, 0] - assert qos.tsa_map[0] == "ETS" and qos.tsa_map[2] == "strict" - assert qos.pfc_enabled == 8 # only priority 3 set (1 << 3) - assert len(qos.app_entries) == 1 and qos.app_entries[0].priority == 7 - - -def test_collect_data_uses_bcmcli_when_detected(collector): - """Both bcmcli probes succeed; bcmcli path used, broadcom_cli_type='bcmcli'.""" - commands_run: list[str] = [] - - def side_effect(cmd, **kwargs): - commands_run.append(cmd) - if "bcmcli_show version" in cmd or "which bcmcli_show" in cmd: - return MagicMock(exit_code=0, stdout="bcmcli v1.140", stderr="", command=cmd) - if "bcmcli_show device_list" in cmd: - return MagicMock(exit_code=0, stdout=BCMCLI_DEVICE_LIST_OUTPUT, stderr="", command=cmd) - if "nicctl show card" in cmd: - return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) - return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) - - collector._run_sut_cmd = MagicMock(side_effect=side_effect) - result, data = collector.collect_data() - - assert result.status == ExecutionStatus.OK - assert data.broadcom_cli_type == "bcmcli" - assert any("bcmcli_" in c for c in commands_run) - - -def test_detect_broadcom_cli_falls_back_when_device_list_empty(collector): - """bcmcli_show version exits 0 but device_list empty; falls back to niccli.""" - - def side_effect(cmd, **kwargs): - if "bcmcli_show version" in cmd: - return MagicMock(exit_code=0, stdout="bcmcli v1.140", stderr="", command=cmd) - if "bcmcli_show device_list" in cmd: - return MagicMock( - exit_code=0, stdout="PCI Address Device Type\n", stderr="", command=cmd - ) - if "niccli --version" in cmd: - return MagicMock(exit_code=0, stdout="niccli v234", stderr="", command=cmd) - if "--list_devices" in cmd or "--listdev" in cmd or "--list" in cmd: - return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) - if "nicctl show card" in cmd: - return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) - return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) - - collector._run_sut_cmd = MagicMock(side_effect=side_effect) - _, data = collector.collect_data() - - assert data.broadcom_cli_type == "niccli" - - -def test_broadcom_cli_override_skips_detection(collector): - """broadcom_cli_override='niccli' bypasses bcmcli probes entirely.""" - from nodescraper.plugins.inband.nic.collector_args import NicCollectorArgs - - commands_run: list[str] = [] - - def side_effect(cmd, **kwargs): - commands_run.append(cmd) - if "niccli --version" in cmd: - return MagicMock(exit_code=0, stdout="niccli v234", stderr="", command=cmd) - if "--list_devices" in cmd or "--listdev" in cmd or "--list" in cmd: - return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) - if "nicctl show card" in cmd: - return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) - return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) - - collector._run_sut_cmd = MagicMock(side_effect=side_effect) - _, data = collector.collect_data(args=NicCollectorArgs(broadcom_cli_override="niccli")) - - assert data.broadcom_cli_type == "niccli" - assert not any("bcmcli_show version" in c for c in commands_run) - - -def test_collect_data_niccli_version_routing(collector): - """niccli v234 uses --dev/--getoption syntax; legacy -dev/-getoption not run.""" - commands_run: list[str] = [] - - def side_effect(cmd, **kwargs): - commands_run.append(cmd) - if cmd == "niccli --version": - return MagicMock(exit_code=0, stdout="niccli v234", stderr="", command=cmd) - if "--list_devices" in cmd or "--listdev" in cmd or "--list" in cmd: - return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) - if "nicctl show card" in cmd: - return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) - return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) - - collector._run_sut_cmd = MagicMock(side_effect=side_effect) - collector.collect_data() - - assert any("--dev" in c and "--getoption" in c for c in commands_run) - assert not any(" -dev " in c for c in commands_run) +############################################################################### +# +# MIT License +# +# Copyright (c) 2025 Advanced Micro Devices, Inc. +# +############################################################################### +from unittest.mock import MagicMock + +import pytest + +from nodescraper.enums.executionstatus import ExecutionStatus +from nodescraper.enums.systeminteraction import SystemInteractionLevel +from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.nic.nic_collector import ( + NicCollector, + _parse_bcmcli_device_list, + _parse_bcmcli_qos, + _parse_niccli_version, +) +from nodescraper.plugins.inband.nic.nic_data import ( + NicCliDevice, + NicCliQos, + NicDataModel, + PensandoNicCard, +) + + +@pytest.fixture +def collector(system_info, conn_mock): + return NicCollector( + system_info=system_info, + system_interaction_level=SystemInteractionLevel.PASSIVE, + connection=conn_mock, + ) + + +NICCLI_LISTDEV_OUTPUT = """1) Broadcom BCM57608 1x400G QSFP-DD PCIe Ethernet NIC (Adp#1 Port#1) + Device Interface : abcd1p1 + MAC Address : 81:82:83:84:85:88 + PCI Address : 0000:22:00.0 +""" + +NICCLI_QOS_OUTPUT = """IEEE 8021QAZ ETS Configuration TLV: + PRIO_MAP: 0:0 1:0 2:0 3:1 4:0 5:0 6:0 7:2 + TC Bandwidth: 50% 50% 0% + TSA_MAP: 0:ets 1:ets 2:strict +IEEE 8021QAZ PFC TLV: + PFC enabled: 3 +IEEE 8021QAZ APP TLV: + APP#0: + Priority: 7 + Sel: 5 + DSCP: 48 + + APP#1: + Priority: 3 + Sel: 5 + DSCP: 26 + + APP#2: + Priority: 3 + Sel: 3 + UDP or DCCP: 4791 + +TC Rate Limit: 100% 100% 100% 0% 0% 0% 0% 0% +""" + + +def test_parse_niccli_listdev_device(collector): + """Test parsing Broadcom NIC device from niccli --list_devices output.""" + devices = collector._parse_niccli_listdev(NICCLI_LISTDEV_OUTPUT) + + assert len(devices) == 1 + device1 = devices[0] + assert device1.device_num == 1 + assert device1.model == "Broadcom BCM57608 1x400G QSFP-DD PCIe Ethernet NIC" + assert device1.adapter_port == "Adp#1 Port#1" + assert device1.interface_name == "abcd1p1" + assert device1.mac_address == "81:82:83:84:85:88" + assert device1.pci_address == "0000:22:00.0" + + +def test_parse_niccli_listdev_empty_output(collector): + """Test parsing empty niccli --list_devices output.""" + devices = collector._parse_niccli_listdev("") + assert len(devices) == 0 + + +def test_parse_niccli_listdev_malformed_output(collector): + """Test parsing malformed niccli --list_devices output gracefully.""" + malformed = """some random text +not a valid device line +123 invalid format +""" + devices = collector._parse_niccli_listdev(malformed) + assert isinstance(devices, list) + + +def test_parse_niccli_qos_complete(collector): + """Test parsing complete Broadcom NIC QoS output with all fields.""" + qos = collector._parse_niccli_qos(1, NICCLI_QOS_OUTPUT) + + assert qos.device_num == 1 + assert qos.raw_output == NICCLI_QOS_OUTPUT + assert len(qos.prio_map) == 8 + assert qos.prio_map[0] == 0 + assert qos.prio_map[3] == 1 + assert qos.prio_map[7] == 2 + assert len(qos.tc_bandwidth) == 3 + assert qos.tc_bandwidth[0] == 50 + assert qos.tc_bandwidth[1] == 50 + assert qos.tc_bandwidth[2] == 0 + assert len(qos.tsa_map) == 3 + assert qos.tsa_map[0] == "ets" + assert qos.tsa_map[2] == "strict" + assert qos.pfc_enabled == 3 + assert len(qos.app_entries) == 3 + assert qos.app_entries[0].priority == 7 + assert qos.app_entries[0].sel == 5 + assert qos.app_entries[0].dscp == 48 + assert qos.app_entries[2].protocol == "UDP or DCCP" + assert qos.app_entries[2].port == 4791 + assert len(qos.tc_rate_limit) == 8 + assert qos.tc_rate_limit[0] == 100 + + +def test_parse_niccli_qos_empty_output(collector): + """Test parsing empty QoS output.""" + qos = collector._parse_niccli_qos(1, "") + assert qos.device_num == 1 + assert qos.raw_output == "" + assert len(qos.prio_map) == 0 + assert len(qos.tc_bandwidth) == 0 + assert len(qos.tsa_map) == 0 + assert qos.pfc_enabled is None + assert len(qos.app_entries) == 0 + assert len(qos.tc_rate_limit) == 0 + + +def test_parse_niccli_qos_multiple_app_protocols(collector): + """Test parsing QoS with APP entries having different protocols.""" + qos_multi_protocol = """IEEE 8021QAZ ETS Configuration TLV: + PRIO_MAP: 0:0 + TC Bandwidth: 100% + TSA_MAP: 0:ets +IEEE 8021QAZ PFC TLV: + PFC enabled: 0 +IEEE 8021QAZ APP TLV: + APP#0: + Priority: 5 + Sel: 3 + TCP: 8080 + + APP#1: + Priority: 6 + Sel: 3 + UDP: 9000 + +TC Rate Limit: 100% +""" + qos = collector._parse_niccli_qos(3, qos_multi_protocol) + assert len(qos.app_entries) == 2 + assert qos.app_entries[0].priority == 5 + assert qos.app_entries[0].sel == 3 + assert qos.app_entries[0].protocol == "TCP" + assert qos.app_entries[0].port == 8080 + assert qos.app_entries[1].priority == 6 + assert qos.app_entries[1].protocol == "UDP" + assert qos.app_entries[1].port == 9000 + + +def test_parse_niccli_qos_malformed_values(collector): + """Test parsing QoS output with malformed values gracefully.""" + malformed = """IEEE 8021QAZ ETS Configuration TLV: + PRIO_MAP: 0:invalid 1:1 bad:data + TC Bandwidth: 50% invalid 50% + TSA_MAP: 0:ets bad:value 1:strict +IEEE 8021QAZ PFC TLV: + PFC enabled: not_a_number +TC Rate Limit: 100% bad% 100% +""" + qos = collector._parse_niccli_qos(1, malformed) + assert qos.device_num == 1 + assert 1 in qos.prio_map + assert qos.prio_map[1] == 1 + assert 50 in qos.tc_bandwidth + assert qos.tsa_map.get(0) == "ets" + assert qos.tsa_map.get(1) == "strict" + assert qos.pfc_enabled is None + + +def test_nic_data_model_with_broadcom_nic(collector): + """Test creating NicDataModel with Broadcom NIC data.""" + device = NicCliDevice( + device_num=1, + model="Broadcom BCM57608 1x400G QSFP-DD PCIe Ethernet NIC", + adapter_port="Adp#1 Port#1", + interface_name="benic1p1", + mac_address="8C:84:74:37:C3:70", + pci_address="0000:06:00.0", + ) + qos = NicCliQos( + device_num=1, + raw_output="test output", + prio_map={0: 0, 1: 1}, + tc_bandwidth=[50, 50], + tsa_map={0: "ets", 1: "strict"}, + pfc_enabled=3, + tc_rate_limit=[100, 100], + ) + data = NicDataModel( + broadcom_nic_devices=[device], + broadcom_nic_qos={1: qos}, + ) + assert len(data.broadcom_nic_devices) == 1 + assert len(data.broadcom_nic_qos) == 1 + assert data.broadcom_nic_devices[0].device_num == 1 + assert data.broadcom_nic_devices[0].interface_name == "benic1p1" + assert data.broadcom_nic_qos[1].device_num == 1 + assert data.broadcom_nic_qos[1].pfc_enabled == 3 + + +def test_nic_data_model_with_pensando_nic(collector): + """Test creating NicDataModel with Pensando NIC data.""" + card1 = PensandoNicCard( + id="42424650-4c32-3533-3330-323934000000", + pcie_bdf="0000:06:00.0", + asic="salina", + fw_partition="A", + serial_number="FPL25330294", + ) + card2 = PensandoNicCard( + id="42424650-4c32-3533-3731-304535000000", + pcie_bdf="0000:16:00.0", + asic="salina", + fw_partition="A", + serial_number="FPL253710E5", + ) + data = NicDataModel( + pensando_nic_cards=[card1, card2], + ) + assert len(data.pensando_nic_cards) == 2 + assert data.pensando_nic_cards[0].id == "42424650-4c32-3533-3330-323934000000" + assert data.pensando_nic_cards[0].pcie_bdf == "0000:06:00.0" + assert data.pensando_nic_cards[0].asic == "salina" + assert data.pensando_nic_cards[1].serial_number == "FPL253710E5" + + +def test_collect_data_not_ran_when_no_nic_hardware(collector, conn_mock): + """Skip collection when discovery finds no Broadcom or Pensando NICs.""" + collector.system_info.os_family = OSFamily.LINUX + collector._run_sut_cmd = MagicMock( + return_value=MagicMock(exit_code=1, stdout="", stderr="not found", command="") + ) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.NOT_RAN + assert data is None + assert "skipped" in result.message.lower() + assert collector._run_sut_cmd.call_count <= 6 + + +def test_collect_data_skips_nicctl_commands_when_no_pensando_cards(collector, conn_mock): + """Do not run nicctl global/legacy commands when nicctl show card finds no cards.""" + collector.system_info.os_family = OSFamily.LINUX + commands_run: list[str] = [] + + def run_sut_cmd_side_effect(cmd, **kwargs): + commands_run.append(cmd) + if "niccli" in cmd and ("--list" in cmd or "--list_devices" in cmd or "--listdev" in cmd): + return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) + if cmd.strip() == "nicctl show card": + return MagicMock(exit_code=1, stdout="", stderr="no card", command=cmd) + return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + assert not any(c.startswith("nicctl show card flash") for c in commands_run) + assert not any(c == "nicctl --version" for c in commands_run) + + +def test_collect_data_success(collector, conn_mock): + """Test successful collection of niccli/nicctl data.""" + collector.system_info.os_family = OSFamily.LINUX + + def run_sut_cmd_side_effect(cmd, **kwargs): + if "niccli" in cmd and ("--list" in cmd or "--list_devices" in cmd): + return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) + if cmd.strip() == "nicctl show card": + return MagicMock( + exit_code=0, + stdout="1111111-4c32-3533-3330-12345000000 0000:06:00.0\n", + stderr="", + command=cmd, + ) + if "nicctl" in cmd or "niccli" in cmd: + return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=run_sut_cmd_side_effect) + + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data is not None + assert isinstance(data, NicDataModel) + assert len(data.results) >= 1 + + +BCMCLI_DEVICE_LIST_OUTPUT = """\ +PCI Address Device Type Firmware Version +-------- ----------- ---------------- +0000:03:00.0 BCM957608 1.140.0 +0000:04:00.0 BCM957608 1.140.0 +""" + +BCMCLI_QOS_OUTPUT = """\ +ETS Configuration +----------------- +TC TX_BW RX_BW TSA RateLimit Priority +0 50 50 ETS 0 0 1 2 +1 50 50 ETS 0 3 4 +2 0 0 strict 0 5 6 7 +PFC configuration +----------------- +priority 0 1 2 3 4 5 6 7 +enabled 0 0 0 1 0 0 0 0 +APP TLV Configuration +--------------------- +Index Selector Priority DSCP/Protocol +0 5 7 48 +""" + + +def test_parse_niccli_version(): + assert _parse_niccli_version("niccli v234") == 234 + assert _parse_niccli_version("") is None + + +def test_parse_bcmcli_device_list(): + """PCI addresses parsed from table; empty input returns empty list.""" + ids = _parse_bcmcli_device_list(BCMCLI_DEVICE_LIST_OUTPUT) + assert ids == ["0000:03:00.0", "0000:04:00.0"] + assert _parse_bcmcli_device_list("") == [] + + +def test_parse_bcmcli_qos(): + """ETS rows, PFC bitmask, and APP entries all parsed correctly.""" + qos = _parse_bcmcli_qos("0000:03:00.0", BCMCLI_QOS_OUTPUT) + assert qos.prio_map[0] == 0 and qos.prio_map[3] == 1 and qos.prio_map[5] == 2 + assert qos.tc_bandwidth == [50, 50, 0] + assert qos.tsa_map[0] == "ETS" and qos.tsa_map[2] == "strict" + assert qos.pfc_enabled == 8 # only priority 3 set (1 << 3) + assert len(qos.app_entries) == 1 and qos.app_entries[0].priority == 7 + + +def test_collect_data_uses_bcmcli_when_detected(collector): + """Both bcmcli probes succeed; bcmcli path used, broadcom_cli_type='bcmcli'.""" + commands_run: list[str] = [] + + def side_effect(cmd, **kwargs): + commands_run.append(cmd) + if "bcmcli_show version" in cmd or "which bcmcli_show" in cmd: + return MagicMock(exit_code=0, stdout="bcmcli v1.140", stderr="", command=cmd) + if "bcmcli_show device_list" in cmd: + return MagicMock(exit_code=0, stdout=BCMCLI_DEVICE_LIST_OUTPUT, stderr="", command=cmd) + if "nicctl show card" in cmd: + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=side_effect) + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.OK + assert data.broadcom_cli_type == "bcmcli" + assert any("bcmcli_" in c for c in commands_run) + + +def test_detect_broadcom_cli_falls_back_when_device_list_empty(collector): + """bcmcli_show version exits 0 but device_list empty; falls back to niccli.""" + + def side_effect(cmd, **kwargs): + if "bcmcli_show version" in cmd: + return MagicMock(exit_code=0, stdout="bcmcli v1.140", stderr="", command=cmd) + if "bcmcli_show device_list" in cmd: + return MagicMock( + exit_code=0, stdout="PCI Address Device Type\n", stderr="", command=cmd + ) + if "niccli --version" in cmd: + return MagicMock(exit_code=0, stdout="niccli v234", stderr="", command=cmd) + if "--list_devices" in cmd or "--listdev" in cmd or "--list" in cmd: + return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) + if "nicctl show card" in cmd: + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=side_effect) + _, data = collector.collect_data() + + assert data.broadcom_cli_type == "niccli" + + +def test_broadcom_cli_override_skips_detection(collector): + """broadcom_cli_override='niccli' bypasses bcmcli probes entirely.""" + from nodescraper.plugins.inband.nic.collector_args import NicCollectorArgs + + commands_run: list[str] = [] + + def side_effect(cmd, **kwargs): + commands_run.append(cmd) + if "niccli --version" in cmd: + return MagicMock(exit_code=0, stdout="niccli v234", stderr="", command=cmd) + if "--list_devices" in cmd or "--listdev" in cmd or "--list" in cmd: + return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) + if "nicctl show card" in cmd: + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=side_effect) + _, data = collector.collect_data(args=NicCollectorArgs(broadcom_cli_override="niccli")) + + assert data.broadcom_cli_type == "niccli" + assert not any("bcmcli_show version" in c for c in commands_run) + + +def test_collect_data_niccli_version_routing(collector): + """niccli v234 uses --dev/--getoption syntax; legacy -dev/-getoption not run.""" + commands_run: list[str] = [] + + def side_effect(cmd, **kwargs): + commands_run.append(cmd) + if cmd == "niccli --version": + return MagicMock(exit_code=0, stdout="niccli v234", stderr="", command=cmd) + if "--list_devices" in cmd or "--listdev" in cmd or "--list" in cmd: + return MagicMock(exit_code=0, stdout=NICCLI_LISTDEV_OUTPUT, stderr="", command=cmd) + if "nicctl show card" in cmd: + return MagicMock(exit_code=1, stdout="", stderr="", command=cmd) + return MagicMock(exit_code=0, stdout="", stderr="", command=cmd) + + collector._run_sut_cmd = MagicMock(side_effect=side_effect) + collector.collect_data() + + assert any("--dev" in c and "--getoption" in c for c in commands_run) + assert not any(" -dev " in c for c in commands_run) diff --git a/test/unit/plugin/test_nvme_analyzer.py b/test/unit/plugin/test_nvme_analyzer.py new file mode 100644 index 00000000..699fe914 --- /dev/null +++ b/test/unit/plugin/test_nvme_analyzer.py @@ -0,0 +1,106 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.plugins.inband.nvme.analyzer_args import NvmeAnalyzerArgs +from nodescraper.plugins.inband.nvme.nvme_analyzer import NvmeAnalyzer +from nodescraper.plugins.inband.nvme.nvmedata import DeviceNvmeData, NvmeDataModel + + +def test_nvme_smart_media_errors_within_threshold(system_info): + analyzer = NvmeAnalyzer(system_info) + data = NvmeDataModel( + devices={ + "nvme0": DeviceNvmeData(smart_log="media_errors : 0"), + } + ) + + result = analyzer.analyze_data(data, NvmeAnalyzerArgs(maximum_smart_error_count=0)) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_nvme_smart_media_errors_exceed_threshold(system_info): + analyzer = NvmeAnalyzer(system_info) + data = NvmeDataModel( + devices={ + "nvme0": DeviceNvmeData(smart_log="media_errors : 2"), + } + ) + + result = analyzer.analyze_data(data, NvmeAnalyzerArgs(maximum_smart_error_count=0)) + + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert result.events[0].priority == EventPriority.ERROR + assert result.events[0].data["media_errors"] == 2 + + +def test_nvme_smart_media_errors_missing_for_every_device(system_info): + analyzer = NvmeAnalyzer(system_info) + data = NvmeDataModel( + devices={ + "nvme0": DeviceNvmeData(smart_log="critical_warning : 0"), + "nvme1": DeviceNvmeData(smart_log=None), + } + ) + + result = analyzer.analyze_data(data, NvmeAnalyzerArgs(maximum_smart_error_count=0)) + + assert result.status == ExecutionStatus.ERROR + assert "unavailable" in result.message + assert len(result.events) == 2 + assert all(event.priority == EventPriority.ERROR for event in result.events) + + +def test_nvme_smart_media_errors_missing_on_one_device_still_checks_others(system_info): + analyzer = NvmeAnalyzer(system_info) + data = NvmeDataModel( + devices={ + "nvme0": DeviceNvmeData(smart_log="critical_warning : 0"), + "nvme1": DeviceNvmeData(smart_log="media_errors : 0"), + } + ) + + result = analyzer.analyze_data(data, NvmeAnalyzerArgs(maximum_smart_error_count=0)) + + assert result.status == ExecutionStatus.OK + assert len(result.events) == 1 + assert result.events[0].priority == EventPriority.WARNING + assert result.events[0].data["device"] == "nvme0" + + +def test_nvme_smart_check_not_run_without_threshold(system_info): + analyzer = NvmeAnalyzer(system_info) + data = NvmeDataModel( + devices={ + "nvme0": DeviceNvmeData(smart_log="media_errors : 2"), + } + ) + + result = analyzer.analyze_data(data, NvmeAnalyzerArgs()) + + assert result.status == ExecutionStatus.NOT_RAN diff --git a/test/unit/plugin/test_nvme_collector.py b/test/unit/plugin/test_nvme_collector.py index 1d0e819b..6cde21bd 100644 --- a/test/unit/plugin/test_nvme_collector.py +++ b/test/unit/plugin/test_nvme_collector.py @@ -186,7 +186,7 @@ def test_parse_nvme_list_json_flat_format(collector): assert e0.fw_rev == "FW-DUMMY-01" assert e0.namespace_id == "0x1" assert e0.usage and "/" in e0.usage - assert "2.1MB" in (e0.usage or "") and "7.68TB" in (e0.usage or "") + assert "2.0 MB" in (e0.usage or "") and "6.99 TB" in (e0.usage or "") assert e0.format_lba == "512 B + 0 B" assert e1.node == "/dev/nvme1n1" assert e1.serial_number == "SN-DUMMY-002" diff --git a/test/unit/plugin/test_os_analyzer.py b/test/unit/plugin/test_os_analyzer.py index c872b121..04c98a69 100644 --- a/test/unit/plugin/test_os_analyzer.py +++ b/test/unit/plugin/test_os_analyzer.py @@ -88,6 +88,87 @@ def test_no_config_data(analyzer, model_obj): assert len(result.events) == 0 +def test_maximum_load_per_cpu_core_passes(analyzer): + model = OsDataModel( + os_name="Ubuntu 22.04.2 LTS", + load_average_1m=8.0, + cpu_count=4, + ) + + result = analyzer.analyze_data( + model, + OsAnalyzerArgs(maximum_load_per_cpu_core=2.0), + ) + + assert result.status == ExecutionStatus.OK + + +def test_os_name_and_maximum_load_per_cpu_core_pass(analyzer): + model = OsDataModel( + os_name="Ubuntu 22.04.2 LTS", + load_average_1m=8.0, + cpu_count=4, + ) + + result = analyzer.analyze_data( + model, + OsAnalyzerArgs( + exp_os="Ubuntu 22.04.2 LTS", + maximum_load_per_cpu_core=2.0, + ), + ) + + assert result.status == ExecutionStatus.OK + + +def test_os_name_mismatch_still_checks_load(analyzer): + model = OsDataModel( + os_name="Ubuntu 22.04.2 LTS", + load_average_1m=9.0, + cpu_count=4, + ) + + result = analyzer.analyze_data( + model, + OsAnalyzerArgs( + exp_os="RHEL 9", + maximum_load_per_cpu_core=2.0, + ), + ) + + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 2 + + +def test_maximum_load_per_cpu_core_fails(analyzer): + model = OsDataModel( + os_name="Ubuntu 22.04.2 LTS", + load_average_1m=9.0, + cpu_count=4, + ) + + result = analyzer.analyze_data( + model, + OsAnalyzerArgs(maximum_load_per_cpu_core=2.0), + ) + + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert result.events[0].data["load_per_cpu_core"] == pytest.approx(2.25) + + +def test_maximum_load_per_cpu_core_missing_data(analyzer): + model = OsDataModel(os_name="Ubuntu 22.04.2 LTS") + + result = analyzer.analyze_data( + model, + OsAnalyzerArgs(maximum_load_per_cpu_core=2.0), + ) + + assert result.status == ExecutionStatus.WARNING + assert len(result.events) == 1 + + def test_invalid_os(analyzer, config): model = OsDataModel(os_name="some invalid os") args = OsAnalyzerArgs(exp_os=config["os_name"]) diff --git a/test/unit/plugin/test_os_collector.py b/test/unit/plugin/test_os_collector.py index 480e76da..a68bd3d1 100644 --- a/test/unit/plugin/test_os_collector.py +++ b/test/unit/plugin/test_os_collector.py @@ -26,6 +26,7 @@ import pytest from nodescraper.connection.inband.inband import CommandArtifact +from nodescraper.enums.eventpriority import EventPriority from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.models.systeminfo import OSFamily @@ -82,16 +83,28 @@ def collector(system_info, conn_mock): ], ) def test_os_collector_linux( - collector, conn_mock, stdout, version_stdout, expected_os, expected_version + collector, conn_mock, system_info, stdout, version_stdout, expected_os, expected_version ): + system_info.os_family = OSFamily.LINUX conn_mock.run_command.side_effect = [ CommandArtifact(exit_code=0, stdout=stdout, stderr="", command="cmd1"), CommandArtifact(exit_code=0, stdout=version_stdout, stderr="", command="cmd2"), + CommandArtifact(exit_code=0, stdout="0.50 0.40 0.30 1/100 1234", stderr="", command="cmd3"), + CommandArtifact(exit_code=0, stdout="4", stderr="", command="cmd4"), ] result, data = collector.collect_data() assert result.status == ExecutionStatus.OK - assert data == OsDataModel(os_name=expected_os, os_version=expected_version) + assert ( + conn_mock.run_command.call_args_list[3].kwargs["command"] + == "grep -c ^processor /proc/cpuinfo" + ) + assert data == OsDataModel( + os_name=expected_os, + os_version=expected_version, + load_average_1m=0.5, + cpu_count=4, + ) def test_os_collector_windows(system_info, conn_mock): @@ -128,3 +141,44 @@ def test_os_collector_error(collector, conn_mock, system_info): _, data = collector.collect_data() assert data is None + + +def test_os_collector_invalid_load_data(collector, conn_mock, system_info): + system_info.os_family = OSFamily.LINUX + conn_mock.run_command.side_effect = [ + CommandArtifact(exit_code=0, stdout="Ubuntu 22.04.4 LTS", stderr="", command="cmd1"), + CommandArtifact(exit_code=0, stdout='VERSION_ID="22.04"', stderr="", command="cmd2"), + CommandArtifact(exit_code=0, stdout="not-a-load", stderr="", command="cmd3"), + CommandArtifact(exit_code=0, stdout="not-a-cpu-count", stderr="", command="cmd4"), + ] + + _, data = collector.collect_data() + + assert data == OsDataModel( + os_name="Ubuntu 22.04.4 LTS", + os_version="22.04", + ) + assert sum(event.priority == EventPriority.WARNING for event in collector.result.events) == 2 + + +def test_os_collector_esxi(collector, conn_mock, system_info): + """ESXi: os_name from `vmware -v`, os_version from the esxcli 'Version:' field.""" + system_info.os_family = OSFamily.ESXI + conn_mock.run_command.side_effect = [ + CommandArtifact( + exit_code=0, + stdout="VMware ESXi 9.1.0 build-25166133", + stderr="", + command="vmware -v", + ), + CommandArtifact( + exit_code=0, + stdout=" Product: VMware ESXi\n Version: 9.1.0\n Build: Releasebuild-25166133", + stderr="", + command="esxcli system version get", + ), + ] + + result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == OsDataModel(os_name="VMware ESXi 9.1.0 build-25166133", os_version="9.1.0") diff --git a/test/unit/plugin/test_package_analyzer.py b/test/unit/plugin/test_package_analyzer.py index 2c16e516..8bb5d15e 100644 --- a/test/unit/plugin/test_package_analyzer.py +++ b/test/unit/plugin/test_package_analyzer.py @@ -107,7 +107,7 @@ def test_data_version_regex(package_analyzer, default_data_lib): def test_data_multiple_errors_regex(package_analyzer, default_data_lib): - """Test that detailed error messages are shown for multiple package errors""" + """Test that the first package error descriptions are included in the summary""" args = PackageAnalyzerArgs( exp_package_ver={ "missing-package": None, @@ -118,8 +118,16 @@ def test_data_multiple_errors_regex(package_analyzer, default_data_lib): ) res = package_analyzer.analyze_data(default_data_lib, args=args) assert res.status == ExecutionStatus.ERROR - assert "missing-package" in res.message - assert "another-missing" in res.message + assert res.message == ( + "Package version mismatch (3 errors: " + "Package missing-package not found in the package list, " + r"Package test-ubuntu-package\.x86_64 Version Mismatch, " + r"Expected 2\.\d+ but found 1.11-1.xx11, " + "Package another-missing not found in the package list)" + ) + descriptions = " ".join(event.description for event in res.events) + assert "missing-package" in descriptions + assert "another-missing" in descriptions assert len(res.events) == 3 diff --git a/test/unit/plugin/test_pcie_collector.py b/test/unit/plugin/test_pcie_collector.py index 6aabc5c0..8bae8d7e 100644 --- a/test/unit/plugin/test_pcie_collector.py +++ b/test/unit/plugin/test_pcie_collector.py @@ -27,8 +27,11 @@ import pytest +from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel +from nodescraper.plugins.inband.pcie.collector_args import PcieCollectorArgs from nodescraper.plugins.inband.pcie.pcie_collector import PcieCollector +from nodescraper.plugins.inband.pcie.pcie_data import PcieCfgSpace @pytest.fixture @@ -109,3 +112,120 @@ def test_log_pcie_artifacts_includes_lspci_pp_d(collector): artifact for artifact in collector.result.artifacts if artifact.filename == "lspci_pp_d.txt" ) assert lspci_pp_d.contents == "0001:00:01.1/0001:00:02.0/0001:00:03.0" + + +# --- ESXi ------------------------------------------------------------------- + +# esxcli hardware pci list: bare BDF header lines, then indented fields. +ESXCLI_PCI_LIST = ( + "0000:05:00.0\n" + " Device ID: 0x75a3\n" # PF + "0000:15:00.0\n" + " Device ID: 0x0000744C\n" # padded + uppercase -> 0x744c + "0000:05:02.0\n" + " Device ID: 0x75b3\n" # VF + "0000:99:00.0\n" + " SubDevice ID: 0x75a3\n" # must NOT be treated as a Device ID + " Device ID: 0xdead\n" # no match +) + +# lspci -e: " " header lines, then hex rows. +LSPCI_E_BLOB = ( + "0000:05:00.0 Processing accelerators: AMD\n" + "00: 12 34 56 78\n" + "10: 9a bc de f0\n" + "0000:05:02.0 Processing accelerators: AMD VF\n" + "00: aa bb cc dd\n" +) + + +def test_get_gpu_vf_bdfs_esxi_matches_pf_and_vf(collector): + """PF/VF BDFs are selected by matching the expected device IDs.""" + collector._run_os_cmd = MagicMock(return_value=ESXCLI_PCI_LIST) + pf, vf = collector._get_gpu_vf_bdfs_esxi(0x75A3, 0x75B3) + assert pf == ["0000:05:00.0"] + assert vf == ["0000:05:02.0"] + + +def test_get_gpu_vf_bdfs_esxi_case_and_padding(collector): + """A padded/uppercase Device ID ("0x0000744C") matches the expected 0x744c.""" + collector._run_os_cmd = MagicMock(return_value=ESXCLI_PCI_LIST) + pf, vf = collector._get_gpu_vf_bdfs_esxi(0x744C, None) + assert pf == ["0000:15:00.0"] + assert vf == [] + + +def test_get_gpu_vf_bdfs_esxi_both_none_short_circuits(collector): + """With no expected device IDs, esxcli is not even queried.""" + collector._run_os_cmd = MagicMock(return_value=ESXCLI_PCI_LIST) + pf, vf = collector._get_gpu_vf_bdfs_esxi(None, None) + assert (pf, vf) == ([], []) + collector._run_os_cmd.assert_not_called() + + +def test_get_gpu_vf_bdfs_esxi_ignores_unparseable_id(collector): + """A non-hex Device ID value is skipped rather than raising.""" + collector._run_os_cmd = MagicMock( + return_value="0000:05:00.0\n Device ID: N/A\n0000:05:02.0\n Device ID: 0x75a3\n" + ) + pf, vf = collector._get_gpu_vf_bdfs_esxi(0x75A3, None) + assert pf == ["0000:05:02.0"] + + +def test_get_all_cfg_space_esxi_splits_by_bdf(collector): + """lspci -e is split into {bdf: hex-rows} and saved as an artifact.""" + collector._run_os_cmd = MagicMock(return_value=LSPCI_E_BLOB) + cfg = collector._get_all_cfg_space_esxi() + assert set(cfg) == {"0000:05:00.0", "0000:05:02.0"} + assert cfg["0000:05:00.0"] == "00: 12 34 56 78\n10: 9a bc de f0" + assert any(a.filename == "lspci_e.txt" for a in collector.result.artifacts) + + +def test_get_all_cfg_space_esxi_empty(collector): + """No lspci output -> empty mapping.""" + collector._run_os_cmd = MagicMock(return_value="") + assert collector._get_all_cfg_space_esxi() == {} + + +def test_get_pcie_data_esxi_no_bdfs_warns(collector): + """When no GPU/VF BDFs match, a warning is logged and None returned.""" + collector._get_gpu_vf_bdfs_esxi = MagicMock(return_value=([], [])) + assert collector._get_pcie_data_esxi(0x75A3, None) is None + assert any("No GPU/VF BDFs" in e.description for e in collector.result.events) + + +def test_get_pcie_data_esxi_empty_cfg_errors(collector): + """BDFs found but no config space dump -> ERROR status, None.""" + collector._get_gpu_vf_bdfs_esxi = MagicMock(return_value=(["0000:05:00.0"], [])) + collector._get_all_cfg_space_esxi = MagicMock(return_value={}) + assert collector._get_pcie_data_esxi(0x75A3, None) is None + assert collector.result.status == ExecutionStatus.ERROR + + +def test_get_pcie_data_esxi_builds_model(collector): + """PF/VF BDFs present in the cfg dump are parsed into the PcieDataModel.""" + collector._get_gpu_vf_bdfs_esxi = MagicMock(return_value=(["0000:05:00.0"], ["0000:05:02.0"])) + collector._get_all_cfg_space_esxi = MagicMock( + return_value={"0000:05:00.0": "pf-hex", "0000:05:02.0": "vf-hex"} + ) + collector._cfg_space_from_hex = MagicMock(return_value=PcieCfgSpace()) + + data = collector._get_pcie_data_esxi(0x75A3, 0x75B3) + assert list(data.pcie_cfg_space) == ["0000:05:00.0"] + assert list(data.vf_pcie_cfg_space) == ["0000:05:02.0"] + collector._cfg_space_from_hex.assert_any_call("pf-hex", "0000:05:00.0") + collector._cfg_space_from_hex.assert_any_call("vf-hex", "0000:05:02.0") + + +def test_cfg_space_from_hex_too_short_logs_error(collector): + """Fewer than 64 parsed bytes logs an error (short/truncated dump).""" + collector._cfg_space_from_hex("00: 12 34 56 78", "0000:05:00.0") + assert any("not the expected length" in e.description for e in collector.result.events) + + +def test_collect_data_threads_devid_from_args(collector): + """collect_data forwards args.devid_ep / devid_ep_vf into the ESXi resolver.""" + collector._get_pcie_data = MagicMock(return_value=None) + args = PcieCollectorArgs(devid_ep=0x75A3, devid_ep_vf=0x75B3) + collector.collect_data(args) + collector._get_pcie_data.assert_called_once_with(None, 0x75A3, 0x75B3) diff --git a/test/unit/plugin/test_process_analyzer.py b/test/unit/plugin/test_process_analyzer.py index e7bbb86c..f3efba9e 100644 --- a/test/unit/plugin/test_process_analyzer.py +++ b/test/unit/plugin/test_process_analyzer.py @@ -26,6 +26,7 @@ import copy import pytest +from pydantic import ValidationError from nodescraper.enums.eventcategory import EventCategory from nodescraper.enums.eventpriority import EventPriority @@ -38,7 +39,6 @@ @pytest.fixture def model_obj(): return ProcessDataModel( - kfd_process=0, cpu_usage=10, processes=[ ("top", "10.0"), @@ -52,7 +52,7 @@ def model_obj(): @pytest.fixture def config(): - return {"max_kfd_processes": 0, "max_cpu_usage": 40} + return {"max_cpu_usage": 40} @pytest.fixture @@ -60,10 +60,28 @@ def analyzer(system_info): return ProcessAnalyzer(system_info=system_info) +def test_process_data_model_ignores_removed_kfd_process_input(): + model = ProcessDataModel(kfd_process=1) + + assert "kfd_process" not in model.model_dump() + + +def test_process_analyzer_args_rejects_removed_kfd_limit(): + with pytest.raises(ValidationError): + ProcessAnalyzerArgs(max_kfd_processes=1) + + +def test_process_analyzer_checks_only_cpu_usage(analyzer): + data = ProcessDataModel(kfd_process=99, cpu_usage=10, processes=[]) + + result = analyzer.analyze_data(data, ProcessAnalyzerArgs(max_cpu_usage=40)) + + assert result.status == ExecutionStatus.OK + assert result.events == [] + + def test_nominal_with_config(analyzer, model_obj, config): - args = ProcessAnalyzerArgs( - max_kfd_processes=config["max_kfd_processes"], max_cpu_usage=config["max_cpu_usage"] - ) + args = ProcessAnalyzerArgs(max_cpu_usage=config["max_cpu_usage"]) result = analyzer.analyze_data(model_obj, args) assert result.status == ExecutionStatus.OK assert len(result.events) == 0 @@ -75,23 +93,9 @@ def test_nominal_no_config(analyzer, model_obj): assert len(result.events) == 0 -def test_error_kfd_process(analyzer, model_obj, config): - modified_model_obj = copy.deepcopy(model_obj) - modified_model_obj.kfd_process = 1 - args = ProcessAnalyzerArgs( - max_kfd_processes=config["max_kfd_processes"], max_cpu_usage=config["max_cpu_usage"] - ) - result = analyzer.analyze_data(modified_model_obj, args) - - assert result.status == ExecutionStatus.ERROR - for event in result.events: - assert event.category == EventCategory.OS.value - assert event.priority == EventPriority.CRITICAL - - def test_error_cpu_usage(analyzer, model_obj, config): modified_model_obj = copy.deepcopy(model_obj) - args = ProcessAnalyzerArgs(max_kfd_processes=config["max_kfd_processes"], max_cpu_usage=5) + args = ProcessAnalyzerArgs(max_cpu_usage=5) result = analyzer.analyze_data(modified_model_obj, args) assert result.status == ExecutionStatus.ERROR diff --git a/test/unit/plugin/test_process_collector.py b/test/unit/plugin/test_process_collector.py index cae8d6f0..beff7869 100644 --- a/test/unit/plugin/test_process_collector.py +++ b/test/unit/plugin/test_process_collector.py @@ -23,17 +23,38 @@ # SOFTWARE. # ############################################################################### +import logging +import time from unittest.mock import MagicMock import pytest +from pydantic import ValidationError +from nodescraper.enums.eventcategory import EventCategory from nodescraper.enums.executionstatus import ExecutionStatus from nodescraper.enums.systeminteraction import SystemInteractionLevel from nodescraper.interfaces.task import SystemCompatibilityError from nodescraper.models.systeminfo import OSFamily +from nodescraper.plugins.inband.process import ( + process_collector as process_collector_module, +) +from nodescraper.plugins.inband.process.collector_args import ProcessCollectorArgs from nodescraper.plugins.inband.process.process_collector import ProcessCollector from nodescraper.plugins.inband.process.processdata import ProcessDataModel +PROC_STAT_1 = "cpu 100 0 0 900 0 0 0 0 0 0\n" +PROC_STAT_2 = "cpu 200 0 0 1800 0 0 0 0 0 0\n" +PROC_DUMP_1 = ( + "__SAMPLER__:99999\n" + "1000|1000 (worker) S 0 0 0 0 -1 0 0 0 0 0 5000 6000\n" + "1|1 (systemd) S 0 0 0 0 -1 0 0 0 0 0 5000 6000\n" +) +PROC_DUMP_2 = ( + "__SAMPLER__:99999\n" + "1000|1000 (worker) S 0 0 0 0 -1 0 0 0 0 0 5100 6000\n" + "1|1 (systemd) S 0 0 0 0 -1 0 0 0 0 0 5000 6000\n" +) + @pytest.fixture def collector(system_info, conn_mock): @@ -44,37 +65,114 @@ def collector(system_info, conn_mock): ) -def test_run_linux(collector, conn_mock): - collector.system_info.os_family = OSFamily.LINUX - conn_mock.run_command.side_effect = [ - MagicMock( - exit_code=0, - stdout="PID PROCESS NAME GPU(s) VRAM USED SDMA USED CU OCCUPANCY\n8246 TransferBench 8 2267283456 0 0", - stderr="", - ), - MagicMock( - exit_code=0, - stdout="%Cpu(s): 0.1 us, 0.1 sy, 0.0 ni, 90.0 id", - stderr="", - ), - MagicMock( - exit_code=0, - stdout="356817 user 20 0 32112 14196 10556 R 10.0 0.0 0:00.07 top\n" - "1 root 20 0 166596 11916 8316 S 0.0 0.0 1:32.14 systemd", - stderr="", - ), - ] +def test_process_read_event_uses_shared_category(): + assert EventCategory.PROCESS_READ.value == "PROCESS_READ" + + +def test_parse_aggregate_cpu_from_proc_stat(): + proc_stat = "cpu0 1 2 3 4 5 6 7 8\ncpu 100 0 0 900 10 0 0 0 0 0\n" + + assert process_collector_module._parse_aggregate_cpu_from_proc_stat(proc_stat) == (1010, 910) + + +def test_parse_aggregate_cpu_excludes_guest_fields(): + proc_stat = "cpu 100 20 30 800 10 5 4 1 40 10\n" + + assert process_collector_module._parse_aggregate_cpu_from_proc_stat(proc_stat) == (970, 810) + + +def test_collector_args_reject_nonpositive_sample_interval(): + with pytest.raises(ValidationError): + ProcessCollectorArgs(sample_interval_seconds=0) + + +def test_global_non_idle_percent_uses_jiffy_deltas(): + assert process_collector_module._global_non_idle_percent(1000, 900, 2000, 1800) == 10.0 + + +def test_parse_proc_pid_stat_handles_process_names_with_spaces(): + stat_line = "1000 (worker process) S 0 0 0 0 -1 0 0 0 0 0 5000 6000" + + assert process_collector_module._parse_proc_pid_stat(stat_line) == (1000, 11000) + + +def test_parse_proc_pid_stat_handles_closing_parenthesis_in_process_name(): + stat_line = "1000 (worker) process) S 0 0 0 0 -1 0 0 0 0 0 5000 6000" + + assert process_collector_module._parse_proc_pid_stat(stat_line) == (1000, 11000) + + +def test_parse_proc_stat_dump_returns_jiffies_and_sampler_pid(): + dump = ( + "__SAMPLER__:99999\n" + "1000|1000 (worker process) S 0 0 0 0 -1 0 0 0 0 0 5000 6000\n" + "invalid line\n" + ) + + assert process_collector_module._parse_proc_stat_dump(dump) == ({1000: 11000}, {99999}) + + +def test_top_process_cpu_shares_ranks_deltas_and_excludes_sampler(): + first_sample = {1: 100, 2: 200, 99: 0} + second_sample = {1: 150, 2: 400, 99: 500} + + assert process_collector_module._top_process_cpu_shares( + first_sample, + second_sample, + total_delta=1000, + top_n=2, + exclude_pids={99}, + ) == [(2, 20.0), (1, 5.0)] + + +def test_top_process_cpu_shares_excludes_processes_missing_from_current_sample(): + assert process_collector_module._top_process_cpu_shares( + sample1={1: 100, 2: 200}, + sample2={2: 400}, + total_delta=1000, + top_n=2, + exclude_pids=set(), + ) == [(2, 20.0)] + + +def test_parse_comm_dump_maps_process_names_by_pid(): + assert process_collector_module._parse_comm_dump("1000:worker\n1:systemd\ninvalid\n") == { + 1000: "worker", + 1: "systemd", + } + + +def test_run_linux_collects_cpu_and_processes_from_procfs(collector, conn_mock, monkeypatch): + proc_stat_calls = 0 + proc_dump_calls = 0 + + def run_command(command, **_kwargs): + nonlocal proc_stat_calls, proc_dump_calls + if command == "cat /proc/stat": + proc_stat_calls += 1 + stdout = PROC_STAT_1 if proc_stat_calls == 1 else PROC_STAT_2 + elif "for f in /proc/" in command and "__SAMPLER__" in command: + proc_dump_calls += 1 + stdout = PROC_DUMP_1 if proc_dump_calls == 1 else PROC_DUMP_2 + elif "cat /proc/$p/comm" in command: + stdout = "1000:\n1:systemd\n" + else: + raise AssertionError(f"unexpected command: {command}") + return MagicMock(exit_code=0, stdout=stdout, stderr="", command=command) + + conn_mock.run_command.side_effect = run_command + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + result, data = collector.collect_data( + ProcessCollectorArgs(top_n_process=2, sample_interval_seconds=0.01) + ) - result, data = collector.collect_data() assert result.status == ExecutionStatus.OK assert data == ProcessDataModel( - kfd_process=1, - cpu_usage=10, - processes=[ - ("top", "10.0"), - ("systemd", "0.0"), - ], + cpu_usage=10.0, + processes=[("pid_1000", "10.0"), ("systemd", "0.0")], ) + assert all("__SAMPLER__" not in artifact.command for artifact in result.artifacts) def test_unsupported_platform(system_info, conn_mock): @@ -98,3 +196,49 @@ def test_exit_failure(collector, conn_mock): result, data = collector.collect_data() assert result.status == ExecutionStatus.EXECUTION_FAILURE assert data is None + + +@pytest.mark.parametrize( + ("failure_index", "expected_warning"), + [ + (0, "first aggregate CPU sample"), + (1, "first process CPU sample"), + (2, "second aggregate CPU sample"), + (3, "second process CPU sample"), + ], +) +def test_procfs_command_failure_logs_warning( + collector, conn_mock, monkeypatch, caplog, failure_index, expected_warning +): + responses = [ + MagicMock(exit_code=0, stdout=PROC_STAT_1, stderr=""), + MagicMock(exit_code=0, stdout=PROC_DUMP_1, stderr=""), + MagicMock(exit_code=0, stdout=PROC_STAT_2, stderr=""), + MagicMock(exit_code=0, stdout=PROC_DUMP_2, stderr=""), + ] + responses[failure_index].exit_code = 1 + conn_mock.run_command.side_effect = responses + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + with caplog.at_level(logging.WARNING): + result, data = collector.collect_data() + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert data is None + assert expected_warning in caplog.text + + +def test_invalid_proc_stat_returns_failure_and_logs_os_event(collector, conn_mock, monkeypatch): + conn_mock.run_command.side_effect = [ + MagicMock(exit_code=0, stdout="not proc stat\n", stderr=""), + MagicMock(exit_code=0, stdout="__SAMPLER__:99999\n", stderr=""), + MagicMock(exit_code=0, stdout=PROC_STAT_2, stderr=""), + MagicMock(exit_code=0, stdout="__SAMPLER__:99999\n", stderr=""), + ] + monkeypatch.setattr(time, "sleep", lambda _seconds: None) + + result, data = collector.collect_data(ProcessCollectorArgs(sample_interval_seconds=0.01)) + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert data is None + assert any(event.category == EventCategory.OS.value for event in result.events) diff --git a/test/unit/plugin/test_rdma_analyzer.py b/test/unit/plugin/test_rdma_analyzer.py index 1a3804a5..ca1531e3 100644 --- a/test/unit/plugin/test_rdma_analyzer.py +++ b/test/unit/plugin/test_rdma_analyzer.py @@ -37,6 +37,7 @@ Cx7RdmaStatistics, PollaraRdmaStatistics, RdmaDataModel, + RdmaDevice, RdmaLink, RdmaStatistics, RdmaVendorStatistics, @@ -160,6 +161,70 @@ def test_empty_statistics(rdma_analyzer): assert result.message == "No RDMA devices found" +def test_roce_firmware_exact_match(rdma_analyzer): + model = RdmaDataModel( + dev_list=[ + RdmaDevice( + device="bnxt_re0", + transport="RoCE", + firmware_version="238.1.169.0", + ) + ] + ) + args = RdmaAnalyzerArgs(expected_nic_firmware="238.1.169.0") + + result = rdma_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_roce_firmware_regex_matches_supported_formats(rdma_analyzer): + pattern = r"^[0-9]+(?:\.[0-9]+){2,}(?:-[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$" + for firmware_version in ("1.117.5-a-77", "28.35.1012", "238.1.168.0"): + model = RdmaDataModel( + dev_list=[ + RdmaDevice( + device="bnxt_re0", + transport="RoCE", + firmware_version=firmware_version, + ) + ] + ) + + result = rdma_analyzer.analyze_data( + model, + RdmaAnalyzerArgs(expected_nic_firmware=pattern), + ) + + assert result.status == ExecutionStatus.OK + assert not result.events + + +def test_infini_band_firmware_mismatch_is_reported(rdma_analyzer): + model = RdmaDataModel( + dev_list=[ + RdmaDevice( + device="mlx5_0", + node_type="CA", + transport="InfiniBand", + firmware_version="32.40.1000", + ) + ] + ) + args = RdmaAnalyzerArgs(expected_nic_firmware="32.50.1000") + + result = rdma_analyzer.analyze_data(model, args) + + assert result.status == ExecutionStatus.WARNING + assert any("firmware" in event.description.lower() for event in result.events) + firmware_event = next( + event for event in result.events if "firmware" in event.description.lower() + ) + assert firmware_event.data["policy"] == {"expected_nic_firmware": "32.50.1000"} + assert "expected_nic_firmware" in firmware_event.data["reason"] + + def test_multiple_interfaces_with_errors(rdma_analyzer, example_stat_dicts): stats_multi_errors = _build_stats(example_stat_dicts) stats_multi_errors[0].vendor_statistics.req_rx_pkt_seq_err = 15 diff --git a/test/unit/plugin/test_rdma_collector.py b/test/unit/plugin/test_rdma_collector.py index 38d162ca..70a8a326 100644 --- a/test/unit/plugin/test_rdma_collector.py +++ b/test/unit/plugin/test_rdma_collector.py @@ -80,6 +80,44 @@ def test_collect_success(collector, conn_mock, rdma_link_output, rdma_statistic_ assert data.link_list[0].ifname == "ionic_0" # netdev is cross-referenced from link data onto statistics assert data.statistic_list[0].netdev == "benic8p1" + link_event = next( + event + for event in res.events + if event.description == "Collected 4 RDMA links from 'rdma link -j'" + ) + assert link_event.data["links"][0] == { + "ifname": "ionic_0", + "port": 1, + "state": "ACTIVE", + "physical_state": "LINK_UP", + "netdev": "benic8p1", + } + + +def test_collect_logs_text_link_state(collector, conn_mock): + """Text 'rdma link' output records link state on the collection event.""" + collector.system_info.os_family = OSFamily.LINUX + conn_mock.run_command.side_effect = [ + CommandArtifact(exit_code=0, stdout="[]", stderr="", command="rdma link -j"), + CommandArtifact(exit_code=0, stdout="[]", stderr="", command="rdma statistic -j"), + CommandArtifact(exit_code=0, stdout="", stderr="", command="rdma dev"), + CommandArtifact(exit_code=0, stdout=RDMA_LINK_OUTPUT, stderr="", command="rdma link"), + ] + res, data = collector.collect_data() + assert res.status == ExecutionStatus.OK + assert data is not None + link_event = next( + event + for event in res.events + if event.description == "Collected 2 RDMA links from 'rdma link'" + ) + assert link_event.data["links"][0] == { + "device": "rocep9s0", + "port": 1, + "state": "DOWN", + "physical_state": "POLLING", + "netdev": "benic8p1", + } def test_collect_both_commands_fail(collector, conn_mock): diff --git a/test/unit/plugin/test_redfish_endpoint_analyzer.py b/test/unit/plugin/test_redfish_endpoint_analyzer.py index 0f014245..1d389b05 100644 --- a/test/unit/plugin/test_redfish_endpoint_analyzer.py +++ b/test/unit/plugin/test_redfish_endpoint_analyzer.py @@ -1,209 +1,269 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import pytest - -from nodescraper.enums import ExecutionStatus -from nodescraper.plugins.ooband.redfish_endpoint import ( - RedfishEndpointAnalyzer, - RedfishEndpointAnalyzerArgs, - RedfishEndpointDataModel, -) -from nodescraper.plugins.ooband.redfish_endpoint.endpoint_analyzer import ( - _check_constraint, - _get_by_path, -) - - -@pytest.fixture -def redfish_endpoint_analyzer(system_info): - return RedfishEndpointAnalyzer(system_info=system_info) - - -def test_get_by_path_empty_returns_obj(): - obj = {"a": 1} - assert _get_by_path(obj, "") == obj - assert _get_by_path(obj, " ") == obj - - -def test_get_by_path_single_key(): - assert _get_by_path({"x": 42}, "x") == 42 - assert _get_by_path({"Status": {"Health": "OK"}}, "Status") == {"Health": "OK"} - - -def test_get_by_path_nested_slash(): - obj = {"Status": {"Health": "OK", "State": "Enabled"}} - assert _get_by_path(obj, "Status/Health") == "OK" - assert _get_by_path(obj, "Status/State") == "Enabled" - - -def test_get_by_path_list_index(): - obj = {"PowerControl": [{"PowerConsumedWatts": 100}, {"PowerConsumedWatts": 200}]} - assert _get_by_path(obj, "PowerControl/0/PowerConsumedWatts") == 100 - assert _get_by_path(obj, "PowerControl/1/PowerConsumedWatts") == 200 - - -def test_get_by_path_missing_returns_none(): - assert _get_by_path({"a": 1}, "b") is None - assert _get_by_path({"a": {"b": 2}}, "a/c") is None - assert _get_by_path(None, "a") is None - - -def test_get_by_path_invalid_list_index(): - obj = {"list": [1, 2, 3]} - assert _get_by_path(obj, "list/10") is None - assert _get_by_path(obj, "list/xyz") is None - - -def test_check_constraint_eq_pass(): - ok, msg = _check_constraint("On", {"eq": "On"}) - assert ok is True - - -def test_check_constraint_eq_fail(): - ok, msg = _check_constraint("Off", {"eq": "On"}) - assert ok is False - assert "On" in msg and "Off" in msg - - -def test_check_constraint_min_max_pass(): - ok, _ = _check_constraint(50, {"min": 0, "max": 100}) - assert ok is True - ok, _ = _check_constraint(0, {"min": 0}) - assert ok is True - ok, _ = _check_constraint(100, {"max": 100}) - assert ok is True - - -def test_check_constraint_min_fail(): - ok, msg = _check_constraint(10, {"min": 20}) - assert ok is False - assert "below min" in msg or "20" in msg - - -def test_check_constraint_max_fail(): - ok, msg = _check_constraint(150, {"max": 100}) - assert ok is False - assert "above max" in msg or "100" in msg - - -def test_check_constraint_any_of_pass(): - ok, _ = _check_constraint("OK", {"anyOf": ["OK", "Warning"]}) - assert ok is True - ok, _ = _check_constraint("Warning", {"anyOf": ["OK", "Warning"]}) - assert ok is True - - -def test_check_constraint_any_of_fail(): - ok, msg = _check_constraint("Critical", {"anyOf": ["OK", "Warning"]}) - assert ok is False - assert "any of" in msg or "OK" in msg - - -def test_check_constraint_literal_match(): - ok, _ = _check_constraint("On", "On") - assert ok is True - ok, msg = _check_constraint("Off", "On") - assert ok is False - - -def test_redfish_endpoint_analyzer_no_checks(redfish_endpoint_analyzer): - data = RedfishEndpointDataModel(responses={"/redfish/v1": {}}) - result = redfish_endpoint_analyzer.analyze_data(data, args=None) - assert result.status == ExecutionStatus.OK - assert result.message == "No checks configured" - - -def test_redfish_endpoint_analyzer_empty_checks(redfish_endpoint_analyzer): - data = RedfishEndpointDataModel(responses={"/redfish/v1": {"Status": {"Health": "OK"}}}) - result = redfish_endpoint_analyzer.analyze_data( - data, args=RedfishEndpointAnalyzerArgs(checks={}) - ) - assert result.status == ExecutionStatus.OK - assert result.message == "No checks configured" - - -def test_redfish_endpoint_analyzer_all_pass(redfish_endpoint_analyzer): - data = RedfishEndpointDataModel( - responses={ - "/redfish/v1/Systems/1": {"Status": {"Health": "OK"}, "PowerState": "On"}, - } - ) - args = RedfishEndpointAnalyzerArgs( - checks={ - "/redfish/v1/Systems/1": { - "Status/Health": {"anyOf": ["OK", "Warning"]}, - "PowerState": "On", - }, - } - ) - result = redfish_endpoint_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.OK - assert result.message == "All Redfish endpoint checks passed" - - -def test_redfish_endpoint_analyzer_one_fail(redfish_endpoint_analyzer): - data = RedfishEndpointDataModel( - responses={ - "/redfish/v1/Systems/1": {"Status": {"Health": "Critical"}}, - } - ) - args = RedfishEndpointAnalyzerArgs( - checks={ - "/redfish/v1/Systems/1": {"Status/Health": {"anyOf": ["OK", "Warning"]}}, - } - ) - result = redfish_endpoint_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.ERROR - assert "check(s) failed" in result.message - - -def test_redfish_endpoint_analyzer_uri_not_in_responses(redfish_endpoint_analyzer): - data = RedfishEndpointDataModel(responses={"/redfish/v1": {}}) - args = RedfishEndpointAnalyzerArgs( - checks={ - "/redfish/v1/Systems/1": {"Status/Health": "OK"}, - } - ) - result = redfish_endpoint_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.ERROR - assert "check(s) failed" in result.message or "failed" in result.message - - -def test_redfish_endpoint_analyzer_wildcard_applies_to_all_bodies(redfish_endpoint_analyzer): - data = RedfishEndpointDataModel( - responses={ - "/redfish/v1/Chassis/1": {"Status": {"Health": "OK"}}, - "/redfish/v1/Chassis/2": {"Status": {"Health": "OK"}}, - } - ) - args = RedfishEndpointAnalyzerArgs( - checks={ - "*": {"Status/Health": {"anyOf": ["OK", "Warning"]}}, - } - ) - result = redfish_endpoint_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.OK - assert result.message == "All Redfish endpoint checks passed" +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import pytest + +from nodescraper.enums import ExecutionStatus +from nodescraper.plugins.ooband.redfish_endpoint import ( + RedfishEndpointAnalyzer, + RedfishEndpointAnalyzerArgs, + RedfishEndpointDataModel, +) +from nodescraper.plugins.ooband.redfish_endpoint.endpoint_analyzer import ( + _check_constraint, + _get_by_path, +) + + +@pytest.fixture +def redfish_endpoint_analyzer(system_info): + return RedfishEndpointAnalyzer(system_info=system_info) + + +def test_get_by_path_empty_returns_obj(): + obj = {"a": 1} + assert _get_by_path(obj, "") == obj + assert _get_by_path(obj, " ") == obj + + +def test_get_by_path_single_key(): + assert _get_by_path({"x": 42}, "x") == 42 + assert _get_by_path({"Status": {"Health": "OK"}}, "Status") == {"Health": "OK"} + + +def test_get_by_path_nested_slash(): + obj = {"Status": {"Health": "OK", "State": "Enabled"}} + assert _get_by_path(obj, "Status/Health") == "OK" + assert _get_by_path(obj, "Status/State") == "Enabled" + + +def test_get_by_path_list_index(): + obj = {"PowerControl": [{"PowerConsumedWatts": 100}, {"PowerConsumedWatts": 200}]} + assert _get_by_path(obj, "PowerControl/0/PowerConsumedWatts") == 100 + assert _get_by_path(obj, "PowerControl/1/PowerConsumedWatts") == 200 + + +def test_get_by_path_missing_returns_none(): + assert _get_by_path({"a": 1}, "b") is None + assert _get_by_path({"a": {"b": 2}}, "a/c") is None + assert _get_by_path(None, "a") is None + + +def test_get_by_path_invalid_list_index(): + obj = {"list": [1, 2, 3]} + assert _get_by_path(obj, "list/10") is None + assert _get_by_path(obj, "list/xyz") is None + + +def test_check_constraint_eq_pass(): + ok, msg = _check_constraint("On", {"eq": "On"}) + assert ok is True + + +def test_check_constraint_eq_fail(): + ok, msg = _check_constraint("Off", {"eq": "On"}) + assert ok is False + assert "On" in msg and "Off" in msg + + +def test_check_constraint_min_max_pass(): + ok, _ = _check_constraint(50, {"min": 0, "max": 100}) + assert ok is True + ok, _ = _check_constraint(0, {"min": 0}) + assert ok is True + ok, _ = _check_constraint(100, {"max": 100}) + assert ok is True + + +def test_check_constraint_min_fail(): + ok, msg = _check_constraint(10, {"min": 20}) + assert ok is False + assert "below min" in msg or "20" in msg + + +def test_check_constraint_max_fail(): + ok, msg = _check_constraint(150, {"max": 100}) + assert ok is False + assert "above max" in msg or "100" in msg + + +def test_check_constraint_any_of_pass(): + ok, _ = _check_constraint("OK", {"anyOf": ["OK", "Warning"]}) + assert ok is True + ok, _ = _check_constraint("Warning", {"anyOf": ["OK", "Warning"]}) + assert ok is True + + +def test_check_constraint_any_of_fail(): + ok, msg = _check_constraint("Critical", {"anyOf": ["OK", "Warning"]}) + assert ok is False + assert "any of" in msg or "OK" in msg + + +def test_check_constraint_literal_match(): + ok, _ = _check_constraint("On", "On") + assert ok is True + ok, msg = _check_constraint("Off", "On") + assert ok is False + + +def test_check_constraint_regex_pass(): + ok, _ = _check_constraint("123e4567-e89b-12d3-a456-426614174000", {"regex": r"^[0-9a-f-]{36}$"}) + assert ok is True + ok, _ = _check_constraint("SN1234", {"regex": r"\d"}) + assert ok is True + + +def test_check_constraint_regex_fail(): + ok, msg = _check_constraint("000000000000", {"regex": r"^(?!0+$).+$"}) + assert ok is False + assert "expected match" in msg + + +def test_check_constraint_regex_non_string_fails(): + for actual in (None, 42, {"a": 1}): + ok, msg = _check_constraint(actual, {"regex": ".*"}) + assert ok is False + assert "expected string" in msg + + +def test_check_constraint_regex_invalid_pattern_fails(): + ok, msg = _check_constraint("abc", {"regex": "("}) + assert ok is False + assert "invalid regex" in msg + + +def test_check_constraint_exists(): + for present in ("x", 0, False, {}, {"@odata.id": "/redfish/v1/Chassis/1/Sensors"}): + assert _check_constraint(present, {"exists": True})[0] is True + assert _check_constraint(present, {"exists": False})[0] is False + assert _check_constraint(None, {"exists": True})[0] is False + assert _check_constraint(None, {"exists": False})[0] is True + + +def test_redfish_endpoint_analyzer_regex_and_exists(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel( + responses={ + "/redfish/v1/Chassis/1": { + "UUID": "123e4567-e89b-12d3-a456-426614174000", + "SerialNumber": "000000000000", + "Sensors": {"@odata.id": "/redfish/v1/Chassis/1/Sensors"}, + }, + } + ) + args = RedfishEndpointAnalyzerArgs( + checks={ + "/redfish/v1/Chassis/1": { + "UUID": {"regex": r"^[0-9a-fA-F-]{36}$"}, + "SerialNumber": {"regex": r"^(?!0+$).+$"}, + "Sensors": {"exists": True}, + "Assembly": {"exists": True}, + }, + } + ) + result = redfish_endpoint_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.ERROR + failures = result.events[-1].data["failures"] + assert sorted(f["path"] for f in failures) == ["Assembly", "SerialNumber"] + + +def test_redfish_endpoint_analyzer_no_checks(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel(responses={"/redfish/v1": {}}) + result = redfish_endpoint_analyzer.analyze_data(data, args=None) + assert result.status == ExecutionStatus.OK + assert result.message == "No checks configured" + + +def test_redfish_endpoint_analyzer_empty_checks(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel(responses={"/redfish/v1": {"Status": {"Health": "OK"}}}) + result = redfish_endpoint_analyzer.analyze_data( + data, args=RedfishEndpointAnalyzerArgs(checks={}) + ) + assert result.status == ExecutionStatus.OK + assert result.message == "No checks configured" + + +def test_redfish_endpoint_analyzer_all_pass(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel( + responses={ + "/redfish/v1/Systems/1": {"Status": {"Health": "OK"}, "PowerState": "On"}, + } + ) + args = RedfishEndpointAnalyzerArgs( + checks={ + "/redfish/v1/Systems/1": { + "Status/Health": {"anyOf": ["OK", "Warning"]}, + "PowerState": "On", + }, + } + ) + result = redfish_endpoint_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.OK + assert result.message == "All Redfish endpoint checks passed" + + +def test_redfish_endpoint_analyzer_one_fail(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel( + responses={ + "/redfish/v1/Systems/1": {"Status": {"Health": "Critical"}}, + } + ) + args = RedfishEndpointAnalyzerArgs( + checks={ + "/redfish/v1/Systems/1": {"Status/Health": {"anyOf": ["OK", "Warning"]}}, + } + ) + result = redfish_endpoint_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.ERROR + assert "check(s) failed" in result.message + + +def test_redfish_endpoint_analyzer_uri_not_in_responses(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel(responses={"/redfish/v1": {}}) + args = RedfishEndpointAnalyzerArgs( + checks={ + "/redfish/v1/Systems/1": {"Status/Health": "OK"}, + } + ) + result = redfish_endpoint_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.ERROR + assert "check(s) failed" in result.message or "failed" in result.message + + +def test_redfish_endpoint_analyzer_wildcard_applies_to_all_bodies(redfish_endpoint_analyzer): + data = RedfishEndpointDataModel( + responses={ + "/redfish/v1/Chassis/1": {"Status": {"Health": "OK"}}, + "/redfish/v1/Chassis/2": {"Status": {"Health": "OK"}}, + } + ) + args = RedfishEndpointAnalyzerArgs( + checks={ + "*": {"Status/Health": {"anyOf": ["OK", "Warning"]}}, + } + ) + result = redfish_endpoint_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.OK + assert result.message == "All Redfish endpoint checks passed" diff --git a/test/unit/plugin/test_redfish_endpoint_collector.py b/test/unit/plugin/test_redfish_endpoint_collector.py index 501a0d8b..7f6852ab 100644 --- a/test/unit/plugin/test_redfish_endpoint_collector.py +++ b/test/unit/plugin/test_redfish_endpoint_collector.py @@ -1,498 +1,721 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from unittest.mock import MagicMock - -import pytest - -from nodescraper.connection.redfish import RedfishGetResult -from nodescraper.enums import EventCategory, ExecutionStatus -from nodescraper.plugins.ooband.redfish_endpoint import ( - RedfishEndpointCollector, - RedfishEndpointCollectorArgs, -) -from nodescraper.plugins.ooband.redfish_endpoint import endpoint_collector as ec - - -@pytest.fixture -def redfish_endpoint_collector(system_info, redfish_conn_mock): - return RedfishEndpointCollector( - system_info=system_info, - connection=redfish_conn_mock, - ) - - -def test_redfish_endpoint_collector_no_uris(redfish_endpoint_collector): - result, data = redfish_endpoint_collector.collect_data() - assert result.status == ExecutionStatus.NOT_RAN - assert "No collection mode configured" in result.message - assert data is None - - -def test_redfish_endpoint_collector_no_uris_with_args(redfish_endpoint_collector): - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=[]) - ) - assert result.status == ExecutionStatus.NOT_RAN - assert "No collection mode configured" in result.message - assert data is None - - -def test_redfish_endpoint_collector_one_uri_success(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"Name": "Root"}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) - ) - assert result.status == ExecutionStatus.OK - assert result.message == "Collected 1 Redfish endpoint(s)" - assert data is not None - assert data.responses["/redfish/v1"]["Name"] == "Root" - redfish_conn_mock.run_get.assert_called_once() - call_path = redfish_conn_mock.run_get.call_args[0][0] - assert call_path == "/redfish/v1" or call_path.strip("/") == "redfish/v1" - - -def test_redfish_endpoint_collector_uri_normalized_with_leading_slash( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1/Systems", - success=True, - data={"Members": []}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["redfish/v1/Systems"]) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert "/redfish/v1/Systems" in data.responses or "redfish/v1/Systems" in data.responses - - -def test_redfish_endpoint_collector_one_fail_no_success( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=False, - error="Connection refused", - status_code=None, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) - ) - assert result.status == ExecutionStatus.ERROR - assert result.message.startswith("No Redfish endpoints could be read") - assert data is None - assert len(result.events) >= 1 - assert any( - e.category == EventCategory.RUNTIME.value or "Redfish GET failed" in (e.description or "") - for e in result.events - ) - - -def test_redfish_endpoint_collector_mixed_success_fail( - redfish_endpoint_collector, redfish_conn_mock -): - def run_get_side_effect(path): - path_str = str(path) - if "Systems" in path_str: - return RedfishGetResult( - path=path_str if path_str.startswith("/") else "/" + path_str, - success=True, - data={"Id": "1"}, - status_code=200, - ) - return RedfishGetResult( - path=path_str if path_str.startswith("/") else "/" + path_str, - success=False, - error="Not Found", - status_code=404, - ) - - redfish_conn_mock.run_get.side_effect = run_get_side_effect - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(uris=["/redfish/v1/Systems", "/redfish/v1/Bad"]) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert len(data.responses) == 1 - keys = list(data.responses.keys()) - assert any("Systems" in k for k in keys) - assert list(data.responses.values())[0].get("Id") == "1" - - -def test_normalize_path_empty_or_invalid(): - api = "redfish/v1" - assert ec._normalize_path("", api) == "" - assert ec._normalize_path(None, api) == "" # type: ignore[arg-type] - assert ec._normalize_path(" ", api) == "" - - -def test_normalize_path_relative_path(): - api = "redfish/v1" - assert ec._normalize_path("/redfish/v1", api) == "/redfish/v1" - assert ec._normalize_path("/redfish/v1/", api) == "/redfish/v1" - assert ec._normalize_path("redfish/v1/Systems", api) == "/redfish/v1/Systems" - assert ec._normalize_path(" /redfish/v1/Chassis ", api) == "/redfish/v1/Chassis" - - -def test_normalize_path_full_url(): - api = "redfish/v1" - assert ec._normalize_path("https://host/redfish/v1/Systems", api) == "/redfish/v1/Systems" - assert ec._normalize_path("http://bmc/redfish/v1", api) == "/redfish/v1" - - -def test_normalize_path_outside_api_root(): - api = "redfish/v1" - assert ec._normalize_path("/other/root", api) == "" - assert ec._normalize_path("https://host/other/path", api) == "" - - -def test_extract_odata_ids_empty(): - assert ec._extract_odata_ids({}) == [] - assert ec._extract_odata_ids([]) == [] - assert ec._extract_odata_ids("x") == [] - - -def test_extract_odata_ids_single(): - assert ec._extract_odata_ids({"@odata.id": "/redfish/v1"}) == ["/redfish/v1"] - assert ec._extract_odata_ids({"@odata.id": "https://host/redfish/v1"}) == [ - "https://host/redfish/v1" - ] - - -def test_extract_odata_ids_members(): - body = { - "Members": [ - {"@odata.id": "/redfish/v1/Systems/1"}, - {"@odata.id": "/redfish/v1/Systems/2"}, - ] - } - assert set(ec._extract_odata_ids(body)) == {"/redfish/v1/Systems/1", "/redfish/v1/Systems/2"} - - -def test_extract_odata_ids_nested_and_members(): - body = { - "@odata.id": "/redfish/v1", - "Systems": {"@odata.id": "/redfish/v1/Systems", "Members": []}, - "Chassis": { - "Members": [{"@odata.id": "/redfish/v1/Chassis/1"}], - }, - } - ids = ec._extract_odata_ids(body) - assert "/redfish/v1" in ids - assert "/redfish/v1/Systems" in ids - assert "/redfish/v1/Chassis/1" in ids - - -def test_uris_from_args_none(): - assert ec._uris_from_args(None) == [] - - -def test_uris_from_args_empty(): - assert ec._uris_from_args(RedfishEndpointCollectorArgs(uris=[])) == [] - - -def test_uris_from_args_with_uris(): - args = RedfishEndpointCollectorArgs(uris=["/redfish/v1", "/redfish/v1/Systems"]) - assert ec._uris_from_args(args) == ["/redfish/v1", "/redfish/v1/Systems"] - - -def test_fetch_one_calls_run_get(): - conn = MagicMock() - conn.run_get.return_value = RedfishGetResult( - path="/redfish/v1", success=True, data={}, status_code=200 - ) - out = ec._fetch_one(conn, "/redfish/v1") - conn.run_get.assert_called_once_with("/redfish/v1") - assert out.success is True - assert out.path == "/redfish/v1" - - -def test_fetch_one_paged_calls_run_get_paged(): - conn = MagicMock() - conn.run_get_paged.return_value = RedfishGetResult( - path="/redfish/v1/Systems", - success=True, - data={"Members": []}, - status_code=200, - ) - out = ec._fetch_one_paged(conn, "/redfish/v1/Systems", max_pages=42) - conn.run_get_paged.assert_called_once_with("/redfish/v1/Systems", max_pages=42) - assert out.success is True - - -def test_run_redfish_get_paged_appends_artifact_when_enabled( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get_paged.return_value = RedfishGetResult( - path="/redfish/v1/Systems", - success=True, - data={"Members": []}, - status_code=200, - ) - redfish_endpoint_collector.result.artifacts.clear() - res = redfish_endpoint_collector._run_redfish_get_paged( - "/redfish/v1/Systems", max_pages=5, log_artifact=True - ) - redfish_conn_mock.run_get_paged.assert_called_once_with("/redfish/v1/Systems", max_pages=5) - assert res.success is True - assert len(redfish_endpoint_collector.result.artifacts) == 1 - - -def test_run_redfish_get_paged_skips_artifact_when_disabled( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get_paged.return_value = RedfishGetResult( - path="/x", - success=True, - data={"Members": []}, - status_code=200, - ) - redfish_endpoint_collector.result.artifacts.clear() - redfish_endpoint_collector._run_redfish_get_paged("/x", log_artifact=False) - assert len(redfish_endpoint_collector.result.artifacts) == 0 - - -def test_collect_follow_next_link_sequential_uses_run_get_paged( - redfish_endpoint_collector, redfish_conn_mock -): - redfish_conn_mock.run_get_paged.return_value = RedfishGetResult( - path="/redfish/v1/Systems", - success=True, - data={"Members": [{"@odata.id": "/redfish/v1/Systems/1"}], "Members@odata.count": 1}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs( - uris=["/redfish/v1/Systems"], - follow_next_link=True, - max_pages=50, - ) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert data.responses["/redfish/v1/Systems"]["Members@odata.count"] == 1 - redfish_conn_mock.run_get_paged.assert_called_once_with("/redfish/v1/Systems", max_pages=50) - redfish_conn_mock.run_get.assert_not_called() - - -def test_collect_follow_next_link_concurrent_uses_connection_copy( - redfish_endpoint_collector, redfish_conn_mock -): - copy_a = MagicMock() - copy_b = MagicMock() - copies = [copy_a, copy_b] - - def next_copy(): - return copies.pop(0) if copies else MagicMock() - - redfish_conn_mock.copy.side_effect = next_copy - - copy_a.run_get_paged.return_value = RedfishGetResult( - path="/a", - success=True, - data={"Members": []}, - status_code=200, - ) - copy_b.run_get_paged.return_value = RedfishGetResult( - path="/b", - success=True, - data={"Members": []}, - status_code=200, - ) - - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs( - uris=["/a", "/b"], - max_workers=2, - follow_next_link=True, - max_pages=10, - ) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert len(data.responses) == 2 - copy_a.run_get_paged.assert_called_once_with("/a", max_pages=10) - copy_b.run_get_paged.assert_called_once_with("/b", max_pages=10) - assert redfish_conn_mock.copy.call_count == 2 - assert len(result.artifacts) == 2 - - -def test_discover_tree_single_root(): - conn = MagicMock() - conn.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"@odata.id": "/redfish/v1", "Name": "Root"}, - status_code=200, - ) - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=2, max_endpoints=0 - ) - assert paths == ["/redfish/v1"] - assert list(responses.keys()) == ["/redfish/v1"] - assert responses["/redfish/v1"]["Name"] == "Root" - assert len(results) == 1 - conn.run_get.assert_called_once_with("/redfish/v1") - - -def test_discover_tree_follows_links(): - conn = MagicMock() - root_data = { - "@odata.id": "/redfish/v1", - "Systems": {"@odata.id": "/redfish/v1/Systems"}, - } - systems_data = { - "@odata.id": "/redfish/v1/Systems", - "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], - } - system1_data = {"@odata.id": "/redfish/v1/Systems/1", "Id": "1"} - - def run_get(path): - if path == "/redfish/v1": - return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) - if path == "/redfish/v1/Systems": - return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) - if path == "/redfish/v1/Systems/1": - return RedfishGetResult(path=path, success=True, data=system1_data, status_code=200) - return RedfishGetResult(path=path, success=False, error="Not Found", status_code=404) - - conn.run_get.side_effect = run_get - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=3, max_endpoints=0 - ) - assert "/redfish/v1" in paths - assert "/redfish/v1/Systems" in paths - assert "/redfish/v1/Systems/1" in paths - assert responses["/redfish/v1"]["@odata.id"] == "/redfish/v1" - assert responses["/redfish/v1/Systems"]["@odata.id"] == "/redfish/v1/Systems" - assert responses["/redfish/v1/Systems/1"]["Id"] == "1" - assert len(results) >= 3 - - -def test_discover_tree_respects_max_depth(): - conn = MagicMock() - root_data = {"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}} - systems_data = { - "@odata.id": "/redfish/v1/Systems", - "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], - } - - def run_get(path): - if path == "/redfish/v1": - return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) - if path == "/redfish/v1/Systems": - return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) - return RedfishGetResult(path=path, success=True, data={}, status_code=200) - - conn.run_get.side_effect = run_get - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=1, max_endpoints=0 - ) - assert "/redfish/v1" in paths - assert "/redfish/v1/Systems" not in paths - assert len(responses) == 1 - - -def test_discover_tree_respects_max_endpoints(): - conn = MagicMock() - conn.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}}, - status_code=200, - ) - paths, responses, results = ec._discover_tree( - conn, api_root="redfish/v1", max_depth=5, max_endpoints=1 - ) - assert len(paths) == 1 - assert len(responses) == 1 - conn.run_get.assert_called_once() - - -def test_collect_data_discover_tree_success(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.api_root = "redfish/v1" - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", - success=True, - data={"@odata.id": "/redfish/v1", "Name": "Root"}, - status_code=200, - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(discover_tree=True) - ) - assert result.status == ExecutionStatus.OK - assert result.message == "Collected 1 Redfish endpoint(s) from tree" - assert data is not None - assert "/redfish/v1" in data.responses - assert data.responses["/redfish/v1"]["Name"] == "Root" - - -def test_collect_data_discover_tree_no_responses(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.api_root = "redfish/v1" - redfish_conn_mock.run_get.return_value = RedfishGetResult( - path="/redfish/v1", success=False, error="Connection refused", status_code=None - ) - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs(discover_tree=True) - ) - assert result.status == ExecutionStatus.ERROR - assert result.message.startswith("No Redfish endpoints discovered from tree") - assert data is None - - -def test_collect_data_concurrent_two_uris(redfish_endpoint_collector, redfish_conn_mock): - redfish_conn_mock.copy.return_value = redfish_conn_mock - call_count = 0 - - def run_get(path): - nonlocal call_count - call_count += 1 - return RedfishGetResult( - path=path if path.startswith("/") else "/" + path, - success=True, - data={"path": path}, - status_code=200, - ) - - redfish_conn_mock.run_get.side_effect = run_get - result, data = redfish_endpoint_collector.collect_data( - args=RedfishEndpointCollectorArgs( - uris=["/redfish/v1", "/redfish/v1/Systems"], max_workers=2 - ) - ) - assert result.status == ExecutionStatus.OK - assert data is not None - assert len(data.responses) == 2 - assert "/redfish/v1" in data.responses or "redfish/v1" in data.responses - assert any("Systems" in k for k in data.responses) - assert redfish_conn_mock.copy.called +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import MagicMock + +import pytest + +from nodescraper.connection.redfish import RedfishGetResult +from nodescraper.enums import EventCategory, ExecutionStatus +from nodescraper.plugins.ooband.redfish_endpoint import ( + RedfishEndpointCollector, + RedfishEndpointCollectorArgs, +) +from nodescraper.plugins.ooband.redfish_endpoint import endpoint_collector as ec + + +@pytest.fixture +def redfish_endpoint_collector(system_info, redfish_conn_mock): + return RedfishEndpointCollector( + system_info=system_info, + connection=redfish_conn_mock, + ) + + +def test_redfish_endpoint_collector_no_uris(redfish_endpoint_collector): + result, data = redfish_endpoint_collector.collect_data() + assert result.status == ExecutionStatus.NOT_RAN + assert "No collection mode configured" in result.message + assert data is None + + +def test_redfish_endpoint_collector_no_uris_with_args(redfish_endpoint_collector): + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=[]) + ) + assert result.status == ExecutionStatus.NOT_RAN + assert "No collection mode configured" in result.message + assert data is None + + +def test_redfish_endpoint_collector_one_uri_success(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"Name": "Root"}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) + ) + assert result.status == ExecutionStatus.OK + assert result.message == "Collected 1 Redfish endpoint(s)" + assert data is not None + assert data.responses["/redfish/v1"]["Name"] == "Root" + redfish_conn_mock.run_get.assert_called_once() + call_path = redfish_conn_mock.run_get.call_args[0][0] + assert call_path == "/redfish/v1" or call_path.strip("/") == "redfish/v1" + + +def test_redfish_endpoint_collector_uri_normalized_with_leading_slash( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1/Systems", + success=True, + data={"Members": []}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["redfish/v1/Systems"]) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert "/redfish/v1/Systems" in data.responses or "redfish/v1/Systems" in data.responses + + +def test_redfish_endpoint_collector_one_fail_no_success( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=False, + error="Connection refused", + status_code=None, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"]) + ) + assert result.status == ExecutionStatus.ERROR + assert result.message.startswith("No Redfish endpoints could be read") + assert data is None + assert len(result.events) >= 1 + assert any( + e.category == EventCategory.RUNTIME.value or "Redfish GET failed" in (e.description or "") + for e in result.events + ) + + +def test_redfish_endpoint_collector_mixed_success_fail( + redfish_endpoint_collector, redfish_conn_mock +): + def run_get_side_effect(path): + path_str = str(path) + if "Systems" in path_str: + return RedfishGetResult( + path=path_str if path_str.startswith("/") else "/" + path_str, + success=True, + data={"Id": "1"}, + status_code=200, + ) + return RedfishGetResult( + path=path_str if path_str.startswith("/") else "/" + path_str, + success=False, + error="Not Found", + status_code=404, + ) + + redfish_conn_mock.run_get.side_effect = run_get_side_effect + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1/Systems", "/redfish/v1/Bad"]) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert len(data.responses) == 1 + keys = list(data.responses.keys()) + assert any("Systems" in k for k in keys) + assert list(data.responses.values())[0].get("Id") == "1" + + +def test_normalize_path_empty_or_invalid(): + api = "redfish/v1" + assert ec._normalize_path("", api) == "" + assert ec._normalize_path(None, api) == "" # type: ignore[arg-type] + assert ec._normalize_path(" ", api) == "" + + +def test_normalize_path_relative_path(): + api = "redfish/v1" + assert ec._normalize_path("/redfish/v1", api) == "/redfish/v1" + assert ec._normalize_path("/redfish/v1/", api) == "/redfish/v1" + assert ec._normalize_path("redfish/v1/Systems", api) == "/redfish/v1/Systems" + assert ec._normalize_path(" /redfish/v1/Chassis ", api) == "/redfish/v1/Chassis" + + +def test_normalize_path_full_url(): + api = "redfish/v1" + assert ec._normalize_path("https://host/redfish/v1/Systems", api) == "/redfish/v1/Systems" + assert ec._normalize_path("http://bmc/redfish/v1", api) == "/redfish/v1" + + +def test_normalize_path_outside_api_root(): + api = "redfish/v1" + assert ec._normalize_path("/other/root", api) == "" + assert ec._normalize_path("https://host/other/path", api) == "" + + +def test_extract_odata_ids_empty(): + assert ec._extract_odata_ids({}) == [] + assert ec._extract_odata_ids([]) == [] + assert ec._extract_odata_ids("x") == [] + + +def test_extract_odata_ids_single(): + assert ec._extract_odata_ids({"@odata.id": "/redfish/v1"}) == ["/redfish/v1"] + assert ec._extract_odata_ids({"@odata.id": "https://host/redfish/v1"}) == [ + "https://host/redfish/v1" + ] + + +def test_extract_odata_ids_members(): + body = { + "Members": [ + {"@odata.id": "/redfish/v1/Systems/1"}, + {"@odata.id": "/redfish/v1/Systems/2"}, + ] + } + assert set(ec._extract_odata_ids(body)) == {"/redfish/v1/Systems/1", "/redfish/v1/Systems/2"} + + +def test_extract_odata_ids_nested_and_members(): + body = { + "@odata.id": "/redfish/v1", + "Systems": {"@odata.id": "/redfish/v1/Systems", "Members": []}, + "Chassis": { + "Members": [{"@odata.id": "/redfish/v1/Chassis/1"}], + }, + } + ids = ec._extract_odata_ids(body) + assert "/redfish/v1" in ids + assert "/redfish/v1/Systems" in ids + assert "/redfish/v1/Chassis/1" in ids + + +def test_uris_from_args_none(): + assert ec._uris_from_args(None) == [] + + +def test_uris_from_args_empty(): + assert ec._uris_from_args(RedfishEndpointCollectorArgs(uris=[])) == [] + + +def test_uris_from_args_with_uris(): + args = RedfishEndpointCollectorArgs(uris=["/redfish/v1", "/redfish/v1/Systems"]) + assert ec._uris_from_args(args) == ["/redfish/v1", "/redfish/v1/Systems"] + + +def test_fetch_one_calls_run_get(): + conn = MagicMock() + conn.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={}, status_code=200 + ) + out = ec._fetch_one(conn, "/redfish/v1") + conn.run_get.assert_called_once_with("/redfish/v1") + assert out.success is True + assert out.path == "/redfish/v1" + + +def test_fetch_one_paged_calls_run_get_paged(): + conn = MagicMock() + conn.run_get_paged.return_value = RedfishGetResult( + path="/redfish/v1/Systems", + success=True, + data={"Members": []}, + status_code=200, + ) + out = ec._fetch_one_paged(conn, "/redfish/v1/Systems", max_pages=42) + conn.run_get_paged.assert_called_once_with("/redfish/v1/Systems", max_pages=42) + assert out.success is True + + +def test_run_redfish_get_paged_appends_artifact_when_enabled( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get_paged.return_value = RedfishGetResult( + path="/redfish/v1/Systems", + success=True, + data={"Members": []}, + status_code=200, + ) + redfish_endpoint_collector.result.artifacts.clear() + res = redfish_endpoint_collector._run_redfish_get_paged( + "/redfish/v1/Systems", max_pages=5, log_artifact=True + ) + redfish_conn_mock.run_get_paged.assert_called_once_with("/redfish/v1/Systems", max_pages=5) + assert res.success is True + assert len(redfish_endpoint_collector.result.artifacts) == 1 + + +def test_run_redfish_get_paged_skips_artifact_when_disabled( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get_paged.return_value = RedfishGetResult( + path="/x", + success=True, + data={"Members": []}, + status_code=200, + ) + redfish_endpoint_collector.result.artifacts.clear() + redfish_endpoint_collector._run_redfish_get_paged("/x", log_artifact=False) + assert len(redfish_endpoint_collector.result.artifacts) == 0 + + +def test_collect_follow_next_link_sequential_uses_run_get_paged( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get_paged.return_value = RedfishGetResult( + path="/redfish/v1/Systems", + success=True, + data={"Members": [{"@odata.id": "/redfish/v1/Systems/1"}], "Members@odata.count": 1}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1/Systems"], + follow_next_link=True, + max_pages=50, + ) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.responses["/redfish/v1/Systems"]["Members@odata.count"] == 1 + redfish_conn_mock.run_get_paged.assert_called_once_with("/redfish/v1/Systems", max_pages=50) + redfish_conn_mock.run_get.assert_not_called() + + +def test_collect_follow_next_link_concurrent_uses_connection_copy( + redfish_endpoint_collector, redfish_conn_mock +): + copy_a = MagicMock() + copy_b = MagicMock() + copies = [copy_a, copy_b] + + def next_copy(): + return copies.pop(0) if copies else MagicMock() + + redfish_conn_mock.copy.side_effect = next_copy + + copy_a.run_get_paged.return_value = RedfishGetResult( + path="/a", + success=True, + data={"Members": []}, + status_code=200, + ) + copy_b.run_get_paged.return_value = RedfishGetResult( + path="/b", + success=True, + data={"Members": []}, + status_code=200, + ) + + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/a", "/b"], + max_workers=2, + follow_next_link=True, + max_pages=10, + ) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert len(data.responses) == 2 + copy_a.run_get_paged.assert_called_once_with("/a", max_pages=10) + copy_b.run_get_paged.assert_called_once_with("/b", max_pages=10) + assert redfish_conn_mock.copy.call_count == 2 + assert len(result.artifacts) == 2 + + +def test_discover_tree_single_root(): + conn = MagicMock() + conn.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"@odata.id": "/redfish/v1", "Name": "Root"}, + status_code=200, + ) + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=2, max_endpoints=0 + ) + assert paths == ["/redfish/v1"] + assert list(responses.keys()) == ["/redfish/v1"] + assert responses["/redfish/v1"]["Name"] == "Root" + assert len(results) == 1 + conn.run_get.assert_called_once_with("/redfish/v1") + + +def test_discover_tree_follows_links(): + conn = MagicMock() + root_data = { + "@odata.id": "/redfish/v1", + "Systems": {"@odata.id": "/redfish/v1/Systems"}, + } + systems_data = { + "@odata.id": "/redfish/v1/Systems", + "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], + } + system1_data = {"@odata.id": "/redfish/v1/Systems/1", "Id": "1"} + + def run_get(path): + if path == "/redfish/v1": + return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) + if path == "/redfish/v1/Systems": + return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) + if path == "/redfish/v1/Systems/1": + return RedfishGetResult(path=path, success=True, data=system1_data, status_code=200) + return RedfishGetResult(path=path, success=False, error="Not Found", status_code=404) + + conn.run_get.side_effect = run_get + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=3, max_endpoints=0 + ) + assert "/redfish/v1" in paths + assert "/redfish/v1/Systems" in paths + assert "/redfish/v1/Systems/1" in paths + assert responses["/redfish/v1"]["@odata.id"] == "/redfish/v1" + assert responses["/redfish/v1/Systems"]["@odata.id"] == "/redfish/v1/Systems" + assert responses["/redfish/v1/Systems/1"]["Id"] == "1" + assert len(results) >= 3 + + +def test_discover_tree_respects_max_depth(): + conn = MagicMock() + root_data = {"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}} + systems_data = { + "@odata.id": "/redfish/v1/Systems", + "Members": [{"@odata.id": "/redfish/v1/Systems/1"}], + } + + def run_get(path): + if path == "/redfish/v1": + return RedfishGetResult(path=path, success=True, data=root_data, status_code=200) + if path == "/redfish/v1/Systems": + return RedfishGetResult(path=path, success=True, data=systems_data, status_code=200) + return RedfishGetResult(path=path, success=True, data={}, status_code=200) + + conn.run_get.side_effect = run_get + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=1, max_endpoints=0 + ) + assert "/redfish/v1" in paths + assert "/redfish/v1/Systems" not in paths + assert len(responses) == 1 + + +def test_discover_tree_respects_max_endpoints(): + conn = MagicMock() + conn.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"@odata.id": "/redfish/v1", "Systems": {"@odata.id": "/redfish/v1/Systems"}}, + status_code=200, + ) + paths, responses, results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=5, max_endpoints=1 + ) + assert len(paths) == 1 + assert len(responses) == 1 + conn.run_get.assert_called_once() + + +def test_collect_data_discover_tree_success(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", + success=True, + data={"@odata.id": "/redfish/v1", "Name": "Root"}, + status_code=200, + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(discover_tree=True) + ) + assert result.status == ExecutionStatus.OK + assert result.message == "Collected 1 Redfish endpoint(s) from tree" + assert data is not None + assert "/redfish/v1" in data.responses + assert data.responses["/redfish/v1"]["Name"] == "Root" + + +def test_collect_data_discover_tree_no_responses(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=False, error="Connection refused", status_code=None + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(discover_tree=True) + ) + assert result.status == ExecutionStatus.ERROR + assert result.message.startswith("No Redfish endpoints discovered from tree") + assert data is None + + +def test_collect_data_concurrent_two_uris(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.copy.return_value = redfish_conn_mock + call_count = 0 + + def run_get(path): + nonlocal call_count + call_count += 1 + return RedfishGetResult( + path=path if path.startswith("/") else "/" + path, + success=True, + data={"path": path}, + status_code=200, + ) + + redfish_conn_mock.run_get.side_effect = run_get + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1", "/redfish/v1/Systems"], max_workers=2 + ) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert len(data.responses) == 2 + assert "/redfish/v1" in data.responses or "redfish/v1" in data.responses + assert any("Systems" in k for k in data.responses) + assert redfish_conn_mock.copy.called + + +@MagicMock +def _ok_post_resp(): + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.content = b"{}" + resp.json.return_value = {} + return resp + + +def test_clear_log_uris_posted_after_successful_collection( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={"Name": "Root"}, status_code=200 + ) + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.content = b"{}" + resp.reason = "OK" + redfish_conn_mock.post.return_value = resp + + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1"], + clear_log_uris=[ + "/redfish/v1/Systems/UBB/LogServices/DiagLogs/Actions/LogService.ClearLog" + ], + ) + ) + assert result.status == ExecutionStatus.OK + redfish_conn_mock.post.assert_called_once_with( + "/redfish/v1/Systems/UBB/LogServices/DiagLogs/Actions/LogService.ClearLog", + json={}, + ) + + +def test_clear_log_uris_failure_downgrades_to_warning( + redfish_endpoint_collector, redfish_conn_mock +): + from nodescraper.enums import EventPriority + + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={"Name": "Root"}, status_code=200 + ) + resp = MagicMock() + resp.ok = False + resp.status_code = 500 + resp.reason = "Internal Server Error" + resp.content = b"" + redfish_conn_mock.post.return_value = resp + + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1"], + clear_log_uris=["/some/ClearLog"], + ) + ) + assert result.status == ExecutionStatus.WARNING + assert "log store(s) failed to clear" in result.message + warning_events = [e for e in result.events if e.priority == EventPriority.WARNING] + assert any("Failed to clear" in e.description for e in warning_events) + + +def test_clear_log_uris_not_posted_when_passive(system_info, redfish_conn_mock): + from nodescraper.enums import SystemInteractionLevel + + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={"Name": "Root"}, status_code=200 + ) + from nodescraper.plugins.ooband.redfish_endpoint import RedfishEndpointCollector + + collector = RedfishEndpointCollector( + system_info=system_info, + connection=redfish_conn_mock, + system_interaction_level=SystemInteractionLevel.PASSIVE, + ) + result, data = collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1"], + clear_log_uris=["/some/ClearLog"], + ) + ) + assert result.status == ExecutionStatus.OK + redfish_conn_mock.post.assert_not_called() + + +def test_clear_log_uris_empty_list_no_post(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={"Name": "Root"}, status_code=200 + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"], clear_log_uris=[]) + ) + assert result.status == ExecutionStatus.OK + redfish_conn_mock.post.assert_not_called() + + +def test_clear_log_uris_not_posted_when_collection_fails( + redfish_endpoint_collector, redfish_conn_mock +): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=False, error="Timeout", status_code=None + ) + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs( + uris=["/redfish/v1"], + clear_log_uris=["/some/ClearLog"], + ) + ) + assert result.status == ExecutionStatus.ERROR + redfish_conn_mock.post.assert_not_called() + + +def test_clear_log_uris_multiple_uris_all_posted(redfish_endpoint_collector, redfish_conn_mock): + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/redfish/v1", success=True, data={"Name": "Root"}, status_code=200 + ) + resp = MagicMock() + resp.ok = True + resp.status_code = 200 + resp.content = b"{}" + resp.reason = "OK" + redfish_conn_mock.post.return_value = resp + + clear_uris = ["/clear/a", "/clear/b"] + result, data = redfish_endpoint_collector.collect_data( + args=RedfishEndpointCollectorArgs(uris=["/redfish/v1"], clear_log_uris=clear_uris) + ) + assert result.status == ExecutionStatus.OK + assert redfish_conn_mock.post.call_count == 2 + posted_paths = {call.args[0] for call in redfish_conn_mock.post.call_args_list} + assert "/clear/a" in posted_paths + assert "/clear/b" in posted_paths + + +def test_discover_tree_skips_oem_origin_and_fragments(): + conn = MagicMock() + root = { + "@odata.id": "/redfish/v1", + "Systems": {"@odata.id": "/redfish/v1/Systems"}, + "Oem": {"Secret": {"@odata.id": "/redfish/v1/Oem/Secret"}}, + "OriginOfCondition": {"@odata.id": "/redfish/v1/Chassis/other"}, + "Assembly": {"@odata.id": "/redfish/v1/Chassis/1/Assembly#/Assemblies/0"}, + } + systems = {"@odata.id": "/redfish/v1/Systems", "Members": []} + + def run_get(path): + body = root if path == "/redfish/v1" else systems + return RedfishGetResult(path=path, success=True, data=body, status_code=200) + + conn.run_get.side_effect = run_get + paths, responses, _results = ec._discover_tree( + conn, api_root="redfish/v1", max_depth=3, max_endpoints=0 + ) + assert "/redfish/v1/Systems" in paths + assert "/redfish/v1/Oem/Secret" not in responses + assert "/redfish/v1/Chassis/other" not in responses + assert "/redfish/v1/Chassis/1/Assembly" not in responses + + +def test_discover_tree_keeps_collection_pages_separate(): + conn = MagicMock() + first = "/redfish/v1/Managers/bmc/LogServices/EventLog/Entries" + second = first + "?$skip=2" + pages = { + "/redfish/v1": { + "@odata.id": "/redfish/v1", + "Managers": {"@odata.id": "/redfish/v1/Managers"}, + }, + "/redfish/v1/Managers": { + "@odata.id": "/redfish/v1/Managers", + "Members": [{"@odata.id": "/redfish/v1/Managers/bmc"}], + }, + "/redfish/v1/Managers/bmc": { + "@odata.id": "/redfish/v1/Managers/bmc", + "LogServices": {"@odata.id": "/redfish/v1/Managers/bmc/LogServices"}, + }, + "/redfish/v1/Managers/bmc/LogServices": { + "@odata.id": "/redfish/v1/Managers/bmc/LogServices", + "Members": [{"@odata.id": "/redfish/v1/Managers/bmc/LogServices/EventLog"}], + }, + "/redfish/v1/Managers/bmc/LogServices/EventLog": { + "@odata.id": "/redfish/v1/Managers/bmc/LogServices/EventLog", + "Entries": {"@odata.id": first}, + }, + first: { + "@odata.id": first, + "Members": [{"@odata.id": first + "/1"}, {"@odata.id": first + "/2"}], + "Members@odata.nextLink": second, + }, + second: { + "@odata.id": second, + "Members": [{"@odata.id": first + "/3"}], + }, + first + "/1": {"@odata.id": first + "/1", "Id": "1"}, + first + "/3": {"@odata.id": first + "/3", "Id": "3"}, + } + + def run_get(path): + body = pages.get(path) + if body is None: + return RedfishGetResult(path=path, success=False, error="missing", status_code=404) + return RedfishGetResult(path=path, success=True, data=body, status_code=200) + + conn.run_get.side_effect = run_get + _paths, responses, _results = ec._discover_tree( + conn, + api_root="redfish/v1", + max_depth=8, + max_endpoints=0, + max_collection_pages=2, + max_log_entry_members=1, + ) + assert responses[first]["Members@odata.nextLink"] == second + assert second in responses + assert responses[second]["Members"][0]["@odata.id"] == first + "/3" + assert first + "/1" in responses + assert first + "/2" not in responses + assert first + "/3" in responses diff --git a/test/unit/plugin/test_redfish_endpoint_plugin.py b/test/unit/plugin/test_redfish_endpoint_plugin.py index 6ca56f6b..a0f9f5ab 100644 --- a/test/unit/plugin/test_redfish_endpoint_plugin.py +++ b/test/unit/plugin/test_redfish_endpoint_plugin.py @@ -1,66 +1,79 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from nodescraper.plugins.ooband.redfish_endpoint import ( - RedfishEndpointAnalyzer, - RedfishEndpointAnalyzerArgs, - RedfishEndpointCollector, - RedfishEndpointCollectorArgs, - RedfishEndpointDataModel, - RedfishEndpointPlugin, -) - - -def test_redfish_endpoint_collector_args_default(): - args = RedfishEndpointCollectorArgs() - assert args.uris == [] - - -def test_redfish_endpoint_collector_args_uris_stripped(): - args = RedfishEndpointCollectorArgs(uris=[" /redfish/v1 ", "/Systems/1 ", " Chassis "]) - assert args.uris == ["/redfish/v1", "/Systems/1", "Chassis"] - - -def test_redfish_endpoint_collector_args_uris_empty_list(): - args = RedfishEndpointCollectorArgs(uris=[]) - assert args.uris == [] - - -def test_redfish_endpoint_data_model_default(): - model = RedfishEndpointDataModel() - assert model.responses == {} - - -def test_redfish_endpoint_data_model_responses(): - model = RedfishEndpointDataModel(responses={"/redfish/v1": {"Name": "Root"}}) - assert model.responses["/redfish/v1"]["Name"] == "Root" - - -def test_redfish_endpoint_plugin_class_attributes(): - assert RedfishEndpointPlugin.DATA_MODEL is RedfishEndpointDataModel - assert RedfishEndpointPlugin.COLLECTOR is RedfishEndpointCollector - assert RedfishEndpointPlugin.ANALYZER is RedfishEndpointAnalyzer - assert RedfishEndpointPlugin.COLLECTOR_ARGS is RedfishEndpointCollectorArgs - assert RedfishEndpointPlugin.ANALYZER_ARGS is RedfishEndpointAnalyzerArgs +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.connection.redfish import RedfishSshProxyConnectionManager +from nodescraper.pluginregistry import PluginRegistry +from nodescraper.plugins.ooband.redfish_endpoint import ( + AmcRedfishEndpointPlugin, + RedfishEndpointAnalyzer, + RedfishEndpointAnalyzerArgs, + RedfishEndpointCollector, + RedfishEndpointCollectorArgs, + RedfishEndpointDataModel, + RedfishEndpointPlugin, +) + + +def test_redfish_endpoint_collector_args_default(): + args = RedfishEndpointCollectorArgs() + assert args.uris == [] + + +def test_redfish_endpoint_collector_args_uris_stripped(): + args = RedfishEndpointCollectorArgs(uris=[" /redfish/v1 ", "/Systems/1 ", " Chassis "]) + assert args.uris == ["/redfish/v1", "/Systems/1", "Chassis"] + + +def test_redfish_endpoint_collector_args_uris_empty_list(): + args = RedfishEndpointCollectorArgs(uris=[]) + assert args.uris == [] + + +def test_redfish_endpoint_data_model_default(): + model = RedfishEndpointDataModel() + assert model.responses == {} + + +def test_redfish_endpoint_data_model_responses(): + model = RedfishEndpointDataModel(responses={"/redfish/v1": {"Name": "Root"}}) + assert model.responses["/redfish/v1"]["Name"] == "Root" + + +def test_redfish_endpoint_plugin_class_attributes(): + assert RedfishEndpointPlugin.DATA_MODEL is RedfishEndpointDataModel + assert RedfishEndpointPlugin.COLLECTOR is RedfishEndpointCollector + assert RedfishEndpointPlugin.ANALYZER is RedfishEndpointAnalyzer + assert RedfishEndpointPlugin.COLLECTOR_ARGS is RedfishEndpointCollectorArgs + assert RedfishEndpointPlugin.ANALYZER_ARGS is RedfishEndpointAnalyzerArgs + + +def test_amc_endpoint_plugin_uses_the_ssh_proxy(): + assert AmcRedfishEndpointPlugin.CONNECTION_TYPE is RedfishSshProxyConnectionManager + assert AmcRedfishEndpointPlugin.COLLECTOR is RedfishEndpointCollector + assert AmcRedfishEndpointPlugin.ANALYZER is RedfishEndpointAnalyzer + assert AmcRedfishEndpointPlugin.DATA_MODEL is RedfishEndpointPlugin.DATA_MODEL + assert AmcRedfishEndpointPlugin.COLLECTOR_ARGS is RedfishEndpointCollectorArgs + assert AmcRedfishEndpointPlugin.ANALYZER_ARGS is RedfishEndpointPlugin.ANALYZER_ARGS + assert "AmcRedfishEndpointPlugin" in PluginRegistry().plugins diff --git a/test/unit/plugin/test_redfish_endpoint_ssh_proxy.py b/test/unit/plugin/test_redfish_endpoint_ssh_proxy.py new file mode 100644 index 00000000..8dbf4ec3 --- /dev/null +++ b/test/unit/plugin/test_redfish_endpoint_ssh_proxy.py @@ -0,0 +1,158 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""RedfishEndpointPlugin run against a host behind a BMC via RedfishSshProxyConnectionManager. + +The plugin config names no connection: the executor falls back to the SSH proxy because it is +the only Redfish connection configured. Only the BMC SSH session and the curl call are faked. +""" +import json +from typing import Any, cast +from unittest.mock import MagicMock, patch + +import pytest + +from nodescraper.connection.redfish.ssh_proxy_connection import ( + CurlResponse, + SshProxyRedfishConnection, +) +from nodescraper.enums import ExecutionStatus, OSFamily, SystemLocation +from nodescraper.models import PluginConfig, SystemInfo +from nodescraper.pluginexecutor import PluginExecutor + +BASE = "http://192.0.2.10:80" + +CHASSIS = { + "/redfish/v1": {"@odata.id": "/redfish/v1"}, + "/redfish/v1/Chassis": {"Members@odata.count": 2}, + "/redfish/v1/Chassis/1": { + "Id": "1", + "UUID": "123e4567-e89b-12d3-a456-426614174000", + "SerialNumber": "SN0001", + "Sensors": {"@odata.id": "/redfish/v1/Chassis/1/Sensors"}, + "Status": {"Health": "OK", "State": "Enabled"}, + }, + "/redfish/v1/Chassis/2": { + "Id": "2", + "UUID": "not-a-uuid", + "SerialNumber": "000000000000", + "Status": {"Health": "OK", "State": "Enabled"}, + }, +} + +PLUGIN_ARGS: dict[str, Any] = { + "collection_args": {"uris": ["/redfish/v1/Chassis", "/redfish/v1/Chassis/1"]}, + "analysis_args": { + "checks": { + "/redfish/v1/Chassis": {"Members@odata.count": 2}, + "/redfish/v1/Chassis/1": { + "UUID": {"regex": r"^[0-9a-fA-F-]{36}$"}, + "SerialNumber": {"regex": r"^(?!0+$).+$"}, + "Sensors": {"exists": True}, + "Status/Health": "OK", + }, + } + }, +} + + +def _target(hostname: str) -> dict[str, Any]: + return { + "host": "192.0.2.10", + "ssh": {"hostname": hostname, "username": "root", "password": "x"}, + } + + +def _fake_curl(self: SshProxyRedfishConnection, url: str, extra_flags: str = "") -> CurlResponse: + cast(MagicMock, self._shell).urls.append(url) + path = url[len(BASE) :].rstrip("/") or "/" + body = CHASSIS.get(path) + if body is None: + return CurlResponse(404, b"") + return CurlResponse(200, json.dumps(body).encode()) + + +def _run(connection_args: dict[str, Any], plugin_args: dict[str, Any]): + shells: dict[str, MagicMock] = {} + + def make_shell(ssh_params): + shell = MagicMock(name=str(ssh_params.hostname)) + shell.urls = [] + shells[str(ssh_params.hostname)] = shell + return shell + + with ( + patch( + "nodescraper.connection.redfish.ssh_proxy_manager.RemoteShell", side_effect=make_shell + ), + patch.object(SshProxyRedfishConnection, "_curl", autospec=True, side_effect=_fake_curl), + ): + executor = PluginExecutor( + plugin_configs=[PluginConfig(plugins={"RedfishEndpointPlugin": plugin_args})], + connections={"RedfishSshProxyConnectionManager": connection_args}, + system_info=SystemInfo( + name="test", location=SystemLocation.REMOTE, os_family=OSFamily.LINUX + ), + ) + results = executor.run_queue() + return results, shells + + +@pytest.fixture(autouse=True) +def _no_log_dir(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + +def test_single_target_collects_and_checks_through_proxy(): + results, shells = _run(_target("bmc-a"), PLUGIN_ARGS) + + assert len(results) == 1 + assert results[0].status == ExecutionStatus.OK, results[0].message + assert f"{BASE}/redfish/v1/Chassis/1" in shells["bmc-a"].urls + + +def test_single_target_reports_failed_checks(): + plugin_args = json.loads(json.dumps(PLUGIN_ARGS)) + plugin_args["collection_args"]["uris"] = ["/redfish/v1/Chassis/2"] + plugin_args["analysis_args"]["checks"] = { + "/redfish/v1/Chassis/2": { + "UUID": {"regex": r"^[0-9a-fA-F-]{36}$"}, + "SerialNumber": {"regex": r"^(?!0+$).+$"}, + "Sensors": {"exists": True}, + } + } + + results, _ = _run(_target("bmc-a"), plugin_args) + + assert results[0].status == ExecutionStatus.ERROR + assert "3 check(s) failed" in results[0].message + + +def test_multi_target_collects_from_each_bmc(): + results, shells = _run({"targets": [_target("bmc-a"), _target("bmc-b")]}, PLUGIN_ARGS) + + assert results[0].status == ExecutionStatus.OK, results[0].message + for host in ("bmc-a", "bmc-b"): + assert f"{BASE}/redfish/v1/Chassis/1" in shells[host].urls diff --git a/test/unit/plugin/test_redfish_event_log_analyzer.py b/test/unit/plugin/test_redfish_event_log_analyzer.py new file mode 100644 index 00000000..91dc61af --- /dev/null +++ b/test/unit/plugin/test_redfish_event_log_analyzer.py @@ -0,0 +1,170 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from typing import Optional + +import pytest +from pydantic import ValidationError + +from nodescraper.enums import EventPriority, ExecutionStatus +from nodescraper.plugins.ooband.redfish_event_log import ( + EventLogEntry, + RedfishEventLogAnalyzer, + RedfishEventLogAnalyzerArgs, + RedfishEventLogDataModel, +) + +ERRORS = "ResourceEvent.1.3.ResourceErrorsDetected" +THRESHOLD = "ResourceEvent.1.3.ResourceWarningThresholdExceeded" +STATUS_OK = "ResourceEvent.1.3.ResourceStatusChangedOK" + + +def _entry( + entry_id: int, + severity: Optional[str], + message_id: Optional[str], + day: int = 1, + message: Optional[str] = None, +) -> EventLogEntry: + return EventLogEntry( + id=str(entry_id), + created=f"2026-01-{day:02d}T00:00:00+00:00", + severity=severity, + message_id=message_id, + message=message if message is not None else f"entry {entry_id}", + ) + + +def _analyze(entries: list[EventLogEntry], system_info, **args): + data = RedfishEventLogDataModel( + log_service_path="/redfish/v1/Systems/1/LogServices/EventLog", + total_count=len(entries), + entries=entries, + ) + analyzer = RedfishEventLogAnalyzer(system_info=system_info) + return analyzer.analyze_data(data, RedfishEventLogAnalyzerArgs(**args)) + + +def test_empty_log_is_ok(system_info): + result = _analyze([], system_info) + + assert result.status == ExecutionStatus.OK + assert result.events == [] + + +def test_only_ok_entries_are_ok(system_info): + result = _analyze([_entry(1, "OK", STATUS_OK)], system_info) + + assert result.status == ExecutionStatus.OK + assert result.events == [] + + +def test_critical_and_warning_map_to_error_and_warning(system_info): + entries = [ + _entry(3, "Critical", ERRORS, day=3), + _entry(2, "Warning", THRESHOLD, day=2), + _entry(1, "OK", STATUS_OK, day=1), + ] + + result = _analyze(entries, system_info) + + assert result.status == ExecutionStatus.ERROR + priorities = {e.data["message_id"]: e.priority for e in result.events} + assert priorities == {ERRORS: EventPriority.ERROR, THRESHOLD: EventPriority.WARNING} + assert "1 critical, 1 warning" in result.message + + +def test_warning_only_is_warning(system_info): + result = _analyze([_entry(1, "Warning", THRESHOLD)], system_info) + + assert result.status == ExecutionStatus.WARNING + + +def test_repeats_collapse_with_count_and_newest_message(system_info): + entries = [ + _entry(3, "Critical", ERRORS, day=3, message="newest"), + _entry(2, "Critical", ERRORS, day=2, message="middle"), + _entry(1, "Critical", ERRORS, day=1, message="oldest"), + ] + + result = _analyze(entries, system_info) + + assert len(result.events) == 1 + event = result.events[0] + assert "(x3)" in event.description + assert event.description.endswith(": newest") + assert event.data["count"] == 3 + assert event.data["ids"] == ["3", "2", "1"] + assert event.data["oldest_created"].startswith("2026-01-01") + + +def test_severity_match_is_case_insensitive(system_info): + result = _analyze([_entry(1, "CRITICAL", ERRORS)], system_info) + + assert result.status == ExecutionStatus.ERROR + + +def test_entries_without_message_id_group_by_message(system_info): + entries = [ + _entry(2, "Warning", None, message="fan degraded"), + _entry(1, "Warning", None, message="fan degraded"), + _entry(3, "Warning", None, message="psu degraded"), + ] + + result = _analyze(entries, system_info) + + assert sorted(e.data["count"] for e in result.events) == [1, 2] + assert all("LogEntry" in e.description for e in result.events) + + +def test_since_skips_older_entries_but_keeps_undated(system_info): + entries = [ + _entry(2, "Critical", ERRORS, day=5), + _entry(1, "Critical", THRESHOLD, day=1), + EventLogEntry(id="9", created="unknown", severity="Warning", message_id=THRESHOLD), + ] + + result = _analyze(entries, system_info, since="2026-01-03T00:00:00") + + assert {(e.data["message_id"], e.data["count"]) for e in result.events} == { + (ERRORS, 1), + (THRESHOLD, 1), + } + assert "in 2 of 3 entries" in result.message + + +def test_message_id_include_and_exclude(system_info): + entries = [_entry(2, "Critical", ERRORS), _entry(1, "Warning", THRESHOLD)] + + included = _analyze(entries, system_info, message_id_include=[r"ResourceErrors"]) + excluded = _analyze(entries, system_info, message_id_exclude=[r"Threshold"]) + + assert [e.data["message_id"] for e in included.events] == [ERRORS] + assert [e.data["message_id"] for e in excluded.events] == [ERRORS] + + +def test_invalid_regex_is_rejected(): + with pytest.raises(ValidationError): + RedfishEventLogAnalyzerArgs(message_id_exclude=["("]) diff --git a/test/unit/plugin/test_redfish_event_log_collector.py b/test/unit/plugin/test_redfish_event_log_collector.py new file mode 100644 index 00000000..5b3eaf80 --- /dev/null +++ b/test/unit/plugin/test_redfish_event_log_collector.py @@ -0,0 +1,223 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from datetime import datetime, timezone +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from nodescraper.connection.redfish import RedfishGetResult +from nodescraper.connection.redfish.redfish_clear_log import ClearLogResult +from nodescraper.enums import ExecutionStatus, SystemInteractionLevel +from nodescraper.plugins.ooband.redfish_event_log import ( + EventLogEntry, + RedfishEventLogCollector, + RedfishEventLogCollectorArgs, +) +from nodescraper.plugins.ooband.redfish_event_log.event_log_data import parse_created + +SERVICE = "/redfish/v1/Systems/1/LogServices/EventLog" +ENTRIES = f"{SERVICE}/Entries" + + +def _entry(entry_id: int, day: int, severity: str = "OK") -> dict[str, Any]: + return { + "@odata.id": f"{ENTRIES}/{entry_id}", + "Id": str(entry_id), + "Created": f"2026-01-{day:02d}T00:00:00+00:00", + "Severity": severity, + "MessageId": "ResourceEvent.1.3.ResourceStatusChangedOK", + "Message": f"entry {entry_id}", + } + + +def _page(members: list, count: Optional[int] = None, next_link: Optional[str] = None) -> dict: + body: dict[str, Any] = {"Members": members} + if count is not None: + body["Members@odata.count"] = count + if next_link is not None: + body["Members@odata.nextLink"] = next_link + return body + + +def _connection(pages: dict[str, Optional[dict]]) -> MagicMock: + conn = MagicMock() + + def run_get(path: str) -> RedfishGetResult: + body = pages.get(str(path)) + if body is None: + return RedfishGetResult( + path=str(path), success=False, error="not found", status_code=404 + ) + return RedfishGetResult(path=str(path), success=True, data=body, status_code=200) + + conn.run_get.side_effect = run_get + return conn + + +def _collect(pages, system_info, level=SystemInteractionLevel.INTERACTIVE, **args): + conn = _connection(pages) + collector = RedfishEventLogCollector( + system_info=system_info, connection=conn, system_interaction_level=level + ) + result, data = collector.collect_data( + RedfishEventLogCollectorArgs(log_service_path=SERVICE, **args) + ) + requested = [str(c.args[0]) for c in conn.run_get.call_args_list] + return result, data, requested + + +def test_pages_with_skip_and_ignores_static_next_link(system_info): + static_next = f"{ENTRIES}?$skip=2" + pages = { + ENTRIES: _page([_entry(1, 1), _entry(2, 3)], count=3, next_link=static_next), + f"{ENTRIES}?$skip=2": _page([_entry(3, 2)], count=3, next_link=static_next), + } + + result, data, requested = _collect(pages, system_info) + + assert result.status == ExecutionStatus.OK + assert requested == [ENTRIES, f"{ENTRIES}?$skip=2"] + assert [e.id for e in data.entries] == ["2", "3", "1"] + assert data.total_count == 3 + assert len(result.artifacts) == 2 + + +def test_single_page_without_count_or_next_link(system_info): + result, data, requested = _collect({ENTRIES: _page([_entry(1, 1)])}, system_info) + + assert result.status == ExecutionStatus.OK + assert requested == [ENTRIES] + assert data.total_count == 1 + + +def test_empty_log(system_info): + result, data, _ = _collect({ENTRIES: _page([], count=0)}, system_info) + + assert result.status == ExecutionStatus.OK + assert data.entries == [] + + +def test_service_ignoring_skip_stops_with_warning(system_info): + same_page = _page([_entry(1, 1), _entry(2, 2)], count=5) + pages = {ENTRIES: same_page, f"{ENTRIES}?$skip=2": same_page} + + result, data, requested = _collect(pages, system_info) + + assert result.status == ExecutionStatus.WARNING + assert requested == [ENTRIES, f"{ENTRIES}?$skip=2"] + assert len(data.entries) == 2 + assert any("no new entries" in e.description for e in result.events) + + +def test_first_page_failure_returns_no_data(system_info): + result, data, _ = _collect({}, system_info) + + assert result.status == ExecutionStatus.EXECUTION_FAILURE + assert data is None + + +def test_later_page_failure_keeps_partial_data(system_info): + pages = {ENTRIES: _page([_entry(1, 1)], count=2)} + + result, data, _ = _collect(pages, system_info) + + assert result.status == ExecutionStatus.WARNING + assert [e.id for e in data.entries] == ["1"] + assert data.total_count == 2 + + +def test_max_entries_keeps_newest(system_info): + pages = {ENTRIES: _page([_entry(1, 1), _entry(2, 3), _entry(3, 2)], count=3)} + + _, data, _ = _collect(pages, system_info, max_entries=2) + + assert [e.id for e in data.entries] == ["2", "3"] + assert data.total_count == 3 + + +def test_max_pages_cap(system_info): + pages = { + ENTRIES: _page([_entry(1, 1)], count=3), + f"{ENTRIES}?$skip=1": _page([_entry(2, 2)], count=3), + } + + result, data, requested = _collect(pages, system_info, max_pages=2) + + assert result.status == ExecutionStatus.WARNING + assert len(requested) == 2 + assert len(data.entries) == 2 + + +@patch("nodescraper.plugins.ooband.redfish_event_log.event_log_collector.clear_redfish_logs") +def test_clear_log_when_interactive(mock_clear, system_info): + target = f"{SERVICE}/Actions/LogService.ClearLog" + pages = { + ENTRIES: _page([_entry(1, 1)], count=1), + SERVICE: {"Actions": {"#LogService.ClearLog": {"target": target}}}, + } + mock_clear.return_value = [ClearLogResult(path=target, success=True)] + + result, _, _ = _collect(pages, system_info, clear_logs_after_collection=True) + + mock_clear.assert_called_once() + assert mock_clear.call_args.args[1][0].path == target + assert "cleared log store" in result.message + + +@patch("nodescraper.plugins.ooband.redfish_event_log.event_log_collector.clear_redfish_logs") +def test_clear_log_skipped_when_passive(mock_clear, system_info): + pages = {ENTRIES: _page([_entry(1, 1)], count=1)} + + _collect( + pages, + system_info, + level=SystemInteractionLevel.PASSIVE, + clear_logs_after_collection=True, + ) + + mock_clear.assert_not_called() + + +def test_entry_from_redfish_tolerates_missing_and_scalar_fields(): + entry = EventLogEntry.from_redfish({"Id": 7, "MessageArgs": "SN0001"}) + + assert entry.id == "7" + assert entry.message_args == ["SN0001"] + assert entry.severity is None + + +@pytest.mark.parametrize( + "value, expected", + [ + ("2026-01-01T00:00:00Z", datetime(2026, 1, 1, tzinfo=timezone.utc)), + ("2026-01-01T00:00:00", datetime(2026, 1, 1, tzinfo=timezone.utc)), + ("not a time", None), + (None, None), + ], +) +def test_parse_created(value, expected): + assert parse_created(value) == expected diff --git a/test/unit/plugin/test_redfish_inventory.py b/test/unit/plugin/test_redfish_inventory.py new file mode 100644 index 00000000..fd7f6648 --- /dev/null +++ b/test/unit/plugin/test_redfish_inventory.py @@ -0,0 +1,417 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import json +from typing import Any, cast +from unittest.mock import MagicMock, patch + +import pytest +from pydantic import ValidationError + +from nodescraper.connection.redfish import ( + RedfishGetResult, + RedfishSshProxyConnectionManager, +) +from nodescraper.connection.redfish.ssh_proxy_connection import ( + CurlResponse, + SshProxyRedfishConnection, +) +from nodescraper.enums import ExecutionStatus, OSFamily, SystemLocation +from nodescraper.models import PluginConfig, SystemInfo +from nodescraper.pluginexecutor import PluginExecutor +from nodescraper.pluginregistry import PluginRegistry +from nodescraper.plugins.ooband.redfish_inventory import ( + AmcRedfishInventoryPlugin, + RedfishInventoryCollector, + RedfishInventoryCollectorArgs, + RedfishInventoryPlugin, +) + +ROOT = "/redfish/v1" +CHASSIS = "/redfish/v1/Chassis" +CHASSIS_MEMBER = "/redfish/v1/Chassis/1" +ASSEMBLY = "/redfish/v1/Chassis/1/Assembly" +POWER = "/redfish/v1/Chassis/1/PowerSubsystem" +SYSTEMS = "/redfish/v1/Systems" +SYSTEM = "/redfish/v1/Systems/1" +MEMORY = "/redfish/v1/Systems/1/Memory" +MEMORY_MEMBER = "/redfish/v1/Systems/1/Memory/1" +PROCESSORS = "/redfish/v1/Systems/1/Processors" +PROCESSOR = "/redfish/v1/Systems/1/Processors/1" + + +def _body(path: str, **extra: Any) -> dict[str, Any]: + payload: dict[str, Any] = {"@odata.id": path} + payload.update(extra) + return payload + + +def _service() -> dict[str, dict]: + return { + ROOT: _body( + ROOT, + Chassis={"@odata.id": CHASSIS}, + Systems={"@odata.id": SYSTEMS}, + Oem={"Secret": {"@odata.id": "/redfish/v1/Oem/Secret"}}, + ), + CHASSIS: _body(CHASSIS, Members=[{"@odata.id": CHASSIS_MEMBER}]), + CHASSIS_MEMBER: _body( + CHASSIS_MEMBER, + Assembly={"@odata.id": ASSEMBLY + "#/Assemblies/0"}, + PowerSubsystem={"@odata.id": POWER}, + ), + POWER: _body(POWER, Id="Power"), + SYSTEMS: _body( + SYSTEMS, + Members=[{"@odata.id": SYSTEM}], + **{"Members@odata.nextLink": SYSTEMS + "?$skip=1"}, + ), + SYSTEMS + "?$skip=1": _body(SYSTEMS + "?$skip=1", Members=[{"@odata.id": SYSTEM + "b"}]), + SYSTEM: _body( + SYSTEM, + Memory={"@odata.id": MEMORY}, + Processors={"@odata.id": PROCESSORS}, + ), + MEMORY: _body(MEMORY, Members=[{"@odata.id": MEMORY_MEMBER}]), + MEMORY_MEMBER: _body(MEMORY_MEMBER, Id="1"), + PROCESSORS: _body(PROCESSORS, Members=[{"@odata.id": PROCESSOR}]), + PROCESSOR: _body(PROCESSOR, Id="CPU1"), + } + + +def _run_get(service: dict[str, dict]): + def run_get(path: str) -> RedfishGetResult: + body = service.get(path) + if body is None: + return RedfishGetResult(path=path, success=False, error="missing", status_code=404) + return RedfishGetResult(path=path, success=True, data=body, status_code=200) + + return run_get + + +def test_bmc_profile_reads_chassis_power_and_system(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(_service()) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data(args=RedfishInventoryCollectorArgs(profile="bmc")) + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.responses[CHASSIS_MEMBER]["@odata.id"] == CHASSIS_MEMBER + assert data.responses[POWER]["Id"] == "Power" + assert data.responses[SYSTEM]["@odata.id"] == SYSTEM + assert ASSEMBLY not in data.responses + assert MEMORY not in data.responses + assert "/redfish/v1/Oem/Secret" not in data.responses + + +def test_smc_profile_reads_memory_and_processors(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(_service()) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data( + args=RedfishInventoryCollectorArgs(profile="smc", max_pages=1) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.responses[MEMORY_MEMBER]["Id"] == "1" + assert data.responses[PROCESSOR]["Id"] == "CPU1" + assert CHASSIS not in data.responses + assert SYSTEMS + "?$skip=1" not in data.responses + + +def test_max_pages_keeps_the_next_collection_page(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(_service()) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data( + args=RedfishInventoryCollectorArgs(profile="smc", max_pages=2) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert SYSTEMS + "?$skip=1" in data.responses + + +def test_missing_service_root_is_an_error(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path=ROOT, success=False, error="refused", status_code=None + ) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data(args=RedfishInventoryCollectorArgs(profile="bmc")) + assert result.status == ExecutionStatus.ERROR + assert data is None + + +def test_plugin_registers_collector(): + assert RedfishInventoryPlugin.COLLECTOR is RedfishInventoryCollector + assert RedfishInventoryPlugin.DATA_MODEL.__name__ == "RedfishInventoryDataModel" + assert RedfishInventoryPlugin.ANALYZER.__name__ == "RedfishInventoryAnalyzer" + + +def test_amc_inventory_plugin_uses_the_ssh_proxy(): + assert AmcRedfishInventoryPlugin.CONNECTION_TYPE is RedfishSshProxyConnectionManager + assert AmcRedfishInventoryPlugin.COLLECTOR is RedfishInventoryCollector + assert AmcRedfishInventoryPlugin.DATA_MODEL is RedfishInventoryPlugin.DATA_MODEL + assert AmcRedfishInventoryPlugin.COLLECTOR_ARGS is RedfishInventoryCollectorArgs + assert AmcRedfishInventoryPlugin.ANALYZER is RedfishInventoryPlugin.ANALYZER + assert "AmcRedfishInventoryPlugin" in PluginRegistry().plugins + + +BASE = "http://192.0.2.10:80" +PROXY_SERVICE = { + ROOT: _body(ROOT, Chassis={"@odata.id": CHASSIS}), + CHASSIS: _body(CHASSIS, Members=[{"@odata.id": CHASSIS_MEMBER}]), + CHASSIS_MEMBER: _body(CHASSIS_MEMBER, PowerSubsystem={"@odata.id": POWER}), + POWER: _body(POWER, Id="Power"), +} + + +def _fake_curl(self: SshProxyRedfishConnection, url: str, extra_flags: str = "") -> CurlResponse: + cast(MagicMock, self._shell).urls.append(url) + path = url[len(BASE) :].rstrip("/") or "/" + body = PROXY_SERVICE.get(path) + if body is None: + return CurlResponse(404, b"") + return CurlResponse(200, json.dumps(body).encode()) + + +def test_inventory_collects_through_ssh_proxy(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + shell = MagicMock(name="bmc-a") + shell.urls = [] + + def make_shell(ssh_params): + return shell + + with ( + patch( + "nodescraper.connection.redfish.ssh_proxy_manager.RemoteShell", side_effect=make_shell + ), + patch.object(SshProxyRedfishConnection, "_curl", autospec=True, side_effect=_fake_curl), + ): + executor = PluginExecutor( + plugin_configs=[ + PluginConfig( + plugins={"RedfishInventoryPlugin": {"collection_args": {"profile": "bmc"}}} + ) + ], + connections={ + "RedfishSshProxyConnectionManager": { + "host": "192.0.2.10", + "ssh": {"hostname": "bmc-a", "username": "root", "password": "x"}, + } + }, + system_info=SystemInfo( + name="test", location=SystemLocation.REMOTE, os_family=OSFamily.LINUX + ), + ) + results = executor.run_queue() + + assert results[0].status == ExecutionStatus.OK, results[0].message + assert f"{BASE}{POWER}" in shell.urls + + +UPDATE = "/redfish/v1/UpdateService" +FIRMWARE = "/redfish/v1/UpdateService/FirmwareInventory" +FIRMWARE_MEMBER = "/redfish/v1/UpdateService/FirmwareInventory/bmc" + + +def _service_with_firmware() -> dict[str, dict]: + service = _service() + service[ROOT]["UpdateService"] = {"@odata.id": UPDATE} + service[UPDATE] = _body(UPDATE, FirmwareInventory={"@odata.id": FIRMWARE}) + service[FIRMWARE] = _body(FIRMWARE, Members=[{"@odata.id": FIRMWARE_MEMBER}]) + service[FIRMWARE_MEMBER] = _body(FIRMWARE_MEMBER, Version="1.0") + return service + + +def test_profiles_read_firmware_inventory_members(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(_service_with_firmware()) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + for profile in ("bmc", "smc"): + result, data = collector.collect_data(args=RedfishInventoryCollectorArgs(profile=profile)) + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.responses[FIRMWARE_MEMBER]["Version"] == "1.0" + + +def test_disabling_update_service_skips_firmware_inventory(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(_service_with_firmware()) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data( + args=RedfishInventoryCollectorArgs(profile="bmc", enabled={"update_service": False}) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert UPDATE not in data.responses + assert FIRMWARE_MEMBER not in data.responses + + +def test_extra_names_append_to_the_profile_set(): + args = RedfishInventoryCollectorArgs( + profile="bmc", + collections=["Sensors", "Chassis"], + member_links=["EnvironmentMetrics", "Assembly"], + ) + assert args.selected_collections()[:5] == [ + "Chassis", + "Managers", + "TelemetryService", + "UpdateService", + "Systems", + ] + assert args.selected_collections()[-1] == "Sensors" + assert args.selected_collections().count("Chassis") == 1 + assert args.selected_links()[-1] == "EnvironmentMetrics" + assert args.selected_links().count("Assembly") == 1 + assert "Memory" not in args.selected_links() + + +def test_enabled_false_still_removes_an_appended_known_area(): + args = RedfishInventoryCollectorArgs( + profile="smc", + collections=["Systems"], + enabled={"systems": False}, + ) + assert "Systems" not in args.selected_collections() + assert "UpdateService" in args.selected_collections() + + +def test_appended_collection_is_read(system_info, redfish_conn_mock): + sensors = "/redfish/v1/Sensors" + sensor = "/redfish/v1/Sensors/1" + service = _service() + service[ROOT]["Sensors"] = {"@odata.id": sensors} + service[sensors] = _body(sensors, Members=[{"@odata.id": sensor}]) + service[sensor] = _body(sensor, Id="1") + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(service) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data( + args=RedfishInventoryCollectorArgs(profile="bmc", collections=["Sensors"]) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert data.responses[sensor]["Id"] == "1" + assert CHASSIS_MEMBER in data.responses + assert MEMORY not in data.responses + + +def test_default_args_enable_every_inventory_area(): + args = RedfishInventoryCollectorArgs() + assert args.selected_collections() == [ + "Chassis", + "Managers", + "TelemetryService", + "UpdateService", + "Systems", + ] + assert "Assembly" in args.selected_links() + assert "PowerSubsystem" in args.selected_links() + assert "FruDevices" in args.selected_links() + assert "Memory" in args.selected_links() + assert "Processors" in args.selected_links() + + +def test_enabled_can_disable_power_and_turn_systems_on(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get(_service()) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data( + args=RedfishInventoryCollectorArgs( + profile="bmc", + enabled={"power": False, "systems": True, "memory": True, "processors": False}, + ) + ) + assert result.status == ExecutionStatus.OK + assert data is not None + assert CHASSIS_MEMBER in data.responses + assert POWER not in data.responses + assert MEMORY_MEMBER in data.responses + assert PROCESSOR not in data.responses + + +def test_unknown_inventory_switch_is_rejected(): + with pytest.raises(ValidationError): + RedfishInventoryCollectorArgs(enabled={"sensors": False}) + + +MISSING_MEMBER = "/redfish/v1/Chassis/missing" +BROKEN_MEMBER = "/redfish/v1/Chassis/broken" + + +def _service_with_extra_members() -> dict[str, dict]: + service = _service() + service[CHASSIS] = _body( + CHASSIS, + Members=[ + {"@odata.id": CHASSIS_MEMBER}, + {"@odata.id": MISSING_MEMBER}, + {"@odata.id": BROKEN_MEMBER}, + ], + ) + return service + + +def _run_get_with_broken_member(service: dict[str, dict]): + base = _run_get(service) + + def run_get(path: str) -> RedfishGetResult: + if path == BROKEN_MEMBER: + return RedfishGetResult(path=path, success=False, error="broken", status_code=500) + return base(path) + + return run_get + + +def test_direct_connection_warns_when_a_listed_member_is_missing(system_info, redfish_conn_mock): + redfish_conn_mock.api_root = "redfish/v1" + redfish_conn_mock.run_get.side_effect = _run_get_with_broken_member( + _service_with_extra_members() + ) + collector = RedfishInventoryCollector(system_info=system_info, connection=redfish_conn_mock) + result, data = collector.collect_data(args=RedfishInventoryCollectorArgs(profile="bmc")) + descriptions = [event.description for event in result.events] + assert data is not None + assert CHASSIS_MEMBER in data.responses + assert any(MISSING_MEMBER in description for description in descriptions) + assert any(BROKEN_MEMBER in description for description in descriptions) + + +def test_ssh_proxy_treats_missing_collection_member_as_empty(system_info): + connection = SshProxyRedfishConnection(shell=MagicMock(), base_url="http://192.0.2.10:80") + connection.api_root = "redfish/v1" + connection.run_get = _run_get_with_broken_member(_service_with_extra_members()) + collector = RedfishInventoryCollector(system_info=system_info, connection=connection) + result, data = collector.collect_data(args=RedfishInventoryCollectorArgs(profile="bmc")) + descriptions = [event.description for event in result.events] + assert result.status == ExecutionStatus.OK + assert data is not None + assert CHASSIS_MEMBER in data.responses + assert MISSING_MEMBER not in data.responses + assert not any(MISSING_MEMBER in description for description in descriptions) + assert any(BROKEN_MEMBER in description for description in descriptions) diff --git a/test/unit/plugin/test_redfish_inventory_firmware.py b/test/unit/plugin/test_redfish_inventory_firmware.py new file mode 100644 index 00000000..8e9daf4d --- /dev/null +++ b/test/unit/plugin/test_redfish_inventory_firmware.py @@ -0,0 +1,198 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from nodescraper.connection.redfish.redfish_manager import MultiTargetRedfishConnection +from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus +from nodescraper.plugins.ooband.redfish_inventory.analyzer_args import ( + RedfishInventoryAnalyzerArgs, +) +from nodescraper.plugins.ooband.redfish_inventory.firmware_compare import ( + firmware_components, +) +from nodescraper.plugins.ooband.redfish_inventory.inventory_analyzer import ( + RedfishInventoryAnalyzer, +) +from nodescraper.plugins.ooband.redfish_inventory.inventory_data import ( + RedfishInventoryDataModel, +) +from nodescraper.plugins.ooband.redfish_inventory.inventory_plugin import ( + RedfishInventoryPlugin, +) + +FW_COLLECTION = "/redfish/v1/UpdateService/FirmwareInventory" +FW_ACTIVE = "/redfish/v1/UpdateService/FirmwareInventory/fw-active" +FW_SLOT_A = "/redfish/v1/UpdateService/FirmwareInventory/fw-slot-0001" +FW_SLOT_B = "/redfish/v1/UpdateService/FirmwareInventory/fw-slot-0002" +TARGET_A = "node-a" +TARGET_B = "node-b" + + +def _member(path: str, fw_id: str, version, **extra) -> dict: + body = { + "@odata.id": path, + "Id": fw_id, + "Version": version, + "Status": {"Health": "OK", "State": "Enabled"}, + "SoftwareId": "SW-1", + "Updateable": True, + "Oem": {"SVN": "4"}, + } + body.update(extra) + return body + + +def _model(*pairs: tuple[str, dict]) -> RedfishInventoryDataModel: + return RedfishInventoryDataModel(responses={path: body for path, body in pairs}) + + +def test_null_version_is_recorded_with_the_other_firmware_fields(): + model = _model( + ( + FW_ACTIVE, + _member(FW_ACTIVE, "fw-active", None, SoftwareId=None, Updateable=False, Oem={}), + ), + ( + FW_COLLECTION, + {"@odata.id": FW_COLLECTION, "Members": [{"@odata.id": FW_ACTIVE}]}, + ), + ) + components = firmware_components(model) + assert list(components) == ["fw-active"] + assert components["fw-active"] == { + "version": "N/A", + "health": "OK", + "state": "Enabled", + "software_id": "N/A", + "updateable": "False", + "svn": "", + } + + +def test_instance_suffixes_stay_separate_ids(): + model = _model( + (FW_SLOT_A, _member(FW_SLOT_A, "fw-slot-0001", "1.0")), + (FW_SLOT_B, _member(FW_SLOT_B, "fw-slot-0002", "1.0")), + ) + assert set(firmware_components(model)) == {"fw-slot-0001", "fw-slot-0002"} + + +def test_version_mismatch_warns_and_includes_firmware_fields(system_info): + analyzer = RedfishInventoryAnalyzer(system_info=system_info) + result = analyzer.compare_firmware( + { + TARGET_A: _model((FW_ACTIVE, _member(FW_ACTIVE, "fw-active", "1.0"))), + TARGET_B: _model((FW_ACTIVE, _member(FW_ACTIVE, "fw-active", "1.1", Oem={"SVN": "9"}))), + } + ) + assert result.status == ExecutionStatus.WARNING + assert result.message.startswith("Firmware differs for 1 component(s)") + assert len(result.events) == 1 + event = result.events[0] + assert event.category == EventCategory.FW.value + assert event.priority == EventPriority.WARNING + assert "node-a=version=1.0 health=OK state=Enabled software_id=SW-1 updateable=True svn=4" in ( + event.description + ) + assert "node-b=version=1.1 health=OK state=Enabled software_id=SW-1 updateable=True svn=9" in ( + event.description + ) + + +def test_missing_member_warns(system_info): + analyzer = RedfishInventoryAnalyzer(system_info=system_info) + result = analyzer.compare_firmware( + { + TARGET_A: _model((FW_SLOT_A, _member(FW_SLOT_A, "fw-slot-0001", "1.0"))), + TARGET_B: _model((FW_SLOT_B, _member(FW_SLOT_B, "fw-slot-0002", "1.0"))), + } + ) + assert result.status == ExecutionStatus.WARNING + ids = {event.data["firmware_id"] for event in result.events} + assert ids == {"fw-slot-0001", "fw-slot-0002"} + missing = next(event for event in result.events if event.data["firmware_id"] == "fw-slot-0001") + assert "node-b=missing" in missing.description + + +def test_matching_versions_are_ok(system_info): + member = _member(FW_ACTIVE, "fw-active", "1.0") + analyzer = RedfishInventoryAnalyzer(system_info=system_info) + result = analyzer.compare_firmware( + { + TARGET_A: _model((FW_ACTIVE, member)), + TARGET_B: _model((FW_ACTIVE, dict(member))), + } + ) + assert result.status == ExecutionStatus.OK + assert result.events == [] + assert result.message.startswith("Firmware versions match across 2 targets") + + +def test_compare_firmware_can_be_turned_off(system_info): + analyzer = RedfishInventoryAnalyzer(system_info=system_info) + result = analyzer.compare_firmware( + { + TARGET_A: _model((FW_ACTIVE, _member(FW_ACTIVE, "fw-active", "1.0"))), + TARGET_B: _model((FW_ACTIVE, _member(FW_ACTIVE, "fw-active", "9.0"))), + }, + RedfishInventoryAnalyzerArgs(compare_firmware=False), + ) + assert result.status == ExecutionStatus.OK + assert result.message == "Firmware comparison disabled" + assert result.events == [] + + +def test_no_firmware_members_warns_once(system_info): + analyzer = RedfishInventoryAnalyzer(system_info=system_info) + result = analyzer.compare_firmware( + { + TARGET_A: RedfishInventoryDataModel(responses={}), + TARGET_B: RedfishInventoryDataModel(responses={}), + } + ) + assert result.status == ExecutionStatus.WARNING + assert len(result.events) == 1 + assert result.events[0].description == ( + "Firmware comparison did not run: no FirmwareInventory members were collected" + ) + + +def test_plugin_compares_multi_target_inventory(system_info): + connection = MultiTargetRedfishConnection({}) + connection.multi_target_data = { + TARGET_A: _model((FW_ACTIVE, _member(FW_ACTIVE, "fw-active", None))), + TARGET_B: _model((FW_ACTIVE, _member(FW_ACTIVE, "fw-active", "2.0"))), + } + + class _Manager: + def __init__(self, open_connection): + self.connection = open_connection + + plugin = RedfishInventoryPlugin( + system_info=system_info, connection_manager=_Manager(connection) + ) + result = plugin.analyze() + assert result.status == ExecutionStatus.WARNING + assert "version=N/A" in result.events[0].description + assert "version=2.0" in result.events[0].description diff --git a/test/unit/plugin/test_redfish_oem_diag_analyzer.py b/test/unit/plugin/test_redfish_oem_diag_analyzer.py index a8d6f1f7..15de0a41 100644 --- a/test/unit/plugin/test_redfish_oem_diag_analyzer.py +++ b/test/unit/plugin/test_redfish_oem_diag_analyzer.py @@ -1,114 +1,114 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import pytest - -from nodescraper.enums import ExecutionStatus -from nodescraper.plugins.ooband.redfish_oem_diag import ( - RedfishOemDiagAnalyzer, - RedfishOemDiagAnalyzerArgs, - RedfishOemDiagDataModel, -) -from nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_data import OemDiagTypeResult - - -@pytest.fixture -def redfish_oem_diag_analyzer(system_info): - return RedfishOemDiagAnalyzer(system_info=system_info) - - -def test_redfish_oem_diag_analyzer_no_results(redfish_oem_diag_analyzer): - data = RedfishOemDiagDataModel(results={}) - result = redfish_oem_diag_analyzer.analyze_data(data) - assert result.status == ExecutionStatus.NOT_RAN - assert result.message == "No OEM diagnostic results to analyze" - - -def test_redfish_oem_diag_analyzer_all_success(redfish_oem_diag_analyzer): - data = RedfishOemDiagDataModel( - results={ - "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), - "AllLogs": OemDiagTypeResult(success=True, error=None, metadata={}), - } - ) - result = redfish_oem_diag_analyzer.analyze_data(data) - assert result.status == ExecutionStatus.OK - assert "2/2 types collected" in result.message - - -def test_redfish_oem_diag_analyzer_some_failed_without_require_all(redfish_oem_diag_analyzer): - data = RedfishOemDiagDataModel( - results={ - "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), - "AllLogs": OemDiagTypeResult(success=False, error="Timeout", metadata=None), - } - ) - result = redfish_oem_diag_analyzer.analyze_data(data) - assert result.status == ExecutionStatus.OK - assert "1/2 types collected" in result.message - - -def test_redfish_oem_diag_analyzer_some_failed_with_require_all_success(redfish_oem_diag_analyzer): - data = RedfishOemDiagDataModel( - results={ - "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), - "AllLogs": OemDiagTypeResult(success=False, error="Task timeout", metadata=None), - } - ) - args = RedfishOemDiagAnalyzerArgs(require_all_success=True) - result = redfish_oem_diag_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.ERROR - assert "1 type(s) failed" in result.message - assert "AllLogs" in result.message - assert len(result.events) >= 1 - assert any( - "AllLogs" in (e.description or "") or "failed" in (e.description or "").lower() - for e in result.events - ) - - -def test_redfish_oem_diag_analyzer_all_failed_require_all_success(redfish_oem_diag_analyzer): - data = RedfishOemDiagDataModel( - results={ - "JournalControl": OemDiagTypeResult(success=False, error="Err1", metadata=None), - "AllLogs": OemDiagTypeResult(success=False, error="Err2", metadata=None), - } - ) - args = RedfishOemDiagAnalyzerArgs(require_all_success=True) - result = redfish_oem_diag_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.ERROR - assert "2 type(s) failed" in result.message - - -def test_redfish_oem_diag_analyzer_require_all_success_all_ok(redfish_oem_diag_analyzer): - data = RedfishOemDiagDataModel( - results={ - "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), - } - ) - args = RedfishOemDiagAnalyzerArgs(require_all_success=True) - result = redfish_oem_diag_analyzer.analyze_data(data, args=args) - assert result.status == ExecutionStatus.OK - assert "1/1 types collected" in result.message +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import pytest + +from nodescraper.enums import ExecutionStatus +from nodescraper.plugins.ooband.redfish_oem_diag import ( + RedfishOemDiagAnalyzer, + RedfishOemDiagAnalyzerArgs, + RedfishOemDiagDataModel, +) +from nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_data import OemDiagTypeResult + + +@pytest.fixture +def redfish_oem_diag_analyzer(system_info): + return RedfishOemDiagAnalyzer(system_info=system_info) + + +def test_redfish_oem_diag_analyzer_no_results(redfish_oem_diag_analyzer): + data = RedfishOemDiagDataModel(results={}) + result = redfish_oem_diag_analyzer.analyze_data(data) + assert result.status == ExecutionStatus.NOT_RAN + assert result.message == "No OEM diagnostic results to analyze" + + +def test_redfish_oem_diag_analyzer_all_success(redfish_oem_diag_analyzer): + data = RedfishOemDiagDataModel( + results={ + "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), + "AllLogs": OemDiagTypeResult(success=True, error=None, metadata={}), + } + ) + result = redfish_oem_diag_analyzer.analyze_data(data) + assert result.status == ExecutionStatus.OK + assert "2/2 types collected" in result.message + + +def test_redfish_oem_diag_analyzer_some_failed_without_require_all(redfish_oem_diag_analyzer): + data = RedfishOemDiagDataModel( + results={ + "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), + "AllLogs": OemDiagTypeResult(success=False, error="Timeout", metadata=None), + } + ) + result = redfish_oem_diag_analyzer.analyze_data(data) + assert result.status == ExecutionStatus.OK + assert "1/2 types collected" in result.message + + +def test_redfish_oem_diag_analyzer_some_failed_with_require_all_success(redfish_oem_diag_analyzer): + data = RedfishOemDiagDataModel( + results={ + "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), + "AllLogs": OemDiagTypeResult(success=False, error="Task timeout", metadata=None), + } + ) + args = RedfishOemDiagAnalyzerArgs(require_all_success=True) + result = redfish_oem_diag_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.ERROR + assert "1 type(s) failed" in result.message + assert "AllLogs" in result.message + assert len(result.events) >= 1 + assert any( + "AllLogs" in (e.description or "") or "failed" in (e.description or "").lower() + for e in result.events + ) + + +def test_redfish_oem_diag_analyzer_all_failed_require_all_success(redfish_oem_diag_analyzer): + data = RedfishOemDiagDataModel( + results={ + "JournalControl": OemDiagTypeResult(success=False, error="Err1", metadata=None), + "AllLogs": OemDiagTypeResult(success=False, error="Err2", metadata=None), + } + ) + args = RedfishOemDiagAnalyzerArgs(require_all_success=True) + result = redfish_oem_diag_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.ERROR + assert "2 type(s) failed" in result.message + + +def test_redfish_oem_diag_analyzer_require_all_success_all_ok(redfish_oem_diag_analyzer): + data = RedfishOemDiagDataModel( + results={ + "JournalControl": OemDiagTypeResult(success=True, error=None, metadata={}), + } + ) + args = RedfishOemDiagAnalyzerArgs(require_all_success=True) + result = redfish_oem_diag_analyzer.analyze_data(data, args=args) + assert result.status == ExecutionStatus.OK + assert "1/1 types collected" in result.message diff --git a/test/unit/plugin/test_redfish_oem_diag_collector.py b/test/unit/plugin/test_redfish_oem_diag_collector.py index 219e4d00..f5e70510 100644 --- a/test/unit/plugin/test_redfish_oem_diag_collector.py +++ b/test/unit/plugin/test_redfish_oem_diag_collector.py @@ -1,126 +1,298 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -from unittest.mock import patch - -import pytest - -from nodescraper.enums import ExecutionStatus -from nodescraper.plugins.ooband.redfish_oem_diag import ( - RedfishOemDiagCollector, - RedfishOemDiagCollectorArgs, - RedfishOemDiagDataModel, -) - - -@pytest.fixture -def redfish_oem_diag_collector(system_info, redfish_conn_mock): - return RedfishOemDiagCollector( - system_info=system_info, - connection=redfish_conn_mock, - ) - - -def test_redfish_oem_diag_collector_no_types_configured(redfish_oem_diag_collector): - result, data = redfish_oem_diag_collector.collect_data() - assert result.status == ExecutionStatus.NOT_RAN - assert result.message == "No OEM diagnostic types configured" - assert data is None - - -def test_redfish_oem_diag_collector_empty_types_with_args(redfish_oem_diag_collector): - result, data = redfish_oem_diag_collector.collect_data( - args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=[]) - ) - assert result.status == ExecutionStatus.NOT_RAN - assert data is None - - -@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") -def test_redfish_oem_diag_collector_one_type_success(mock_collect, redfish_oem_diag_collector): - mock_collect.return_value = (b"log bytes", {"Size": 123}, None) - result, data = redfish_oem_diag_collector.collect_data( - args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["JournalControl"]) - ) - assert result.status == ExecutionStatus.OK - assert "1/1 types collected" in result.message - assert data is not None - assert isinstance(data, RedfishOemDiagDataModel) - assert "JournalControl" in data.results - assert data.results["JournalControl"].success is True - assert data.results["JournalControl"].error is None - assert data.results["JournalControl"].metadata == {"Size": 123} - mock_collect.assert_called_once() - - -@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") -def test_redfish_oem_diag_collector_one_type_failure(mock_collect, redfish_oem_diag_collector): - mock_collect.return_value = (None, None, "Task timeout") - result, data = redfish_oem_diag_collector.collect_data( - args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["AllLogs"]) - ) - assert result.status == ExecutionStatus.ERROR - assert "0/1 types collected" in result.message - assert data is not None - assert data.results["AllLogs"].success is False - assert data.results["AllLogs"].error == "Task timeout" - - -@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") -def test_redfish_oem_diag_collector_mixed_success_fail(mock_collect, redfish_oem_diag_collector): - def side_effect(conn, log_service_path, oem_diagnostic_type, **kwargs): - if oem_diagnostic_type == "JournalControl": - return (b"data", {}, None) - return (None, None, "Not supported") - - mock_collect.side_effect = side_effect - result, data = redfish_oem_diag_collector.collect_data( - args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["JournalControl", "AllLogs"]) - ) - assert result.status == ExecutionStatus.OK - assert "1/2 types collected" in result.message - assert data.results["JournalControl"].success is True - assert data.results["AllLogs"].success is False - assert data.results["AllLogs"].error == "Not supported" - assert mock_collect.call_count == 2 - - -@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") -def test_redfish_oem_diag_collector_passes_args_to_connection( - mock_collect, redfish_oem_diag_collector, redfish_conn_mock -): - mock_collect.return_value = (b"", {}, None) - args = RedfishOemDiagCollectorArgs( - log_service_path="redfish/v1/Systems/1/LogServices/DiagLogs", - oem_diagnostic_types=["JournalControl"], - task_timeout_s=300, - ) - redfish_oem_diag_collector.collect_data(args=args) - mock_collect.assert_called_once() - call_kw = mock_collect.call_args[1] - assert call_kw["log_service_path"] == "redfish/v1/Systems/1/LogServices/DiagLogs" - assert call_kw["oem_diagnostic_type"] == "JournalControl" - assert call_kw["task_timeout_s"] == 300 - assert call_kw["output_dir"] is None +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from unittest.mock import patch + +import pytest + +from nodescraper.enums import ExecutionStatus +from nodescraper.plugins.ooband.redfish_oem_diag import ( + RedfishOemDiagCollector, + RedfishOemDiagCollectorArgs, + RedfishOemDiagDataModel, +) + + +@pytest.fixture +def redfish_oem_diag_collector(system_info, redfish_conn_mock): + return RedfishOemDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + ) + + +def test_redfish_oem_diag_collector_no_types_configured(redfish_oem_diag_collector): + result, data = redfish_oem_diag_collector.collect_data() + assert result.status == ExecutionStatus.NOT_RAN + assert result.message == "No OEM diagnostic types configured" + assert data is None + + +def test_redfish_oem_diag_collector_empty_types_with_args(redfish_oem_diag_collector): + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=[]) + ) + assert result.status == ExecutionStatus.NOT_RAN + assert data is None + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +def test_redfish_oem_diag_collector_one_type_success(mock_collect, redfish_oem_diag_collector): + mock_collect.return_value = (b"log bytes", {"Size": 123}, None) + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["JournalControl"]) + ) + assert result.status == ExecutionStatus.OK + assert "1/1 types collected" in result.message + assert data is not None + assert isinstance(data, RedfishOemDiagDataModel) + assert "JournalControl" in data.results + assert data.results["JournalControl"].success is True + assert data.results["JournalControl"].error is None + assert data.results["JournalControl"].metadata == {"Size": 123} + mock_collect.assert_called_once() + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +def test_redfish_oem_diag_collector_one_type_failure(mock_collect, redfish_oem_diag_collector): + mock_collect.return_value = (None, None, "Task timeout") + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["AllLogs"]) + ) + assert result.status == ExecutionStatus.ERROR + assert "0/1 types collected" in result.message + assert data is not None + assert data.results["AllLogs"].success is False + assert data.results["AllLogs"].error == "Task timeout" + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +def test_redfish_oem_diag_collector_mixed_success_fail(mock_collect, redfish_oem_diag_collector): + def side_effect(conn, log_service_path, oem_diagnostic_type, **kwargs): + if oem_diagnostic_type == "JournalControl": + return (b"data", {}, None) + return (None, None, "Not supported") + + mock_collect.side_effect = side_effect + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["JournalControl", "AllLogs"]) + ) + assert result.status == ExecutionStatus.OK + assert "1/2 types collected" in result.message + assert data.results["JournalControl"].success is True + assert data.results["AllLogs"].success is False + assert data.results["AllLogs"].error == "Not supported" + assert mock_collect.call_count == 2 + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +def test_redfish_oem_diag_collector_passes_args_to_connection( + mock_collect, redfish_oem_diag_collector, redfish_conn_mock +): + mock_collect.return_value = (b"", {}, None) + args = RedfishOemDiagCollectorArgs( + log_service_path="redfish/v1/Systems/1/LogServices/DiagLogs", + oem_diagnostic_types=["JournalControl"], + task_timeout_s=300, + ) + redfish_oem_diag_collector.collect_data(args=args) + mock_collect.assert_called_once() + call_kw = mock_collect.call_args[1] + assert call_kw["log_service_path"] == "redfish/v1/Systems/1/LogServices/DiagLogs" + assert call_kw["oem_diagnostic_type"] == "JournalControl" + assert call_kw["task_timeout_s"] == 300 + assert call_kw["output_dir"] is None + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +def test_redfish_oem_diag_collector_output_dir_is_diag_logs( + mock_collect, system_info, redfish_conn_mock, tmp_path +): + mock_collect.return_value = (b"log bytes", {}, None) + collector = RedfishOemDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + log_path=str(tmp_path), + ) + collector.collect_data(args=RedfishOemDiagCollectorArgs(oem_diagnostic_types=["AllLogs"])) + output_dir = mock_collect.call_args.kwargs["output_dir"] + assert output_dir == (tmp_path / "diag_logs").resolve() + assert not output_dir.exists() + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.clear_redfish_logs") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector._parse_clear_endpoint") +def test_clear_logs_called_after_successful_collection( + mock_parse, mock_clear, mock_collect, redfish_oem_diag_collector, redfish_conn_mock +): + from nodescraper.connection.redfish import RedfishGetResult + from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + ) + + mock_collect.return_value = (b"data", {}, None) + mock_parse.return_value = ClearLogEndpoint(path="/clear/path") + mock_clear.return_value = [ClearLogResult(path="/clear/path", success=True, status_code=200)] + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/svc", success=True, data={"Actions": {}}, status_code=200 + ) + + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs( + oem_diagnostic_types=["AllLogs"], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.OK + mock_clear.assert_called_once() + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.clear_redfish_logs") +def test_clear_logs_not_called_when_collection_fails( + mock_clear, mock_collect, redfish_oem_diag_collector +): + mock_collect.return_value = (None, None, "Task timeout") + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs( + oem_diagnostic_types=["AllLogs"], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.ERROR + mock_clear.assert_not_called() + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.clear_redfish_logs") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector._parse_clear_endpoint") +def test_clear_logs_called_on_partial_success( + mock_parse, mock_clear, mock_collect, redfish_oem_diag_collector, redfish_conn_mock +): + """Clear runs when at least one type succeeded — mirrors AMC: clear the log service + whose collections (even partial) are complete.""" + from nodescraper.connection.redfish import RedfishGetResult + from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + ) + + def side_effect(conn, log_service_path, oem_diagnostic_type, **kwargs): + if oem_diagnostic_type == "AllLogs": + return (b"data", {}, None) + return (None, None, "Not supported") + + mock_collect.side_effect = side_effect + mock_parse.return_value = ClearLogEndpoint(path="/clear/path") + mock_clear.return_value = [ClearLogResult(path="/clear/path", success=True, status_code=200)] + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/svc", success=True, data={"Actions": {}}, status_code=200 + ) + + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs( + oem_diagnostic_types=["AllLogs", "JournalControl"], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.OK + assert data.results["AllLogs"].success is True + assert data.results["JournalControl"].success is False + mock_clear.assert_called_once() + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.clear_redfish_logs") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector._parse_clear_endpoint") +def test_clear_log_failure_downgrades_to_warning( + mock_parse, mock_clear, mock_collect, redfish_oem_diag_collector, redfish_conn_mock +): + from nodescraper.connection.redfish import RedfishGetResult + from nodescraper.connection.redfish.redfish_clear_log import ( + ClearLogEndpoint, + ClearLogResult, + ) + from nodescraper.enums import EventPriority + + mock_collect.return_value = (b"data", {}, None) + mock_parse.return_value = ClearLogEndpoint(path="/clear/path") + mock_clear.return_value = [ + ClearLogResult( + path="/clear/path", success=False, status_code=500, error="POST returned 500" + ) + ] + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path="/svc", success=True, data={"Actions": {}}, status_code=200 + ) + + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs( + oem_diagnostic_types=["AllLogs"], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.WARNING + assert "log clear failed" in result.message + warning_events = [e for e in result.events if e.priority == EventPriority.WARNING] + assert any("Failed to clear" in e.description for e in warning_events) + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.clear_redfish_logs") +def test_clear_logs_skipped_when_passive_interaction( + mock_clear, mock_collect, system_info, redfish_conn_mock +): + from nodescraper.enums import SystemInteractionLevel + + mock_collect.return_value = (b"data", {}, None) + collector = RedfishOemDiagCollector( + system_info=system_info, + connection=redfish_conn_mock, + system_interaction_level=SystemInteractionLevel.PASSIVE, + ) + result, data = collector.collect_data( + args=RedfishOemDiagCollectorArgs( + oem_diagnostic_types=["AllLogs"], + clear_logs_after_collection=True, + ) + ) + assert result.status == ExecutionStatus.OK + mock_clear.assert_not_called() + + +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.collect_oem_diagnostic_data") +@patch("nodescraper.plugins.ooband.redfish_oem_diag.oem_diag_collector.clear_redfish_logs") +def test_clear_logs_not_called_when_disabled(mock_clear, mock_collect, redfish_oem_diag_collector): + mock_collect.return_value = (b"data", {}, None) + result, data = redfish_oem_diag_collector.collect_data( + args=RedfishOemDiagCollectorArgs( + oem_diagnostic_types=["AllLogs"], + clear_logs_after_collection=False, + ) + ) + assert result.status == ExecutionStatus.OK + mock_clear.assert_not_called() diff --git a/test/unit/plugin/test_regex_search_analyzer.py b/test/unit/plugin/test_regex_search_analyzer.py index e93b93da..c9ccb32c 100644 --- a/test/unit/plugin/test_regex_search_analyzer.py +++ b/test/unit/plugin/test_regex_search_analyzer.py @@ -1,196 +1,197 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### -import logging -import os -import tempfile - -from nodescraper.enums.executionstatus import ExecutionStatus -from nodescraper.plugins.inband.regex_search.analyzer_args import ( - RegexSearchAnalyzerArgs, -) -from nodescraper.plugins.inband.regex_search.regex_search_analyzer import ( - RegexSearchAnalyzer, -) -from nodescraper.plugins.inband.regex_search.regex_search_data import RegexSearchData -from nodescraper.plugins.inband.regex_search.regex_search_plugin import ( - RegexSearchPlugin, -) - -EXPECTED_MISSING_ANALYSIS_MSG = "Analysis args need to be provided for the analyzer to run" - - -def test_regex_search_data_from_file(): - with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False, encoding="utf-8") as f: - f.write("alpha\nbeta ERROR gamma\n") - path = f.name - try: - data = RegexSearchData.import_model(path) - assert "ERROR" in data.content - assert os.path.basename(path) in data.files - assert data.data_root == os.path.dirname(path) - finally: - os.unlink(path) - - -def test_regex_search_data_from_directory(): - with tempfile.TemporaryDirectory() as tmp: - with open(f"{tmp}/a.txt", "w", encoding="utf-8") as f: - f.write("one") - with open(f"{tmp}/b.txt", "w", encoding="utf-8") as f: - f.write("two") - data = RegexSearchData.import_model(tmp) - assert data.data_root == os.path.abspath(tmp) - assert set(data.files.keys()) == {"a.txt", "b.txt"} - assert data.files["a.txt"] == "one" - assert data.files["b.txt"] == "two" - assert "===== a.txt =====" in data.content - assert "===== b.txt =====" in data.content - - -def test_regex_search_analyzer_match(system_info): - data = RegexSearchData(content="line1\nFATAL: boom\nline3") - analyzer = RegexSearchAnalyzer(system_info=system_info) - args = RegexSearchAnalyzerArgs( - error_regex=[{"regex": r"FATAL:.*", "message": "fatal seen"}], - ) - result = analyzer.analyze_data(data, args) - assert result.status == ExecutionStatus.ERROR - assert "task detected errors" in result.message - assert "fatal seen" in result.message - assert len(result.events) == 1 - assert result.events[0].description == "fatal seen" - - -def test_regex_search_analyzer_missing_args(system_info): - data = RegexSearchData(content="x") - analyzer = RegexSearchAnalyzer(system_info=system_info) - result = analyzer.analyze_data(data, None) - assert result.status == ExecutionStatus.NOT_RAN - assert result.message == EXPECTED_MISSING_ANALYSIS_MSG - - result = analyzer.analyze_data(data, RegexSearchAnalyzerArgs(error_regex=None)) - assert result.status == ExecutionStatus.NOT_RAN - assert result.message == EXPECTED_MISSING_ANALYSIS_MSG - - result = analyzer.analyze_data(data, RegexSearchAnalyzerArgs(error_regex=[])) - assert result.status == ExecutionStatus.NOT_RAN - assert result.message == EXPECTED_MISSING_ANALYSIS_MSG - - -def test_regex_search_plugin_missing_error_regex_not_ran_and_warning( - system_info, logger, caplog, tmp_path -): - log_file = tmp_path / "sample.log" - log_file.write_text("line\n", encoding="utf-8") - plugin = RegexSearchPlugin(system_info=system_info, logger=logger) - with caplog.at_level(logging.WARNING, logger=logger.name): - out = plugin.run( - collection=False, - analysis=True, - data=str(log_file), - analysis_args=None, - ) - assert out.result_data.analysis_result.status == ExecutionStatus.NOT_RAN - assert out.result_data.analysis_result.message == EXPECTED_MISSING_ANALYSIS_MSG - assert any( - "analysis args need to be provided" in r.getMessage().lower() for r in caplog.records - ) - - -def test_regex_search_plugin_empty_analysis_args_dict_not_ran(system_info, logger, tmp_path): - log_file = tmp_path / "sample.log" - log_file.write_text("line\n", encoding="utf-8") - plugin = RegexSearchPlugin(system_info=system_info, logger=logger) - out = plugin.run( - collection=False, - analysis=True, - data=str(log_file), - analysis_args={}, - ) - assert out.result_data.analysis_result.status == ExecutionStatus.NOT_RAN - assert out.result_data.analysis_result.message == EXPECTED_MISSING_ANALYSIS_MSG - - -def test_regex_search_plugin_no_data_warns_and_data_message(system_info, logger, caplog): - plugin = RegexSearchPlugin(system_info=system_info, logger=logger) - with caplog.at_level(logging.WARNING, logger=logger.name): - out = plugin.run( - collection=False, - analysis=True, - data=None, - analysis_args=None, - ) - assert out.result_data.analysis_result.status == ExecutionStatus.NOT_RAN - assert "No data available to analyze for RegexSearchPlugin" in ( - out.result_data.analysis_result.message - ) - assert any( - "analysis args need to be provided" in r.getMessage().lower() for r in caplog.records - ) - - -def test_regex_search_plugin_analyzer_only(system_info, logger): - with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False, encoding="utf-8") as f: - f.write("match_me_here\n") - path = f.name - try: - plugin = RegexSearchPlugin(system_info=system_info, logger=logger) - out = plugin.run( - collection=False, - analysis=True, - data=path, - analysis_args={ - "error_regex": [{"regex": r"match_me_here", "message": "found"}], - }, - ) - assert out.status == ExecutionStatus.ERROR - assert "Analysis error:" in out.message - assert "found" in out.message - assert out.result_data.analysis_result.status == ExecutionStatus.ERROR - assert len(out.result_data.analysis_result.events) == 1 - desc = out.result_data.analysis_result.events[0].description - assert "found" in desc - assert "[file:" in desc - assert path.replace("\\", "/") in desc.replace("\\", "/") - finally: - os.unlink(path) - - -def test_regex_search_multi_file_event_paths(system_info): - with tempfile.TemporaryDirectory() as tmp: - open(os.path.join(tmp, "clean.log"), "w", encoding="utf-8").write("ok\n") - open(os.path.join(tmp, "bad.log"), "w", encoding="utf-8").write("ERROR: boom\n") - data = RegexSearchData.import_model(tmp) - analyzer = RegexSearchAnalyzer(system_info=system_info) - args = RegexSearchAnalyzerArgs( - error_regex=[{"regex": r"ERROR[: ].*", "message": "err line"}], - ) - result = analyzer.analyze_data(data, args) - assert result.status == ExecutionStatus.ERROR - assert len(result.events) == 1 - assert "err line" in result.events[0].description - assert "[file:" in result.events[0].description - assert "bad.log" in result.events[0].description +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging +import os +import tempfile + +from nodescraper.enums.executionstatus import ExecutionStatus +from nodescraper.plugins.inband.regex_search.analyzer_args import ( + RegexSearchAnalyzerArgs, +) +from nodescraper.plugins.inband.regex_search.regex_search_analyzer import ( + RegexSearchAnalyzer, +) +from nodescraper.plugins.inband.regex_search.regex_search_data import RegexSearchData +from nodescraper.plugins.inband.regex_search.regex_search_plugin import ( + RegexSearchPlugin, +) + +EXPECTED_MISSING_ANALYSIS_MSG = "Analysis args need to be provided for the analyzer to run" + + +def test_regex_search_data_from_file(): + with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False, encoding="utf-8") as f: + f.write("alpha\nbeta ERROR gamma\n") + path = f.name + try: + data = RegexSearchData.import_model(path) + assert "ERROR" in data.content + assert os.path.basename(path) in data.files + assert data.data_root == os.path.dirname(path) + finally: + os.unlink(path) + + +def test_regex_search_data_from_directory(): + with tempfile.TemporaryDirectory() as tmp: + with open(f"{tmp}/a.txt", "w", encoding="utf-8") as f: + f.write("one") + with open(f"{tmp}/b.txt", "w", encoding="utf-8") as f: + f.write("two") + data = RegexSearchData.import_model(tmp) + assert data.data_root == os.path.abspath(tmp) + assert set(data.files.keys()) == {"a.txt", "b.txt"} + assert data.files["a.txt"] == "one" + assert data.files["b.txt"] == "two" + assert "===== a.txt =====" in data.content + assert "===== b.txt =====" in data.content + + +def test_regex_search_analyzer_match(system_info): + data = RegexSearchData(content="line1\nFATAL: boom\nline3") + analyzer = RegexSearchAnalyzer(system_info=system_info) + args = RegexSearchAnalyzerArgs( + error_regex=[{"regex": r"FATAL:.*", "message": "fatal seen"}], + ) + result = analyzer.analyze_data(data, args) + assert result.status == ExecutionStatus.ERROR + assert result.message == "task detected errors (1 errors: fatal seen)" + assert len(result.events) == 1 + assert result.events[0].description == "fatal seen" + + +def test_regex_search_analyzer_missing_args(system_info): + data = RegexSearchData(content="x") + analyzer = RegexSearchAnalyzer(system_info=system_info) + result = analyzer.analyze_data(data, None) + assert result.status == ExecutionStatus.NOT_RAN + assert result.message == EXPECTED_MISSING_ANALYSIS_MSG + + result = analyzer.analyze_data(data, RegexSearchAnalyzerArgs(error_regex=None)) + assert result.status == ExecutionStatus.NOT_RAN + assert result.message == EXPECTED_MISSING_ANALYSIS_MSG + + result = analyzer.analyze_data(data, RegexSearchAnalyzerArgs(error_regex=[])) + assert result.status == ExecutionStatus.NOT_RAN + assert result.message == EXPECTED_MISSING_ANALYSIS_MSG + + +def test_regex_search_plugin_missing_error_regex_not_ran_and_warning( + system_info, logger, caplog, tmp_path +): + log_file = tmp_path / "sample.log" + log_file.write_text("line\n", encoding="utf-8") + plugin = RegexSearchPlugin(system_info=system_info, logger=logger) + with caplog.at_level(logging.WARNING, logger=logger.name): + out = plugin.run( + collection=False, + analysis=True, + data=str(log_file), + analysis_args=None, + ) + assert out.result_data.analysis_result.status == ExecutionStatus.NOT_RAN + assert out.result_data.analysis_result.message == EXPECTED_MISSING_ANALYSIS_MSG + assert any( + "analysis args need to be provided" in r.getMessage().lower() for r in caplog.records + ) + + +def test_regex_search_plugin_empty_analysis_args_dict_not_ran(system_info, logger, tmp_path): + log_file = tmp_path / "sample.log" + log_file.write_text("line\n", encoding="utf-8") + plugin = RegexSearchPlugin(system_info=system_info, logger=logger) + out = plugin.run( + collection=False, + analysis=True, + data=str(log_file), + analysis_args={}, + ) + assert out.result_data.analysis_result.status == ExecutionStatus.NOT_RAN + assert out.result_data.analysis_result.message == EXPECTED_MISSING_ANALYSIS_MSG + + +def test_regex_search_plugin_no_data_warns_and_data_message(system_info, logger, caplog): + plugin = RegexSearchPlugin(system_info=system_info, logger=logger) + with caplog.at_level(logging.WARNING, logger=logger.name): + out = plugin.run( + collection=False, + analysis=True, + data=None, + analysis_args=None, + ) + assert out.result_data.analysis_result.status == ExecutionStatus.NOT_RAN + assert "No data available to analyze for RegexSearchPlugin" in ( + out.result_data.analysis_result.message + ) + assert any( + "analysis args need to be provided" in r.getMessage().lower() for r in caplog.records + ) + + +def test_regex_search_plugin_analyzer_only(system_info, logger): + with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False, encoding="utf-8") as f: + f.write("match_me_here\n") + path = f.name + try: + plugin = RegexSearchPlugin(system_info=system_info, logger=logger) + out = plugin.run( + collection=False, + analysis=True, + data=path, + analysis_args={ + "error_regex": [{"regex": r"match_me_here", "message": "found"}], + }, + ) + assert out.status == ExecutionStatus.ERROR + assert out.message.startswith( + "Analysis error: task detected errors (1 errors: found [file: " + ) + assert out.message.endswith("])") + assert out.result_data.analysis_result.status == ExecutionStatus.ERROR + assert len(out.result_data.analysis_result.events) == 1 + desc = out.result_data.analysis_result.events[0].description + assert "found" in desc + assert "[file:" in desc + assert path.replace("\\", "/") in desc.replace("\\", "/") + finally: + os.unlink(path) + + +def test_regex_search_multi_file_event_paths(system_info): + with tempfile.TemporaryDirectory() as tmp: + open(os.path.join(tmp, "clean.log"), "w", encoding="utf-8").write("ok\n") + open(os.path.join(tmp, "bad.log"), "w", encoding="utf-8").write("ERROR: boom\n") + data = RegexSearchData.import_model(tmp) + analyzer = RegexSearchAnalyzer(system_info=system_info) + args = RegexSearchAnalyzerArgs( + error_regex=[{"regex": r"ERROR[: ].*", "message": "err line"}], + ) + result = analyzer.analyze_data(data, args) + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + assert "err line" in result.events[0].description + assert "[file:" in result.events[0].description + assert "bad.log" in result.events[0].description diff --git a/test/unit/plugin/test_se_runner.py b/test/unit/plugin/test_se_runner.py index c972a27a..bb750bb5 100644 --- a/test/unit/plugin/test_se_runner.py +++ b/test/unit/plugin/test_se_runner.py @@ -24,6 +24,7 @@ # ############################################################################### import json +import logging from pathlib import Path from types import SimpleNamespace from typing import Any @@ -195,6 +196,63 @@ def test_resolve_hub_events_payload_honors_prefer_rf_events_false(): assert payload[0]["location"] == EXAMPLE_EVENTS[0].serviceable_unit +def _rf_member(odata_id, *identifiers): + return {"@odata.id": odata_id, "Oem": {"AMD": {"AMDFieldIdentifiers": list(identifiers)}}} + + +_UNIT = {"@odata.id": "/redfish/v1/Chassis/Instinct_EAM_0"} + + +def test_resolve_hub_events_payload_waives_afid_without_serviceable_units(caplog): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_hub_events_payload, + ) + + good = _rf_member("/entries/1", {"AFID": 11101, "ServiceableUnits": [_UNIT]}) + bad = _rf_member("/entries/2", {"AFID": 11045, "ServiceableUnits": []}) + with caplog.at_level(logging.WARNING): + payload = resolve_hub_events_payload( + afid_events=EXAMPLE_EVENTS, rf_events=[good, bad], prefer_rf_events=True + ) + assert payload == [good] + assert "Waiving AFID 11045 from /entries/2" in caplog.text + + +def test_resolve_hub_events_payload_keeps_valid_entries_on_mixed_member(caplog): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_hub_events_payload, + ) + + member = _rf_member( + "/entries/3", + {"AFID": 11101, "ServiceableUnits": [_UNIT]}, + {"AFID": 11045}, + {"AFID": 11046, "ServiceableUnits": [{"Name": "no odata id"}]}, + ) + with caplog.at_level(logging.WARNING): + payload = resolve_hub_events_payload( + afid_events=EXAMPLE_EVENTS, rf_events=[member], prefer_rf_events=True + ) + identifiers = payload[0]["Oem"]["AMD"]["AMDFieldIdentifiers"] + assert [entry["AFID"] for entry in identifiers] == [11101] + assert len(member["Oem"]["AMD"]["AMDFieldIdentifiers"]) == 3 + assert "Waiving AFID 11045" in caplog.text + assert "Waiving AFID 11046" in caplog.text + + +def test_resolve_hub_events_payload_falls_back_to_afid_events_when_all_waived(): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_hub_events_payload, + ) + + payload = resolve_hub_events_payload( + afid_events=EXAMPLE_EVENTS[:1], + rf_events=[_rf_member("/entries/4", {"AFID": 11045})], + prefer_rf_events=True, + ) + assert payload[0]["afid"] == EXAMPLE_EVENTS[0].afid + + def test_resolved_hub_options_explicit_fields_override_options_bag(): args = ServiceabilityAnalyzerArgs( hub_python_module="dummy.test.module", @@ -307,9 +365,10 @@ def test_serviceability_block_from_entry_point_hub_uses_sag_labels(tmp_path): assert triage.service_action_steps assert triage.service_action_category == "Reflash" lines = format_serviceability_solution_lines(block) - assert "Hub triage results:" in lines - assert "priority=" in "\n".join(lines) - assert "step 0:" in "\n".join(lines) + assert any( + "prioritized recommendation(s); see recommendation tables below" in line for line in lines + ) + assert "Hub triage results:" not in lines def test_serviceability_block_from_service_result(): @@ -598,13 +657,113 @@ def test_mi3xx_analyzer_runs_python_hub(system_info): def test_mi3xx_analyzer_writes_serviceability_json(tmp_path, system_info): + hub_response = { + "schema_version": "1.0", + "status": "ok", + "triage": {"top": [], "results": [], "multi_afid_summary": []}, + } data = ServiceabilityDataModel( afid_events=EXAMPLE_EVENTS[:1], serviceability=ServiceabilityBlock( afid_events=EXAMPLE_EVENTS[:1], solution=[], + solution_reasoning="node-scraper summary", + hub_analyze_response=hub_response, ), ) data.log_model(str(tmp_path)) payload = json.loads((tmp_path / "serviceability.json").read_text(encoding="utf-8")) - assert payload["afid_events"][0]["afid"] == DUMMY_AFID_A + assert payload == hub_response + assert "afid_events" not in payload + + +class _ValidatingHub: + name = "AFSE" + + def __init__(self, response=None, default_path=None): + self._response = response if response is not None else {"status": "ok"} + self._default_path = default_path + self.validated: list[str] = [] + + def validate(self, afid_sag_path): + self.validated.append(afid_sag_path) + return self._response + + def default_sag_path(self): + return self._default_path + + +def test_validate_afid_sag_with_hub_accepts_ok_status(tmp_path): + from nodescraper.plugins.serviceability.se_runner import validate_afid_sag_with_hub + + hub = _ValidatingHub() + assert validate_afid_sag_with_hub(hub, str(tmp_path / "sag.json")) is True + assert hub.validated == [str(tmp_path / "sag.json")] + + +def test_validate_afid_sag_with_hub_raises_on_error_status(): + from nodescraper.plugins.serviceability.se_runner import ( + HubRunError, + validate_afid_sag_with_hub, + ) + + hub = _ValidatingHub({"status": "error", "error": {"code": "E1", "message": "bad schema"}}) + with pytest.raises(HubRunError, match="AFSE SAG validation failed: bad schema"): + validate_afid_sag_with_hub(hub, "/tmp/sag.json") + + +def test_validate_afid_sag_with_hub_skips_hub_without_validate(): + from nodescraper.plugins.serviceability.se_runner import validate_afid_sag_with_hub + + assert validate_afid_sag_with_hub(SimpleNamespace(name="Other"), "/tmp/sag.json") is False + + +def test_resolve_runtime_afid_sag_path_prefers_configured_path(tmp_path): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_runtime_afid_sag_path, + ) + + args = ServiceabilityAnalyzerArgs( + hub_entry_point="afse", afid_sag_path=str(tmp_path / "x.json") + ) + assert resolve_runtime_afid_sag_path(args) == str(tmp_path / "x.json") + + +def test_resolve_runtime_afid_sag_path_uses_hub_default(monkeypatch, tmp_path): + from nodescraper.plugins.serviceability import se_runner + + hub_default = str(tmp_path / "hub_default.json") + monkeypatch.setattr( + se_runner, + "load_hub_from_entry_point", + lambda name: _ValidatingHub(default_path=hub_default), + ) + args = ServiceabilityAnalyzerArgs(hub_entry_point="afse") + assert se_runner.resolve_runtime_afid_sag_path(args) == hub_default + + +def test_resolve_runtime_afid_sag_path_falls_back_to_builtin_default(monkeypatch): + from nodescraper.plugins.serviceability import se_runner + from nodescraper.plugins.serviceability.afid_sag_paths import DEFAULT_AFID_SAG_PATH + + monkeypatch.setattr( + se_runner, "load_hub_from_entry_point", lambda name: SimpleNamespace(name="AFSE") + ) + args = ServiceabilityAnalyzerArgs(hub_entry_point="afse") + assert se_runner.resolve_runtime_afid_sag_path(args) == DEFAULT_AFID_SAG_PATH + + +def test_run_entry_point_hub_stops_when_afse_rejects_sag(monkeypatch, tmp_path): + from nodescraper.plugins.serviceability import se_runner + + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + hub = _ValidatingHub({"status": "error", "error": {"message": "schema mismatch"}}) + monkeypatch.setattr(se_runner, "load_hub_from_entry_point", lambda name: hub) + with pytest.raises(se_runner.HubRunError, match="schema mismatch"): + se_runner.run_entry_point_hub( + hub_entry_point="afse", + afid_events=EXAMPLE_EVENTS[:1], + afid_sag_path=str(sag), + ) + assert hub.validated == [str(sag.resolve())] diff --git a/test/unit/plugin/test_serviceability_api.py b/test/unit/plugin/test_serviceability_api.py new file mode 100644 index 00000000..cf8e1d15 --- /dev/null +++ b/test/unit/plugin/test_serviceability_api.py @@ -0,0 +1,94 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from serviceability_dummy_data import DUMMY_AFID_A, DUMMY_UNIT_A + +from nodescraper.plugins.serviceability.se_models import ( + AfidEvent, + HubTriageResult, + ServiceabilityBlock, +) +from nodescraper.plugins.serviceability.serviceability_api import ( + export_serviceability_json, +) + + +def test_export_serviceability_json_is_only_the_afse_response(): + hub_response = { + "schema_version": "1.0", + "status": "ok", + "triage": { + "top": [{"afid": DUMMY_AFID_A, "location": DUMMY_UNIT_A, "se_sort_priority": 1000}], + "results": [{"afid": DUMMY_AFID_A, "location": DUMMY_UNIT_A, "se_sort_priority": 1000}], + "multi_afid_summary": [], + }, + "pid": "SAG-00000", + "revision": "1.0.0", + } + block = ServiceabilityBlock( + afid_events=[ + AfidEvent( + afid=DUMMY_AFID_A, serviceable_unit="Instinct_EAM_0", time="2025-01-01T00:00:00Z" + ) + ], + solution_reasoning="node-scraper summary", + hub_version="0.2.0", + hub_analyze_response=hub_response, + hub_top_results=[ + HubTriageResult( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + service_action_num=199, + hub_sort_priority=1000, + ) + ], + hub_triage_results=[ + HubTriageResult( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + service_action_num=199, + hub_sort_priority=1000, + ) + ], + ) + + payload = export_serviceability_json(block) + + assert payload == hub_response + assert "afid_events" not in payload + assert "solution" not in payload + assert "hub_triage_results" not in payload + + +def test_export_serviceability_json_omits_file_without_afse_response(): + block = ServiceabilityBlock( + afid_events=[ + AfidEvent( + afid=DUMMY_AFID_A, serviceable_unit="Instinct_EAM_0", time="2025-01-01T00:00:00Z" + ) + ] + ) + + assert export_serviceability_json(block) is None diff --git a/test/unit/plugin/test_serviceability_recommendations_table.py b/test/unit/plugin/test_serviceability_recommendations_table.py new file mode 100644 index 00000000..17971e82 --- /dev/null +++ b/test/unit/plugin/test_serviceability_recommendations_table.py @@ -0,0 +1,373 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +import logging + +from serviceability_dummy_data import ( + DUMMY_AFID_A, + DUMMY_AFID_B, + DUMMY_SERVICE_ACTION_NUM, + DUMMY_TIER_CRITICAL, + DUMMY_TIER_LABEL, + DUMMY_UNIT_A, + DUMMY_UNIT_B, +) + +from nodescraper.enums import ExecutionStatus +from nodescraper.models import DataPluginResult, PluginResult +from nodescraper.plugins.serviceability.se_models import ( + HubTriageResult, + ServiceabilityBlock, +) +from nodescraper.plugins.serviceability.serviceability_api import ( + build_top_service_actions, + split_recommendation_actions, +) +from nodescraper.plugins.serviceability.serviceability_data import ( + ServiceabilityDataModel, +) +from nodescraper.plugins.serviceability.serviceability_recommendations_table import ( + emit_serviceability_recommendation_tables, + render_serviceability_recommendation_tables, + render_serviceability_recommendation_tables_for_plugin_results, +) + + +def _row( + *, + afid: int, + location: str, + sort_priority: int, + priority: int, + title: str, +) -> HubTriageResult: + return HubTriageResult( + afid=afid, + location=location, + service_action_num=DUMMY_SERVICE_ACTION_NUM, + service_action_title=title, + priority=priority, + sa_severity=20, + tier_label=DUMMY_TIER_CRITICAL if priority == 1 else DUMMY_TIER_LABEL, + hub_sort_priority=sort_priority, + ) + + +def test_split_recommendation_rows_uses_hub_top(): + block = ServiceabilityBlock( + hub_top_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ) + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ), + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=2000, + priority=20, + title="Update FW", + ), + ], + ) + + top, additional = split_recommendation_actions(block) + + assert len(top) == 1 + assert top[0].afid == DUMMY_AFID_A + assert len(additional) == 1 + assert additional[0].afid == DUMMY_AFID_B + + +def test_render_serviceability_recommendation_tables_prints_only_top_action(): + block = ServiceabilityBlock( + hub_top_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ) + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ), + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=2000, + priority=20, + title="Update FW", + ), + ], + ) + + output = render_serviceability_recommendation_tables(block) + + assert "Top Hub service action (rank 1)" in output + assert f"{DUMMY_AFID_A}: Contact Support" in output + assert "Additional Hub service actions" not in output + assert f"{DUMMY_AFID_B}: Update FW" not in output + + +def test_table_prints_afse_triage_top_service_action(): + block = ServiceabilityBlock( + hub_analyze_response={ + "status": "ok", + "triage": { + "top": [ + { + "afid": DUMMY_AFID_A, + "location": DUMMY_UNIT_A, + "service_action_num": DUMMY_SERVICE_ACTION_NUM, + "priority": 20, + "sa_severity": 2, + "tier_label": "Secondary", + "service_action": { + "title": "Contact AMD", + "severity": 2, + "steps": [ + {"description": "Contact your AMD service provider."}, + ], + }, + } + ], + "results": [ + { + "afid": DUMMY_AFID_B, + "location": DUMMY_UNIT_B, + "service_action_num": 1, + } + ], + }, + }, + hub_top_results=[ + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=9000, + priority=1, + title="Update FW", + ) + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=9000, + priority=1, + title="Update FW", + ) + ], + ) + + output = render_serviceability_recommendation_tables(block) + + assert "Top Hub service action (rank 1)" in output + assert f"{DUMMY_AFID_A}: Contact AMD" in output + assert "Contact your AMD service provider." in output + assert f"{DUMMY_AFID_B}: Update FW" not in output + + +def test_missing_afse_top_skips_table_and_logs_warning(caplog): + block = ServiceabilityBlock( + hub_analyze_response={ + "status": "ok", + "triage": { + "top": [], + "results": [ + { + "afid": DUMMY_AFID_B, + "location": DUMMY_UNIT_B, + "service_action_num": 1, + } + ], + }, + }, + hub_top_results=[ + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=9000, + priority=1, + title="Update FW", + ) + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=9000, + priority=1, + title="Update FW", + ) + ], + ) + plugin_results = [ + PluginResult( + status=ExecutionStatus.OK, + source="Mi4xxServiceabilityPlugin", + message="ok", + result_data=DataPluginResult( + system_data=ServiceabilityDataModel(serviceability=block), + ), + ) + ] + + with caplog.at_level(logging.WARNING, logger="nodescraper"): + output = render_serviceability_recommendation_tables_for_plugin_results(plugin_results) + + assert output == "" + assert "Rank" not in output + assert "Update FW" not in output + assert "AFSE did not return a top action item" in caplog.text + assert render_serviceability_recommendation_tables(block) == "" + + +def test_build_top_service_actions_includes_tied_entries(): + block = ServiceabilityBlock( + hub_top_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Replace Unit", + ), + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=1000, + priority=1, + title="Replace Unit", + ), + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Replace Unit", + ), + _row( + afid=DUMMY_AFID_B, + location=DUMMY_UNIT_B, + sort_priority=1000, + priority=1, + title="Replace Unit", + ), + ], + ) + + top = build_top_service_actions(block) + + assert len(top) == 2 + assert {action.afid for action in top} == {DUMMY_AFID_A, DUMMY_AFID_B} + + +def test_render_serviceability_recommendation_tables_for_plugin_results(): + block = ServiceabilityBlock( + hub_top_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ) + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ), + ], + ) + plugin_results = [ + PluginResult( + status=ExecutionStatus.OK, + source="Mi4xxServiceabilityPlugin", + message="ok", + result_data=DataPluginResult( + system_data=ServiceabilityDataModel(serviceability=block), + ), + ) + ] + + output = render_serviceability_recommendation_tables_for_plugin_results(plugin_results) + + assert "Top Hub service action" in output + assert "Contact Support" in output + + +def test_emit_serviceability_recommendation_tables_writes_stdout(capsys): + block = ServiceabilityBlock( + hub_top_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ) + ], + hub_triage_results=[ + _row( + afid=DUMMY_AFID_A, + location=DUMMY_UNIT_A, + sort_priority=1000, + priority=1, + title="Contact Support", + ), + ], + ) + + emit_serviceability_recommendation_tables(block) + captured = capsys.readouterr() + + assert "Top Hub service action" in captured.out + assert "Rank" in captured.out + assert "Contact Support" in captured.out diff --git a/test/unit/plugin/test_storage_analyzer.py b/test/unit/plugin/test_storage_analyzer.py index bf2b4ef6..1253dd8d 100644 --- a/test/unit/plugin/test_storage_analyzer.py +++ b/test/unit/plugin/test_storage_analyzer.py @@ -61,11 +61,45 @@ def multiple_dev(): ) +@pytest.fixture +def loop_dev_model(): + return StorageDataModel( + storage_data={ + "/dev/nvme0n1p2": DeviceStorageData( + total=943441641472, + free=869796294656, + used=25645850624, + percent=3, + ), + "/dev/loop1": DeviceStorageData( + total=1966080, + free=0, + used=1966080, + percent=100.0, + ), + "/dev/loop2": DeviceStorageData( + total=52559872, + free=0, + used=52559872, + percent=100.0, + ), + } + ) + + @pytest.fixture def analyzer(system_info): return StorageAnalyzer(system_info=system_info) +def _critical_devices(result): + return sorted( + e.data["offending_device"]["device"] + for e in result.events + if e.priority == EventPriority.CRITICAL + ) + + def test_filter_exact_match(analyzer): assert analyzer._matches_device_filter("foo", ["foo", "bar"], regex_match=False) assert not analyzer._matches_device_filter("baz", ["foo", "bar"], regex_match=False) @@ -166,3 +200,123 @@ def test_device_filter(analyzer, model_obj): ) result2 = analyzer.analyze_data(model_obj, args2) assert result2.status == ExecutionStatus.OK + + +def test_skip_loop_devices_default_true(): + assert StorageAnalyzerArgs().skip_loop_devices is True + + +@pytest.mark.parametrize( + "args", + [StorageAnalyzerArgs(min_required_free_space_prct=10), None], + ids=["explicit_args", "args_none"], +) +def test_loop_devices_skipped_by_default(analyzer, loop_dev_model, args): + result = analyzer.analyze_data(loop_dev_model, args) + assert result.status == ExecutionStatus.OK + assert result.events == [] + assert not any(e.priority == EventPriority.CRITICAL for e in result.events) + assert "loop" not in result.message + assert result.message == "Sufficient disk space available on [/dev/nvme0n1p2]" + + +def test_loop_devices_checked_when_skip_disabled(analyzer, loop_dev_model): + args = StorageAnalyzerArgs(min_required_free_space_prct=10, skip_loop_devices=False) + result = analyzer.analyze_data(loop_dev_model, args) + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 2 + assert all(e.category == EventCategory.STORAGE.value for e in result.events) + assert all(e.priority == EventPriority.CRITICAL for e in result.events) + assert all("Insufficient disk space" in e.description for e in result.events) + assert _critical_devices(result) == ["/dev/loop1", "/dev/loop2"] + assert result.message.startswith("Insufficient disk space on [/dev/loop1, /dev/loop2]") + + +def test_check_devices_overrides_loop_skip(analyzer, loop_dev_model): + args = StorageAnalyzerArgs(min_required_free_space_prct=10, check_devices=["/dev/loop1"]) + assert args.skip_loop_devices is True + result = analyzer.analyze_data(loop_dev_model, args) + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + event = result.events[0] + assert event.category == EventCategory.STORAGE.value + assert event.priority == EventPriority.CRITICAL + assert "Insufficient disk space" in event.description + assert "/dev/loop1" in event.description + assert event.data["offending_device"]["device"] == "/dev/loop1" + assert result.message.startswith("Insufficient disk space on [/dev/loop1]") + + +def test_loop_skip_does_not_hide_real_full_disk(analyzer, loop_dev_model): + loop_dev_model.storage_data["/dev/sda1"] = DeviceStorageData( + total=1000, free=50, used=950, percent=95 + ) + args = StorageAnalyzerArgs(min_required_free_space_prct=10) + result = analyzer.analyze_data(loop_dev_model, args) + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + event = result.events[0] + assert event.category == EventCategory.STORAGE.value + assert event.priority == EventPriority.CRITICAL + assert "Insufficient disk space" in event.description + assert event.data["offending_device"]["device"] == "/dev/sda1" + assert result.message.startswith("Insufficient disk space on [/dev/sda1]") + assert "loop" not in result.message + + +def test_loop_skip_composes_with_ignore_devices(analyzer, loop_dev_model): + loop_dev_model.storage_data["/dev/sda1"] = DeviceStorageData( + total=1000, free=50, used=950, percent=95 + ) + args = StorageAnalyzerArgs(min_required_free_space_prct=10, ignore_devices=["/dev/sda1"]) + result = analyzer.analyze_data(loop_dev_model, args) + assert result.status == ExecutionStatus.OK + assert result.events == [] + assert result.message == "Sufficient disk space available on [/dev/nvme0n1p2]" + + +@pytest.mark.parametrize( + "device_name, is_loop", + [ + ("/dev/loop0", True), + ("/dev/loop12", True), + ("/dev/loopback-foo", False), + ("loop1", False), + ("/dev/loop", False), + ("/dev/sda1", False), + ("C:", False), + ], +) +def test_is_loop_device(analyzer, device_name, is_loop): + assert analyzer._is_loop_device(device_name) is is_loop + + +@pytest.mark.parametrize( + "device_name, skipped", + [ + ("/dev/loop0", True), + ("/dev/loop12", True), + ("/dev/loopback-foo", False), + ("loop1", False), + ], +) +def test_loop_skip_name_matching(analyzer, device_name, skipped): + model = StorageDataModel( + storage_data={ + device_name: DeviceStorageData(total=1000, free=0, used=1000, percent=100.0), + } + ) + args = StorageAnalyzerArgs(min_required_free_space_prct=10) + result = analyzer.analyze_data(model, args) + if skipped: + assert result.status == ExecutionStatus.OK + assert result.events == [] + assert result.message == "Sufficient disk space available on []" + else: + assert result.status == ExecutionStatus.ERROR + assert len(result.events) == 1 + event = result.events[0] + assert event.category == EventCategory.STORAGE.value + assert event.priority == EventPriority.CRITICAL + assert event.data["offending_device"]["device"] == device_name + assert result.message.startswith(f"Insufficient disk space on [{device_name}]") diff --git a/test/unit/plugin/test_storage_collector.py b/test/unit/plugin/test_storage_collector.py index 02a96c4a..31688aca 100644 --- a/test/unit/plugin/test_storage_collector.py +++ b/test/unit/plugin/test_storage_collector.py @@ -76,6 +76,29 @@ def test_run_linux(collector, conn_mock): ) +def test_run_esxi(collector, conn_mock): + """ESXi parses fixed esxcli filesystem-list columns (Size/Free by position).""" + collector.system_info.os_family = OSFamily.ESXI + conn_mock.run_command.return_value = CommandArtifact( + exit_code=0, + stdout=( + "Mount Point Volume Name UUID Mounted Type Size Free\n" + "------------------- ----------- -------- ------- ------ ---- ----\n" + "/vmfs/volumes/abc datastore1 uuid-1 true VMFS-6 2000 500" + ), + stderr="", + command="esxcli storage filesystem list", + ) + + result, data = collector.collect_data() + assert result.status == ExecutionStatus.OK + assert data == StorageDataModel( + storage_data={ + "/vmfs/volumes/abc": DeviceStorageData(total=2000, free=500, used=1500, percent=75.0) + } + ) + + def test_run_windows(system_info, conn_mock): system_info.os_family = OSFamily.WINDOWS collector = StorageCollector( diff --git a/test/unit/plugin/test_sys_settings_analyzer.py b/test/unit/plugin/test_sys_settings_analyzer.py index 318093c4..17e4b256 100644 --- a/test/unit/plugin/test_sys_settings_analyzer.py +++ b/test/unit/plugin/test_sys_settings_analyzer.py @@ -105,3 +105,23 @@ def test_analyzer_unknown_path(analyzer, sample_data): assert result.status == ExecutionStatus.ERROR assert "mismatch" in result.message.lower() assert "unknown" in result.message + + +def test_analyzer_gpu_vbios_match(analyzer): + path = "/sys/class/drm/card*/device/vbios_version" + data = SysSettingsDataModel(readings={path: "113-M355-01-1K1-040C"}) + args = SysSettingsAnalyzerArgs( + checks=[SysfsCheck(path=path, expected=["113-M355-01-1K1-040C"], name="gpu_vbios")] + ) + result = analyzer.analyze_data(data, args) + assert result.status == ExecutionStatus.OK + + +def test_analyzer_gpu_vbios_mismatch(analyzer): + path = "/sys/class/drm/card*/device/vbios_version" + data = SysSettingsDataModel(readings={path: "wrong"}) + args = SysSettingsAnalyzerArgs( + checks=[SysfsCheck(path=path, expected=["113-M355-01-1K1-040C"], name="gpu_vbios")] + ) + result = analyzer.analyze_data(data, args) + assert result.status == ExecutionStatus.ERROR diff --git a/test/unit/plugin/test_sys_settings_collector.py b/test/unit/plugin/test_sys_settings_collector.py index b2da8b2e..555b79ee 100644 --- a/test/unit/plugin/test_sys_settings_collector.py +++ b/test/unit/plugin/test_sys_settings_collector.py @@ -165,7 +165,7 @@ def run_cmd(cmd, **kwargs): assert "/etc" not in str(seen_commands) -def test_collect_data_glob_path_uses_ls_long(linux_sys_settings_collector): +def test_collect_data_glob_path_falls_back_to_ls_long(linux_sys_settings_collector): seen_commands = [] def run_cmd(cmd, **kwargs): @@ -174,7 +174,7 @@ def run_cmd(cmd, **kwargs): return make_artifact( 0, "lrwxrwxrwx 1 root root 0 Jan 1 00:00 device -> ../../pci0000:00/0000:00:01.0" ) - return make_artifact(0, "[always] madvise never") + return make_artifact(1, "") linux_sys_settings_collector._run_sut_cmd = run_cmd args = {"paths": ["class/net/*/device"]} @@ -182,9 +182,8 @@ def run_cmd(cmd, **kwargs): assert result.status == ExecutionStatus.OK assert data is not None - assert len(seen_commands) == 1 - assert "ls -l /sys/class/net/*/device" in seen_commands[0] - assert "bash -c" in seen_commands[0] + assert any("cat /sys/class/net/*/device" in cmd for cmd in seen_commands) + assert any("ls -l /sys/class/net/*/device" in cmd for cmd in seen_commands) assert data.readings.get("/sys/class/net/*/device") == ( "lrwxrwxrwx 1 root root 0 Jan 1 00:00 device -> ../../pci0000:00/0000:00:01.0" ) @@ -195,6 +194,8 @@ def test_collect_data_mixed_paths_cat_and_glob(linux_sys_settings_collector): def run_cmd(cmd, **kwargs): if "ls -l" in cmd: return make_artifact(0, "lrwx 1 root root 0 device -> ../../device") + if "net/*" in cmd: + return make_artifact(1, "") if "enabled" in cmd: return make_artifact(0, "[always] madvise never") return make_artifact(0, "[madvise] always never defer") @@ -213,3 +214,36 @@ def run_cmd(cmd, **kwargs): data.readings.get("/sys/class/net/*/device") == "lrwx 1 root root 0 device -> ../../device" ) assert "Sysfs collected 3 path(s)" in result.message + + +VBIOS_PATH = "/sys/class/drm/card*/device/vbios_version" +VBIOS_VALUE = "113-M355-01-1K1-040C" + + +def test_collect_data_glob_cat_vbios_collapses_identical_lines(linux_sys_settings_collector): + seen_commands = [] + + def run_cmd(cmd, **kwargs): + seen_commands.append(cmd) + if "cat /sys/class/drm/card*/device/vbios_version" in cmd: + return make_artifact(0, f"{VBIOS_VALUE}\n{VBIOS_VALUE}\n{VBIOS_VALUE}\n") + return make_artifact(1, "") + + linux_sys_settings_collector._run_sut_cmd = run_cmd + result, data = linux_sys_settings_collector.collect_data({"paths": [VBIOS_PATH]}) + + assert result.status == ExecutionStatus.OK + assert data.readings.get(VBIOS_PATH) == VBIOS_VALUE + assert any("cat /sys/class/drm/card*/device/vbios_version" in cmd for cmd in seen_commands) + assert not any("ls -l" in cmd for cmd in seen_commands) + + +def test_collect_data_glob_cat_vbios_keeps_mixed_versions(linux_sys_settings_collector): + def run_cmd(cmd, **kwargs): + return make_artifact(0, f"{VBIOS_VALUE}\n113-OTHER-0000\n") + + linux_sys_settings_collector._run_sut_cmd = run_cmd + result, data = linux_sys_settings_collector.collect_data({"paths": [VBIOS_PATH]}) + + assert result.status == ExecutionStatus.OK + assert data.readings.get(VBIOS_PATH) == f"{VBIOS_VALUE}\n113-OTHER-0000" diff --git a/test/unit/plugin/test_sysctl_analyzer.py b/test/unit/plugin/test_sysctl_analyzer.py index dacca202..c96bd6c8 100644 --- a/test/unit/plugin/test_sysctl_analyzer.py +++ b/test/unit/plugin/test_sysctl_analyzer.py @@ -63,6 +63,7 @@ def test_analyzer_mismatch(analyzer, correct_data): args = SysctlAnalyzerArgs(exp_vm_swappiness=3, exp_vm_numa_balancing=4) result = analyzer.analyze_data(correct_data, args) assert result.status == ExecutionStatus.ERROR - assert "2 sysctl parameter(s) mismatched." in result.message - assert "1 errors" in result.message - assert "Sysctl mismatch detected" in result.message + assert result.message == ( + "2 sysctl parameter(s) mismatched. (1 errors: Sysctl mismatch detected)" + ) + assert result.events[0].description == "Sysctl mismatch detected" diff --git a/test/unit/test_version.py b/test/unit/test_version.py index a58d6345..b6bcc334 100644 --- a/test/unit/test_version.py +++ b/test/unit/test_version.py @@ -1,73 +1,73 @@ -############################################################################### -# -# MIT License -# -# Copyright (c) 2026 Advanced Micro Devices, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -############################################################################### - -import importlib -import re -from pathlib import Path -from unittest.mock import patch - -import pytest - - -def _get_project_name_from_pyproject() -> str: - repo_root = Path(__file__).resolve().parent.parent.parent - pyproject = repo_root / "pyproject.toml" - content = pyproject.read_text() - in_project = False - for line in content.splitlines(): - if line.strip() == "[project]": - in_project = True - continue - if in_project and line.strip().startswith("["): - break - if in_project: - match = re.match(r'^\s*name\s*=\s*["\']([^"\']+)["\']', line) - if match: - return match.group(1) - pytest.fail("Could not find name in [project] section of pyproject.toml") - - -def test_version_resolved_using_package_name_from_pyproject(): - expected_name = _get_project_name_from_pyproject() - with patch("importlib.metadata.version", return_value="1.2.3") as mock_version: - import nodescraper - - importlib.reload(nodescraper) - assert nodescraper.__version__ == "1.2.3" - mock_version.assert_called_once_with( - expected_name - ), "nodescraper.__init__ must call version() with the same name as pyproject.toml [project] name" - - -def test_version_unknown_when_package_not_found(): - from importlib.metadata import PackageNotFoundError - - expected_name = _get_project_name_from_pyproject() - with patch("importlib.metadata.version", side_effect=PackageNotFoundError(expected_name)): - import nodescraper - - importlib.reload(nodescraper) - assert nodescraper.__version__ == "unknown" +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### + +import importlib +import re +from pathlib import Path +from unittest.mock import patch + +import pytest + + +def _get_project_name_from_pyproject() -> str: + repo_root = Path(__file__).resolve().parent.parent.parent + pyproject = repo_root / "pyproject.toml" + content = pyproject.read_text() + in_project = False + for line in content.splitlines(): + if line.strip() == "[project]": + in_project = True + continue + if in_project and line.strip().startswith("["): + break + if in_project: + match = re.match(r'^\s*name\s*=\s*["\']([^"\']+)["\']', line) + if match: + return match.group(1) + pytest.fail("Could not find name in [project] section of pyproject.toml") + + +def test_version_resolved_using_package_name_from_pyproject(): + expected_name = _get_project_name_from_pyproject() + with patch("importlib.metadata.version", return_value="1.2.3") as mock_version: + import nodescraper + + importlib.reload(nodescraper) + assert nodescraper.__version__ == "1.2.3" + mock_version.assert_called_once_with( + expected_name + ), "nodescraper.__init__ must call version() with the same name as pyproject.toml [project] name" + + +def test_version_unknown_when_package_not_found(): + from importlib.metadata import PackageNotFoundError + + expected_name = _get_project_name_from_pyproject() + with patch("importlib.metadata.version", side_effect=PackageNotFoundError(expected_name)): + import nodescraper + + importlib.reload(nodescraper) + assert nodescraper.__version__ == "unknown"