diff --git a/ansible/inventory/group_vars/all/bmc b/ansible/inventory/group_vars/all/bmc index 2298fc070..07428a04b 100644 --- a/ansible/inventory/group_vars/all/bmc +++ b/ansible/inventory/group_vars/all/bmc @@ -16,3 +16,45 @@ ipmi_password: # Address to use to access a host's BMC via Redfish. redfish_address: + +############################################################################### +# Firmware update configuration. + +# Number of hosts to update firmware on in each batch. By default, all hosts +# are updated at once. Set to 1 to update hosts one at a time. This may be +# useful to limit the impact of reboots on clustered services. +firmware_update_serial: 100% + +############################################################################### +# Dell firmware update configuration. + +# Network share path of the firmware update repository. CIFS, NFS, HTTP, +# HTTPS and FTP share types are supported. The share must contain a valid +# repository of Dell Update Packages (DUPs) and a catalog file describing +# them. For example, an NFS share: +# dell_firmware_update_share_name: "192.168.0.1:/share" +# or an HTTPS share: +# dell_firmware_update_share_name: "https://downloads.dell.com" +dell_firmware_update_share_name: + +# Username for the firmware update repository. Required for CIFS shares. +dell_firmware_update_share_user: + +# Password for the firmware update repository. Required for CIFS shares. +dell_firmware_update_share_password: + +# Catalog file name relative to dell_firmware_update_share_name. This +# may be omitted when using the official Dell repository at +# https://downloads.dell.com. +dell_firmware_update_catalog_file_name: + +# Whether to reboot the host immediately to apply the firmware update. If +# false, packages are staged and applied on the next reboot. Default is false. +dell_firmware_update_reboot: false + +# Whether to ignore certificate warnings when an HTTPS share is used. Default +# is false. +dell_firmware_update_ignore_cert_warning: false + +# Whether to validate the iDRAC SSL certificates. Default is true. +dell_firmware_update_validate_certs: true diff --git a/ansible/overcloud-firmware-update.yml b/ansible/overcloud-firmware-update.yml new file mode 100644 index 000000000..e5d63bb3c --- /dev/null +++ b/ansible/overcloud-firmware-update.yml @@ -0,0 +1,112 @@ +--- +# Perform firmware update of the overcloud nodes. The firmware update is +# performed out of band via the hosts' BMCs using the +# dellemc.openmanage.idrac_firmware module. As such, only Dell servers with +# iDRACs are currently supported. During the update, we set the nodes to +# maintenance mode to prevent bifrost from managing their power states. + +- name: Group overcloud nodes by their BMC type + hosts: overcloud + gather_facts: no + max_fail_percentage: >- + {{ overcloud_firmware_update_max_fail_percentage | + default(kayobe_max_fail_percentage) | + default(100) }} + tags: + - firmware-update + - firmware-update-idrac + tasks: + - name: Group overcloud hosts by their BMC type + group_by: + key: "overcloud_with_bmcs_of_type_{{ bmc_type | default('unknown') }}" + changed_when: false + +- name: Update overcloud node firmware - iDRAC + hosts: overcloud_with_bmcs_of_type_idrac + gather_facts: no + max_fail_percentage: >- + {{ overcloud_firmware_update_max_fail_percentage | + default(kayobe_max_fail_percentage) | + default(100) }} + # Optionally update nodes in batches, for example one at a time. Defaults to + # updating all nodes at once. + serial: "{{ firmware_update_serial | default('100%') }}" + tags: + - firmware-update + - firmware-update-idrac + vars: + seed_host: "{{ groups['seed'][0] }}" + pre_tasks: + - name: Fail if the firmware repository share is not configured + fail: + msg: > + dell_firmware_update_share_name must be set to the network + share containing the firmware repository. + when: not dell_firmware_update_share_name + + - name: Set the overcloud nodes' maintenance mode + command: > + {{ container_engine }} exec bifrost_deploy + bash -c ' + export OS_CLOUD=bifrost && + export BIFROST_INVENTORY_SOURCE=ironic && + export BIFROST_NODE_NAMES="{{ inventory_hostname }}" && + ansible baremetal + --connection local + --inventory /etc/bifrost/inventory/ + -e @/etc/bifrost/bifrost.yml + -e @/etc/bifrost/dib.yml + --limit {{ inventory_hostname }} + -m command + -a "baremetal node maintenance set {% raw %}{{ inventory_hostname }}{% endraw %} --reason firmware-update"' + delegate_to: "{{ seed_host }}" + vars: + # NOTE: Without this, the seed's ansible_host variable will not be + # respected when using delegate_to. + ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}" + become: "{{ container_engine == 'podman' }}" + + tasks: + - name: Update firmware from the repository + dellemc.openmanage.idrac_firmware: + idrac_ip: "{{ redfish_address | default(ipmi_address, true) }}" + idrac_user: "{{ ipmi_username }}" + idrac_password: "{{ ipmi_password }}" + share_name: "{{ dell_firmware_update_share_name }}" + share_user: "{{ dell_firmware_update_share_user | default(omit, true) }}" + share_password: "{{ dell_firmware_update_share_password | default(omit, true) }}" + catalog_file_name: "{{ dell_firmware_update_catalog_file_name | default(omit, true) }}" + ignore_cert_warning: "{{ dell_firmware_update_ignore_cert_warning | bool }}" + validate_certs: "{{ dell_firmware_update_validate_certs | bool }}" + apply_update: "{{ not ansible_check_mode }}" + reboot: "{{ dell_firmware_update_reboot | bool }}" + job_wait: true + delegate_to: localhost + register: dell_firmware_update_result + + - name: Display firmware update results + debug: + var: dell_firmware_update_result + + post_tasks: + - name: Unset the overcloud nodes' maintenance mode + command: > + {{ container_engine }} exec bifrost_deploy + bash -c ' + export OS_CLOUD=bifrost && + export BIFROST_INVENTORY_SOURCE=ironic && + export BIFROST_NODE_NAMES="{{ inventory_hostname }}" && + ansible baremetal + --connection local + --inventory /etc/bifrost/inventory/ + -e @/etc/bifrost/bifrost.yml + -e @/etc/bifrost/dib.yml + --limit {{ inventory_hostname }} + -m command + -a "baremetal node maintenance unset {% raw %}{{ inventory_hostname }}{% endraw %}"' + delegate_to: "{{ seed_host }}" + vars: + # NOTE: Without this, the seed's ansible_host variable will not be + # respected when using delegate_to. + ansible_host: "{{ hostvars[seed_host].ansible_host | default(seed_host) }}" + become: "{{ container_engine == 'podman' }}" diff --git a/doc/source/administration/overcloud.rst b/doc/source/administration/overcloud.rst index a5433d5f3..ece4c9ccc 100644 --- a/doc/source/administration/overcloud.rst +++ b/doc/source/administration/overcloud.rst @@ -63,6 +63,59 @@ For example:: To execute the command with root privileges, add the ``--become`` argument. Adding the ``--verbose`` argument allows the output of the command to be seen. +Updating Firmware +================= + +It is possible to update firmware on the overcloud hosts:: + + (kayobe) $ kayobe overcloud firmware update + +To show the available firmware updates without applying them, run the command +in Ansible check mode using the ``--check`` argument:: + + (kayobe) $ kayobe overcloud firmware update --check + +Firmware updates are performed out of band via the host's BMC. Currently only +Dell servers with iDRACs (``bmc_type: idrac``) are supported. + +During the update, the nodes are placed in maintenance mode to prevent Bifrost +from managing their power states. + +By default, firmware is updated on all hosts at once. To update hosts in +batches, for example one at a time to limit the impact of reboots on clustered +services, set ``firmware_update_serial``: + +.. code-block:: yaml + :caption: ``$KAYOBE_CONFIG_PATH/bmc.yml`` + + firmware_update_serial: 1 + +Dell Firmware Updates +--------------------- + +Dell firmware updates are performed using the +``dellemc.openmanage.idrac_firmware`` Ansible module, and require an iDRAC +Enterprise licence. + +The firmware update repository must be configured before running the command. +Set the network share containing a valid repository of Dell Update Packages +(DUPs) and a catalog file describing them: + +.. code-block:: yaml + :caption: ``$KAYOBE_CONFIG_PATH/bmc.yml`` + + dell_firmware_update_share_name: "https://downloads.dell.com" + +CIFS, NFS, HTTP, HTTPS and FTP share types are supported. For CIFS shares, +``dell_firmware_update_share_user`` and +``dell_firmware_update_share_password`` must also be set. Other options +include the catalog file name, whether to reboot the host to apply the update, +and whether to ignore certificate warnings for HTTPS shares. + +By default the iDRAC SSL certificates are validated. To skip this, for example +when using self-signed certificates, set +``dell_firmware_update_validate_certs`` to ``false``. + Provisioning Hosts Without Kolla Services ========================================== diff --git a/doc/source/custom-ansible-playbooks.rst b/doc/source/custom-ansible-playbooks.rst index ab031fffd..5a6909c0f 100644 --- a/doc/source/custom-ansible-playbooks.rst +++ b/doc/source/custom-ansible-playbooks.rst @@ -165,11 +165,6 @@ Then, to run the ``foo.yml`` playbook:: Hooks ===== -.. warning:: - Hooks are an experimental feature and the design could change in the future. - You may have to update your config if there are any changes to the design. - This warning will be removed when the design has been stabilised. - Hooks allow you to automatically execute custom playbooks at certain points during the execution of a kayobe command. The point at which a hook is run is referred to as a ``target``. Please see the :ref:`list of available targets`. diff --git a/doc/source/deployment.rst b/doc/source/deployment.rst index a0a975096..8840fad05 100644 --- a/doc/source/deployment.rst +++ b/doc/source/deployment.rst @@ -339,11 +339,6 @@ Leave the seed VM and return to the shell on the Ansible control host:: Infrastructure VMs =================== -.. warning:: - - Support for infrastructure VMs is considered experimental: its - design may change in future versions without a deprecation period. - .. note:: It necessary to perform some configuration before these steps diff --git a/doc/source/multiple-environments.rst b/doc/source/multiple-environments.rst index e211fc6db..dca1f2019 100644 --- a/doc/source/multiple-environments.rst +++ b/doc/source/multiple-environments.rst @@ -4,11 +4,6 @@ Multiple Environments ===================== -.. warning:: - - Support for multiple Kayobe environments is considered experimental: its - design may change in future versions without a deprecation period. - Sometimes it can be useful to support deployment of multiple environments from a single Kayobe configuration. Most commonly this is to support a deployment pipeline, such as the traditional development, test, staging and production @@ -414,11 +409,6 @@ This would configure the external FQDN for the staging environment at Environment Dependencies ------------------------ -.. warning:: - - This is an experimental feature and is still subject to change whilst - the design is finalised. - Since the Antelope 14.0.0 release, multiple environments can be layered on top of each of each other by declaring dependencies in a ``.kayobe-environment`` file located in the environment subdirectory. For example: diff --git a/etc/kayobe/bmc.yml b/etc/kayobe/bmc.yml index b29ecf670..708111096 100644 --- a/etc/kayobe/bmc.yml +++ b/etc/kayobe/bmc.yml @@ -17,6 +17,48 @@ # Address to use to access a host's BMC via Redfish. #redfish_address: +############################################################################### +# Firmware update configuration. + +# Number of hosts to update firmware on in each batch. By default, all hosts +# are updated at once. Set to 1 to update hosts one at a time. This may be +# useful to limit the impact of reboots on clustered services. +#firmware_update_serial: + +############################################################################### +# Dell firmware update configuration. + +# Network share path of the firmware update repository. CIFS, NFS, HTTP, +# HTTPS and FTP share types are supported. The share must contain a valid +# repository of Dell Update Packages (DUPs) and a catalog file describing +# them. For example, an NFS share: +# dell_firmware_update_share_name: "192.168.0.1:/share" +# or an HTTPS share: +# dell_firmware_update_share_name: "https://downloads.dell.com" +#dell_firmware_update_share_name: + +# Username for the firmware update repository. Required for CIFS shares. +#dell_firmware_update_share_user: + +# Password for the firmware update repository. Required for CIFS shares. +#dell_firmware_update_share_password: + +# Catalog file name relative to dell_firmware_update_share_name. This +# may be omitted when using the official Dell repository at +# https://downloads.dell.com. +#dell_firmware_update_catalog_file_name: + +# Whether to reboot the host immediately to apply the firmware update. If +# false, packages are staged and applied on the next reboot. Default is false. +#dell_firmware_update_reboot: + +# Whether to ignore certificate warnings when an HTTPS share is used. Default +# is false. +#dell_firmware_update_ignore_cert_warning: + +# Whether to validate the iDRAC SSL certificates. Default is true. +#dell_firmware_update_validate_certs: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 1e365bc5e..956f0abc2 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -1370,6 +1370,15 @@ def take_action(self, parsed_args): self.run_kayobe_playbooks(parsed_args, playbooks) +class OvercloudFirmwareUpdate(KayobeAnsibleMixin, VaultMixin, Command): + """Update firmware for the overcloud hosts.""" + + def take_action(self, parsed_args): + self.app.LOG.debug("Updating overcloud host firmware") + playbooks = _build_playbook_list("overcloud-firmware-update") + self.run_kayobe_playbooks(parsed_args, playbooks) + + class OvercloudHardwareRegister(KayobeAnsibleMixin, VaultMixin, Command): """Register overcloud hosts in Bifrost""" diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 8fd8750be..46b526383 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -1487,6 +1487,27 @@ def test_overcloud_facts_gather(self, mock_kolla_run, mock_run): ] self.assertListEqual(expected_calls, mock_kolla_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_overcloud_firmware_update(self, mock_run): + command = commands.OvercloudFirmwareUpdate(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + + result = command.run(parsed_args) + self.assertEqual(0, result) + + expected_calls = [ + mock.call( + mock.ANY, + [ + utils.get_data_files_path( + "ansible", "overcloud-firmware-update.yml"), + ], + ), + ] + self.assertListEqual(expected_calls, mock_run.call_args_list) + @mock.patch.object(commands.KayobeAnsibleMixin, "run_kayobe_playbooks") def test_overcloud_host_configure(self, mock_run): diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2 index 149e2953f..47cce9776 100644 --- a/playbooks/kayobe-seed-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-base/overrides.yml.j2 @@ -22,6 +22,12 @@ kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}" kolla_ansible_source_version: "{{ zuul.projects['opendev.org/openstack/kolla-ansible'].checkout }}" kolla_ansible_requirements_yml: "/tmp/kolla-ansible-requirements.yml" +{% if build_images | bool %} +# NOTE: The ironic-python-agent-builder repository is copied to +# /tmp/ironic-python-agent-builder so that it is accessible to the stack user. +ipa_builder_source_url: "/tmp/ironic-python-agent-builder" +ipa_builder_source_version: "{{ zuul.projects['opendev.org/openstack/ironic-python-agent-builder'].checkout }}" +{% endif %} kolla_openstack_logging_debug: True pip_upper_constraints_file: "/tmp/upper-constraints.txt" diff --git a/playbooks/kayobe-seed-base/pre.yml b/playbooks/kayobe-seed-base/pre.yml index 0a1288946..4c509f07b 100644 --- a/playbooks/kayobe-seed-base/pre.yml +++ b/playbooks/kayobe-seed-base/pre.yml @@ -39,6 +39,23 @@ dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/dib.yml" when: not build_images + # NOTE: The stack user cannot access the ironic-python-agent-builder source + # code, which lives in the zuul user's home directory. Copy it to /tmp. The + # stack user does not exist yet, so the copy is owned by root. Mark it as a + # safe directory to allow the stack user to clone it. Some git versions + # check the path of the .git directory, so use that. + - block: + - name: Create a copy of ironic-python-agent-builder source code + copy: + src: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/ironic-python-agent-builder'].src_dir }}" + dest: /tmp + remote_src: true + + - name: Mark ironic-python-agent-builder source code as a safe directory + command: git config --system --add safe.directory /tmp/ironic-python-agent-builder/.git + become: true + when: build_images | bool + - name: Ensure kayobe is installed shell: cmd: dev/install.sh &> {{ logs_dir }}/ansible/install diff --git a/releasenotes/notes/add-overcloud-firmware-update-command-3b0c9d8e5f1a2f47.yaml b/releasenotes/notes/add-overcloud-firmware-update-command-3b0c9d8e5f1a2f47.yaml new file mode 100644 index 000000000..b0148346a --- /dev/null +++ b/releasenotes/notes/add-overcloud-firmware-update-command-3b0c9d8e5f1a2f47.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds a new ``kayobe overcloud firmware update`` command to update firmware + on the overcloud hosts. The firmware update is performed out of band via + the hosts' BMCs using the ``dellemc.openmanage.idrac_firmware`` Ansible + module, and currently supports Dell servers with iDRACs. The firmware + update repository is configured via the ``dell_firmware_update_*`` + variables in ``bmc.yml``. Use Ansible check mode (``--check``) to show the + available firmware updates without applying them. diff --git a/releasenotes/notes/stabilise-hooks-94e704e157a7840c.yaml b/releasenotes/notes/stabilise-hooks-94e704e157a7840c.yaml new file mode 100644 index 000000000..4e3204233 --- /dev/null +++ b/releasenotes/notes/stabilise-hooks-94e704e157a7840c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Custom playbook hooks are no longer considered experimental. Future + changes to their design will follow the standard deprecation process. diff --git a/releasenotes/notes/stabilise-infra-vms-21063f066b1a5006.yaml b/releasenotes/notes/stabilise-infra-vms-21063f066b1a5006.yaml new file mode 100644 index 000000000..ffe151b14 --- /dev/null +++ b/releasenotes/notes/stabilise-infra-vms-21063f066b1a5006.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Support for infrastructure VMs is no longer considered experimental. + Future changes to its design will follow the standard deprecation + process. diff --git a/releasenotes/notes/stabilise-multiple-environments-5e818adba7b14070.yaml b/releasenotes/notes/stabilise-multiple-environments-5e818adba7b14070.yaml new file mode 100644 index 000000000..43e18465d --- /dev/null +++ b/releasenotes/notes/stabilise-multiple-environments-5e818adba7b14070.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Support for multiple Kayobe environments, including environment + dependencies declared in ``.kayobe-environment`` files, is no longer + considered experimental. Future changes to their design will follow the + standard deprecation process. diff --git a/releasenotes/source/2025.1.rst b/releasenotes/source/2025.1.rst index 3add0e53a..7a4c586d3 100644 --- a/releasenotes/source/2025.1.rst +++ b/releasenotes/source/2025.1.rst @@ -3,4 +3,4 @@ =========================== .. release-notes:: - :branch: stable/2025.1 + :branch: unmaintained/2025.1 diff --git a/requirements.yml b/requirements.yml index 679bfa833..cfd2b6949 100644 --- a/requirements.yml +++ b/requirements.yml @@ -13,6 +13,8 @@ collections: version: 5.1.0 - name: dellemc.enterprise_sonic version: 4.1.0 + - name: dellemc.openmanage + version: 11.0.0 - name: dellemc.os6 version: 1.0.7 - name: dellemc.os9 diff --git a/setup.cfg b/setup.cfg index a96bf711e..f9c50a0e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,6 +66,7 @@ kayobe.cli= overcloud_deployment_image_build = kayobe.cli.commands:OvercloudDeploymentImageBuild overcloud_deprovision = kayobe.cli.commands:OvercloudDeprovision overcloud_facts_gather = kayobe.cli.commands:OvercloudFactsGather + overcloud_firmware_update = kayobe.cli.commands:OvercloudFirmwareUpdate overcloud_hardware_inspect = kayobe.cli.commands:OvercloudHardwareInspect overcloud_hardware_register = kayobe.cli.commands:OvercloudHardwareRegister overcloud_host_configure = kayobe.cli.commands:OvercloudHostConfigure @@ -170,6 +171,8 @@ kayobe.cli.overcloud_deprovision = hooks = kayobe.cli.commands:HookDispatcher kayobe.cli.overcloud_facts_gather = hooks = kayobe.cli.commands:HookDispatcher +kayobe.cli.overcloud_firmware_update = + hooks = kayobe.cli.commands:HookDispatcher kayobe.cli.overcloud_hardware_inspect = hooks = kayobe.cli.commands:HookDispatcher kayobe.cli.overcloud_hardware_register = diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 49239c0ce..6494cb0c1 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -305,6 +305,8 @@ Base job for testing seed image builds. Configures the primary VM as a seed. + required-projects: + - name: openstack/ironic-python-agent-builder vars: build_images: true