25.3 Fips CICD fixes and backports

.github/actions/create_workflow_report/ci_run_report.html.jinja

@@ -5,6 +5,9 @@
   <meta http-equiv="content-type" content="text/html; charset=UTF-8">
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  {%- if is_preview %}
+  <meta http-equiv="Cache-Control" content="max-age=300">
+  {%- endif %}
   <style>
     /* Base colors for Altinity */
     :root {
@@ -152,7 +155,7 @@
       </tr>
       <tr>
         <th class="hth no-sort">Build Report</th>
-        <td><a href="https://s3.amazonaws.com/{{ s3_bucket }}/{{ pr_number }}/{{ commit_sha }}/builds/report.html">Build Report</a></td>
+        <td>{% for job_name, link in build_report_links.items() %}<a href="{{ link }}">[{{ job_name }}]</a> {% endfor %}</td>
       </tr>
       <tr>
         <th class="hth no-sort">Date</th>
@@ -223,6 +226,8 @@
           element.className = element.className.replace(regex_dir, '') + dir
         }
         function getValue(element) {
+          var childWithSort = element.querySelector('[data-sort]')
+          if (childWithSort) return childWithSort.getAttribute('data-sort')
           return (
             (alt_sort && element.getAttribute('data-sort-alt')) ||
             element.getAttribute('data-sort') || element.innerText

.github/actions/create_workflow_report/create_workflow_report.py

@@ -55,6 +55,9 @@ def query_dataframe_with_retry(
             time.sleep(wait)
 
 
+CVE_SEVERITY_ORDER = {"critical": 1, "high": 2, "medium": 3, "low": 4, "negligible": 5}
+
+
 def get_commit_statuses(sha: str) -> pd.DataFrame:
     """
     Fetch commit statuses for a given SHA and return as a pandas DataFrame.
@@ -431,7 +434,9 @@ def get_cves(pr_number, commit_sha):
         Bucket=S3_BUCKET, Prefix=s3_prefix, Delimiter="/"
     )
     grype_result_dirs = [
-        content["Prefix"] for content in response.get("CommonPrefixes", [])
+        content["Prefix"]
+        for content in response.get("CommonPrefixes", [])
+        if isinstance(content, dict) and content.get("Prefix")
     ]
 
     if len(grype_result_dirs) == 0:
@@ -461,12 +466,13 @@ def get_cves(pr_number, commit_sha):
         return pd.DataFrame()
 
     df = pd.DataFrame(rows).drop_duplicates()
-    df = df.sort_values(
-        by="severity",
-        key=lambda col: col.str.lower().map(
-            {"critical": 1, "high": 2, "medium": 3, "low": 4, "negligible": 5}
-        ),
-    )
+
+    def _cve_sort_key(col):
+        if col.name == "severity":
+            return col.str.lower().map(CVE_SEVERITY_ORDER)
+        return col
+
+    df = df.sort_values(by=["severity", "docker_image"], key=_cve_sort_key)
     return df
 
 
@@ -510,6 +516,9 @@ def format_results_as_html_table(results) -> str:
             "Identifier": lambda i: url_to_html_link(
                 "https://nvd.nist.gov/vuln/detail/" + i
             ),
+            "Severity": lambda s: (
+                f'<span data-sort="{CVE_SEVERITY_ORDER.get(str(s).lower(), 6)}">{s}</span>'
+            ),
         },
         escape=False,
         border=0,
@@ -575,11 +584,16 @@ def main():
         "pr_new_fails": [],
         "checks_errors": get_checks_errors(db_client, args.commit_sha, branch_name),
         "regression_fails": get_regression_fails(db_client, args.actions_run_url),
-        "docker_images_cves": (
-            [] if not args.cves else get_cves(args.pr_number, args.commit_sha)
-        ),
+        "docker_images_cves": [],
     }
 
+    try:
+        fail_results["docker_images_cves"] = (
+            [] if not args.cves else get_cves(args.pr_number, args.commit_sha)
+        )
+    except Exception as e:
+        print(f"Error in get_cves: {e}")
+
     # get_cves returns ... in the case where no Grype result files were found.
     # This might occur when run in preview mode.
     cves_not_checked = not args.cves or fail_results["docker_images_cves"] is ...

.github/workflows/release_branches.yml

@@ -1,5 +1,6 @@
 # yamllint disable rule:comments-indentation
 name: ReleaseBranchCI
+run-name: "${{ github.event.inputs.workflow_name || ' ' }}"
 
 env:
   # Force the stdout and stderr streams to be unbuffered
@@ -21,9 +22,8 @@ on: # yamllint disable-line rule:truthy
       - opened
     branches:
       - 'releases/*'
+      - 'stable-*'
   push:
-    branches:
-      - 'releases/*'
     tags:
       - '*'
   workflow_dispatch:
@@ -418,21 +418,21 @@ jobs:
     secrets: inherit
     with:
       runner_type: altinity-regression-tester
-      commit: e1f4e6ae0d862388e113cbc74b8dab32b9db64fc
+      commit: b28bcd03f30440f25a17917e5005670c28e3a703
       arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
+      timeout_minutes: 210
   RegressionTestsAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'regression') && !contains(fromJson(needs.RunConfig.outputs.data).ci_settings.exclude_keywords, 'aarch64')}}
     uses: ./.github/workflows/regression.yml
     secrets: inherit
     with:
       runner_type: altinity-regression-tester-aarch64
-      commit: e1f4e6ae0d862388e113cbc74b8dab32b9db64fc
+      commit: b28bcd03f30440f25a17917e5005670c28e3a703
       arch: aarch64
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout_minutes: 300
+      timeout_minutes: 210
   SignRelease:
     needs: [RunConfig, BuilderDebRelease]
     if: ${{ !failure() && !cancelled() }}
@@ -478,6 +478,7 @@ jobs:
       - RegressionTestsAarch64
       - GrypeScan
       - SignRelease
+      - SignAarch64
     runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64]
     steps:
       - name: Check out repository code

.github/workflows/reusable_sign.yml

@@ -160,7 +160,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ${{inputs.test_name}} signed-hashes
-          path: ${{ env.TEMP_PATH }}/*.gpg
+          path: |
+            ${{ runner.temp }}/signed/*.gpg
+            ${{ runner.temp }}/signed/signing_pubkey.asc
       - name: Clean
         if: always()
         uses: ./.github/actions/clean

ci/jobs/functional_stateless_tests.py

@@ -1,5 +1,4 @@
 import argparse
-import os
 import time
 from pathlib import Path
 
@@ -62,15 +61,6 @@ def main():
         if "/" in to:
             batch_num, total_batches = map(int, to.split("/"))
 
-    # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
-    os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
-        f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
-        verbose=True,
-    )
-    no_azure = False
-    if not os.environ["AZURE_CONNECTION_STRING"]:
-        no_azure = True
-
     ch_path = args.ch_path
     assert (
         Path(ch_path + "/clickhouse").is_file()
@@ -109,7 +99,7 @@ def main():
             f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-format",
             f"rm -rf {temp_dir}/etc/ && mkdir -p {temp_dir}/etc/clickhouse-client {temp_dir}/etc/clickhouse-server",
             f"cp programs/server/config.xml programs/server/users.xml {temp_dir}/etc/clickhouse-server/",
-            f"./tests/config/install.sh {temp_dir}/etc/clickhouse-server {temp_dir}/etc/clickhouse-client --s3-storage {'--no-azure' if no_azure else ''}",
+            f"./tests/config/install.sh {temp_dir}/etc/clickhouse-server {temp_dir}/etc/clickhouse-client --s3-storage",
             # clickhouse benchmark segfaults with --config-path, so provide client config by its default location
             f"cp {temp_dir}/etc/clickhouse-client/* /etc/clickhouse-client/",
             # update_path_ch_config,
@@ -140,6 +130,9 @@ def main():
         res = res and Shell.check(
             "aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
         )
+        azurite_log = f"{temp_dir}/azurite.log"
+        res = res and CH.start_azurite(log_file_path=azurite_log)
+        logs_to_attach += [azurite_log]
         res = res and CH.log_cluster_config()
         res = res and CH.start()
         res = res and CH.wait_ready()

ci/jobs/scripts/clickhouse_proc.py

@@ -59,6 +59,22 @@ def start_minio(self, test_type, log_file_path):
         print(f"Started setup_minio.sh asynchronously with PID {process.pid}")
         return True
 
+    def start_azurite(self, log_file_path):
+        command = f"cd {temp_dir} && azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence"
+        with open(log_file_path, "w") as log_file:
+            process = subprocess.Popen(
+                command, stdout=log_file, stderr=subprocess.STDOUT, shell=True
+            )
+        print(f"Started azurite asynchronously with PID {process.pid}")
+        if Shell.check(
+            "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:10000/ | grep -qE '400|200'",
+            verbose=False,
+            retries=6,
+        ):
+            return True
+        print("Failed to start azurite")
+        return False
+
     @staticmethod
     def log_cluster_config():
         return Shell.check(

tests/ci/build_download_helper.py

@@ -154,6 +154,16 @@ def read_build_urls(build_name: str, reports_path: Union[Path, str]) -> List[str
     if artifact_report.is_file():
         with open(artifact_report, "r", encoding="utf-8") as f:
             return json.load(f)["build_urls"]  # type: ignore
+    pr_info = None
+    try:
+        from pr_info import PRInfo  # pylint: disable=import-outside-toplevel
+
+        pr_info = PRInfo()
+    except Exception as ex:
+        logger.warning("Failed to init PRInfo while selecting build report: %s", ex)
+
+    reports_by_sha = []  # type: List[List[str]]
+    fallback_reports = []  # type: List[List[str]]
     for root, _, files in os.walk(reports_path):
         for file in files:
             if file.endswith(f"_{build_name}.json"):
@@ -162,7 +172,29 @@ def read_build_urls(build_name: str, reports_path: Union[Path, str]) -> List[str
                     os.path.join(root, file), "r", encoding="utf-8"
                 ) as file_handler:
                     build_report = json.load(file_handler)
-                    return build_report["build_urls"]  # type: ignore
+                    build_urls = build_report.get("build_urls", [])
+                    if not isinstance(build_urls, list):
+                        continue
+                    fallback_reports.append(build_urls)
+                    if pr_info and pr_info.sha and any(
+                        f"/{pr_info.sha}/" in str(url) for url in build_urls
+                    ):
+                        reports_by_sha.append(build_urls)
+
+    if reports_by_sha:
+        logger.info(
+            "Using build report matched by SHA [%s] for [%s]",
+            pr_info.sha if pr_info else "",
+            build_name,
+        )
+        return reports_by_sha[0]
+
+    if fallback_reports:
+        logger.warning(
+            "No SHA-matched build report found for [%s], fallback to first discovered",
+            build_name,
+        )
+        return fallback_reports[0]
 
     logger.info("A build report is not found for %s", build_name)
     return []

tests/ci/ci.py

@@ -303,6 +303,10 @@ def _pre_action(s3, job_name, batch, indata, pr_info):
         #   testing), otherwise reports won't be found
         if not (pr_info.is_scheduled or pr_info.is_dispatched):
             report_prefix = Utils.normalize_string(pr_info.head_ref)
+    elif isinstance(pr_info.ref, str) and pr_info.ref.startswith("refs/tags/"):
+        # For tag-triggered runs, use tag name as prefix to avoid downloading
+        # reports from other runs with the same digest.
+        report_prefix = Utils.normalize_string(pr_info.head_ref)
     elif pr_info.is_pr:
         report_prefix = str(pr_info.number)
     print(