From 569151846bb6fb63461c851fc45bcf60f6f1de90 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:48:37 -0400 Subject: [PATCH 1/3] replace azure with azurite --- ci/jobs/functional_tests.py | 20 -------------------- ci/jobs/scripts/clickhouse_proc.py | 12 +++++++++++- ci/jobs/stress_job.py | 15 --------------- tests/config/config.d/azure_storage_conf.xml | 6 ++---- tests/config/install.sh | 2 +- tests/docker_scripts/stress_runner.sh | 1 + 6 files changed, 15 insertions(+), 41 deletions(-) diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index 84775a71137e..f63a1d9c6f7e 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -247,26 +247,6 @@ def main(): print(f"Rerun count set to 5 for targeted check") rerun_count = 5 - if not info.is_local_run: - # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled - # os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output( - # f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value", - # verbose=True, - # ) - # NOTE(strtgbb): We pass azure credentials through the docker command, not SSM. - # NOTE(strtgbb): Azure credentials don't exist in community workflow - if info.is_community_pr: - print( - "NOTE: No azure credentials provided for community PR - disable azure storage" - ) - config_installs_args += " --no-azure" - - # NOTE(strtgbb): With the above, some tests are still trying to use azure, try this: - os.environ["USE_AZURE_STORAGE_FOR_MERGE_TREE"] = "0" - else: - print("Disable azure for a local run") - config_installs_args += " --no-azure" - if (is_azure_storage or is_s3_storage) and is_encrypted_storage: config_installs_args += " --encrypted-storage" runner_options += f" --encrypted-storage" diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index 0c119f771a12..7fb7e081f908 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -171,7 +171,15 @@ def start_azurite(self): command, stdout=log_file, stderr=subprocess.STDOUT, shell=True ) print(f"Started azurite asynchronously with PID {self.azurite_proc.pid}") - return True + + if Shell.check( + "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:10000/ | grep -qE '400|200'", + verbose=False, + retries=6, + ): + return True + print("Failed to start azurite") + return False def start_kafka(self): command = [ @@ -1358,6 +1366,8 @@ def set_random_timezone(): param = sys.argv[2] assert param in ["stateless"] res = ch.start_minio(param) + elif command == "start_azurite": + res = ch.start_azurite() else: raise ValueError(f"Unknown command: {command}") except Exception as e: diff --git a/ci/jobs/stress_job.py b/ci/jobs/stress_job.py index 2812b379c484..68bd5a9f01e0 100644 --- a/ci/jobs/stress_job.py +++ b/ci/jobs/stress_job.py @@ -62,21 +62,6 @@ def get_additional_envs(info, check_name: str) -> List[str]: from ci.jobs.ci_utils import is_extended_run result = [] - if not info.is_local_run: - # azure_connection_string = Shell.get_output( - # f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value", - # verbose=True, - # strict=True, - # ) - # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'") - # NOTE(strtgbb): We pass azure credentials through the docker command, not SSM. - # NOTE(strtgbb): Azure credentials don't exist in community workflow - if info.is_community_pr: - print( - "NOTE: No azure credentials provided for community PR - disable azure storage" - ) - os.environ["USE_AZURE_STORAGE_FOR_MERGE_TREE"] = "0" - # some cloud-specific features require feature flags enabled # so we need this ENV to be able to disable the randomization # of feature flags diff --git a/tests/config/config.d/azure_storage_conf.xml b/tests/config/config.d/azure_storage_conf.xml index ca2ec4a4be82..663856a18b9e 100644 --- a/tests/config/config.d/azure_storage_conf.xml +++ b/tests/config/config.d/azure_storage_conf.xml @@ -6,10 +6,8 @@ azure false 33554432 - - - - + clickhouse-tests + DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; cache diff --git a/tests/config/install.sh b/tests/config/install.sh index f08faa656e57..4b663e91a14b 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -356,7 +356,7 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then fi if [[ "$EXPORT_S3_STORAGE_POLICIES" == "1" ]]; then - if [[ "$NO_AZURE" != "1" ]] && [[ -v AZURE_STORAGE_KEY ]]; then + if [[ "$NO_AZURE" != "1" ]]; then ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/ fi diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 284d6788f90f..085c79f38af7 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -53,6 +53,7 @@ configure cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py logs_export_config || echo "ERROR: Failed to create log export config" cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_minio stateless || { echo "Failed to start minio"; exit 1; } +cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; } start_server || { echo "Failed to start server"; exit 1; } From 6f0655fbf0ee5fd760d4bb0dbe02649cbeb25a37 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:09:06 -0400 Subject: [PATCH 2/3] Use azurite-rs for Azure functional tests. npm azurite-blob cannot carry MergeTree-on-Azure under ASan: the dedicated azure shards timed out or died. Match 26.6: bake azurite-rs into the test image, raise the nofile ulimit, and lower azure FT concurrency. --- ci/defs/job_configs.py | 6 ++++-- ci/docker/stateless-test/Dockerfile | 8 ++++++++ ci/jobs/functional_tests.py | 5 +++++ ci/jobs/scripts/clickhouse_proc.py | 13 ++++++++++--- ci/jobs/stress_job.py | 2 ++ tests/docker_scripts/upgrade_runner.sh | 2 +- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 7bdaa4318e5e..7c6c41fcd591 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -84,7 +84,9 @@ # some tests can be flaky due to very slow disks - use tmpfs for temporary ClickHouse files # --cap-add=SYS_PTRACE and --privileged for gdb in docker # --root/--privileged/--cgroupns=host is required for clickhouse-test --memory-limit - run_in_docker=f"altinityinfra/stateless-test+--memory={LIMITED_MEM}+--cgroupns=host+--cap-add=SYS_PTRACE+--privileged+--security-opt seccomp=unconfined+--tmpfs /tmp/clickhouse:mode=1777+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/etc/clickhouse-server1:/etc/clickhouse-server1+--volume=./ci/tmp/etc/clickhouse-server2:/etc/clickhouse-server2+--volume=./ci/tmp/var/log:/var/log+root+--env=AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY+--env=AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME+--env=AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME+--env=AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD", + # --ulimit nofile is raised so that azurite-rs (the in-process Azure Blob + # Storage emulator) does not run out of file descriptors under parallel load + run_in_docker=f"altinityinfra/stateless-test+--memory={LIMITED_MEM}+--cgroupns=host+--cap-add=SYS_PTRACE+--privileged+--security-opt seccomp=unconfined+--ulimit nofile=1048576:1048576+--tmpfs /tmp/clickhouse:mode=1777+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/etc/clickhouse-server1:/etc/clickhouse-server1+--volume=./ci/tmp/etc/clickhouse-server2:/etc/clickhouse-server2+--volume=./ci/tmp/var/log:/var/log+root+--env=AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY+--env=AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME+--env=AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME+--env=AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/functional_tests.py", @@ -540,7 +542,7 @@ class JobConfigs: runs_on=RunnerLabels.FUNC_TESTER_ARM, command="python3 ./ci/jobs/functional_tests.py --options BugfixValidation", # some tests can be flaky due to very slow disks - use tmpfs for temporary ClickHouse files - run_in_docker="altinityinfra/stateless-test+--network=host+--privileged+--cgroupns=host+root+--security-opt seccomp=unconfined+--tmpfs /tmp/clickhouse:mode=1777", + run_in_docker="altinityinfra/stateless-test+--network=host+--privileged+--cgroupns=host+root+--security-opt seccomp=unconfined+--ulimit nofile=1048576:1048576+--tmpfs /tmp/clickhouse:mode=1777", digest_config=Job.CacheDigestConfig( include_paths=[ "./ci/jobs/functional_tests.py", diff --git a/ci/docker/stateless-test/Dockerfile b/ci/docker/stateless-test/Dockerfile index a62578f0adc3..9ff9fe1c1646 100644 --- a/ci/docker/stateless-test/Dockerfile +++ b/ci/docker/stateless-test/Dockerfile @@ -109,6 +109,14 @@ ENV HOME="/home" RUN npm install -g azurite@3.35.0 \ && npm install -g tslib +# Download azurite-rs (Azure Blob Storage emulator in Rust) +# https://github.com/ClickHouse/azurite-rs +ARG AZURITE_RS_VERSION=v0.1.0-1bc0ba9 +RUN arch=${TARGETARCH:-amd64} \ + && curl -fsSL "https://github.com/ClickHouse/azurite-rs/releases/download/${AZURITE_RS_VERSION}/azurite-rs-linux-${arch}.tar.gz" | tar -xz -C /usr/local/bin/ \ + && mv /usr/local/bin/azurite-rs-linux-${arch} /usr/local/bin/azurite-rs \ + && chmod +x /usr/local/bin/azurite-rs + # Install Redpanda (Kafka-compatible broker, no JVM required) COPY --from=redpanda /opt/redpanda /opt/redpanda ENV PATH="/opt/redpanda/bin:${PATH}" diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index f63a1d9c6f7e..57e971e0fb86 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -214,6 +214,11 @@ def main(): elif "msan" in args.options: # MSan is slow nproc = int(Utils.cpu_count() * 0.4) + elif is_azure_storage: + # azure FT runs only under ASan; concurrent heavy queries overrun the + # shared server memory cap, so the OvercommitTracker kills queries across + # all co-scheduled tests. Lower concurrency to keep peak total RSS under it. + nproc = int(Utils.cpu_count() * 0.4) elif is_coverage: cidb_cluster = CIDBCluster() assert cidb_cluster.is_ready() diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index 7fb7e081f908..425f05094991 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -163,14 +163,21 @@ def start_minio(self, test_type): return False def start_azurite(self): + # Raise the open files limit before launching azurite-rs. + # Each concurrent test query opens a TCP connection plus an in-memory + # blob handle, and the default soft limit (1024) was exhausted under + # parallel load, causing `accept error: Too many open files`. + # Fall back to the hard limit if 1048576 cannot be set. command = ( - f"cd {temp_dir} && azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence", + f"cd {temp_dir} && " + "(ulimit -n 1048576 2>/dev/null || ulimit -n $(ulimit -Hn)) && " + "azurite-rs --host 0.0.0.0 --blob-port 10000 --silent --in-memory" ) with open(self.AZURITE_LOG, "w") as log_file: self.azurite_proc = subprocess.Popen( command, stdout=log_file, stderr=subprocess.STDOUT, shell=True ) - print(f"Started azurite asynchronously with PID {self.azurite_proc.pid}") + print(f"Started azurite-rs asynchronously with PID {self.azurite_proc.pid}") if Shell.check( "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:10000/ | grep -qE '400|200'", @@ -178,7 +185,7 @@ def start_azurite(self): retries=6, ): return True - print("Failed to start azurite") + print("Failed to start azurite-rs") return False def start_kafka(self): diff --git a/ci/jobs/stress_job.py b/ci/jobs/stress_job.py index 68bd5a9f01e0..68e13202ca01 100644 --- a/ci/jobs/stress_job.py +++ b/ci/jobs/stress_job.py @@ -106,6 +106,8 @@ def get_run_command( "docker run --cap-add=SYS_PTRACE " # For dmesg and sysctl "--privileged " + # azurite-rs (in-process Azure Blob Storage emulator) needs many fds under parallel load + "--ulimit nofile=1048576:1048576 " # a static link, don't use S3_URL or S3_DOWNLOAD "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' " "--tmpfs /tmp/clickhouse:mode=1777 " diff --git a/tests/docker_scripts/upgrade_runner.sh b/tests/docker_scripts/upgrade_runner.sh index 9e11d9a4b6fc..92634135a0bc 100755 --- a/tests/docker_scripts/upgrade_runner.sh +++ b/tests/docker_scripts/upgrade_runner.sh @@ -21,7 +21,7 @@ source /repo/tests/docker_scripts/attach_gdb.lib # shellcheck source=../stateless/stress_tests.lib source /repo/tests/docker_scripts/stress_tests.lib -azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & +cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; } cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_minio stateless || ( echo "Failed to start minio" && exit 1 ) # to have a proper environment echo "Get previous release tag" From d5faa357d5b9d795cf201465ceef4d39a7337479 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:30:41 -0400 Subject: [PATCH 3/3] Use DockerHub for redpanda. docker.redpanda.com rate-limits the stateless-test rebuild (429). 26.8 pulls redpandadata/redpanda from Docker Hub so the Hub proxy can serve it. --- ci/docker/stateless-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/stateless-test/Dockerfile b/ci/docker/stateless-test/Dockerfile index 9ff9fe1c1646..b79d47893b54 100644 --- a/ci/docker/stateless-test/Dockerfile +++ b/ci/docker/stateless-test/Dockerfile @@ -1,7 +1,7 @@ # docker build -t altinityinfra/stateless-test . ARG FROM_TAG=latest ARG REDPANDA_VERSION=v25.1.3 -FROM docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION} AS redpanda +FROM redpandadata/redpanda:${REDPANDA_VERSION} AS redpanda FROM altinityinfra/test-base:$FROM_TAG