Antalya 26.3 - replace azure with azurite

ci/defs/job_configs.py

@@ -84,7 +84,9 @@
     # some tests can be flaky due to very slow disks - use tmpfs for temporary ClickHouse files
     # --cap-add=SYS_PTRACE and --privileged for gdb in docker
     # --root/--privileged/--cgroupns=host is required for clickhouse-test --memory-limit
-    run_in_docker=f"altinityinfra/stateless-test+--memory={LIMITED_MEM}+--cgroupns=host+--cap-add=SYS_PTRACE+--privileged+--security-opt seccomp=unconfined+--tmpfs /tmp/clickhouse:mode=1777+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/etc/clickhouse-server1:/etc/clickhouse-server1+--volume=./ci/tmp/etc/clickhouse-server2:/etc/clickhouse-server2+--volume=./ci/tmp/var/log:/var/log+root+--env=AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY+--env=AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME+--env=AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME+--env=AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD",
+    # --ulimit nofile is raised so that azurite-rs (the in-process Azure Blob
+    # Storage emulator) does not run out of file descriptors under parallel load
+    run_in_docker=f"altinityinfra/stateless-test+--memory={LIMITED_MEM}+--cgroupns=host+--cap-add=SYS_PTRACE+--privileged+--security-opt seccomp=unconfined+--ulimit nofile=1048576:1048576+--tmpfs /tmp/clickhouse:mode=1777+--volume=./ci/tmp/var/lib/clickhouse:/var/lib/clickhouse+--volume=./ci/tmp/etc/clickhouse-client:/etc/clickhouse-client+--volume=./ci/tmp/etc/clickhouse-server:/etc/clickhouse-server+--volume=./ci/tmp/etc/clickhouse-server1:/etc/clickhouse-server1+--volume=./ci/tmp/etc/clickhouse-server2:/etc/clickhouse-server2+--volume=./ci/tmp/var/log:/var/log+root+--env=AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY+--env=AZURE_ACCOUNT_NAME=$AZURE_ACCOUNT_NAME+--env=AZURE_CONTAINER_NAME=$AZURE_CONTAINER_NAME+--env=AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL+--env=CLICKHOUSE_TEST_STAT_URL=$CLICKHOUSE_TEST_STAT_URL+--env=CLICKHOUSE_TEST_STAT_LOGIN=$CLICKHOUSE_TEST_STAT_LOGIN+--env=CLICKHOUSE_TEST_STAT_PASSWORD=$CLICKHOUSE_TEST_STAT_PASSWORD",
     digest_config=Job.CacheDigestConfig(
         include_paths=[
             "./ci/jobs/functional_tests.py",
@@ -540,7 +542,7 @@ class JobConfigs:
         runs_on=RunnerLabels.FUNC_TESTER_ARM,
         command="python3 ./ci/jobs/functional_tests.py --options BugfixValidation",
         # some tests can be flaky due to very slow disks - use tmpfs for temporary ClickHouse files
-        run_in_docker="altinityinfra/stateless-test+--network=host+--privileged+--cgroupns=host+root+--security-opt seccomp=unconfined+--tmpfs /tmp/clickhouse:mode=1777",
+        run_in_docker="altinityinfra/stateless-test+--network=host+--privileged+--cgroupns=host+root+--security-opt seccomp=unconfined+--ulimit nofile=1048576:1048576+--tmpfs /tmp/clickhouse:mode=1777",
         digest_config=Job.CacheDigestConfig(
             include_paths=[
                 "./ci/jobs/functional_tests.py",

ci/docker/stateless-test/Dockerfile

@@ -1,7 +1,7 @@
 # docker build -t altinityinfra/stateless-test .
 ARG FROM_TAG=latest
 ARG REDPANDA_VERSION=v25.1.3
-FROM docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION} AS redpanda
+FROM redpandadata/redpanda:${REDPANDA_VERSION} AS redpanda
 
 FROM altinityinfra/test-base:$FROM_TAG
 
@@ -109,6 +109,14 @@ ENV HOME="/home"
 RUN npm install -g azurite@3.35.0 \
     && npm install -g tslib
 
+# Download azurite-rs (Azure Blob Storage emulator in Rust)
+# https://github.com/ClickHouse/azurite-rs
+ARG AZURITE_RS_VERSION=v0.1.0-1bc0ba9
+RUN arch=${TARGETARCH:-amd64} \
+    && curl -fsSL "https://github.com/ClickHouse/azurite-rs/releases/download/${AZURITE_RS_VERSION}/azurite-rs-linux-${arch}.tar.gz" | tar -xz -C /usr/local/bin/ \
+    && mv /usr/local/bin/azurite-rs-linux-${arch} /usr/local/bin/azurite-rs \
+    && chmod +x /usr/local/bin/azurite-rs
+
 # Install Redpanda (Kafka-compatible broker, no JVM required)
 COPY --from=redpanda /opt/redpanda /opt/redpanda
 ENV PATH="/opt/redpanda/bin:${PATH}"

ci/jobs/functional_tests.py

@@ -214,6 +214,11 @@ def main():
         elif "msan" in args.options:
             # MSan is slow
             nproc = int(Utils.cpu_count() * 0.4)
+        elif is_azure_storage:
+            # azure FT runs only under ASan; concurrent heavy queries overrun the
+            # shared server memory cap, so the OvercommitTracker kills queries across
+            # all co-scheduled tests. Lower concurrency to keep peak total RSS under it.
+            nproc = int(Utils.cpu_count() * 0.4)
         elif is_coverage:
             cidb_cluster = CIDBCluster()
             assert cidb_cluster.is_ready()
@@ -247,26 +252,6 @@ def main():
         print(f"Rerun count set to 5 for targeted check")
         rerun_count = 5
 
-    if not info.is_local_run:
-        # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
-        # os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
-        #     f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
-        #     verbose=True,
-        # )
-        # NOTE(strtgbb): We pass azure credentials through the docker command, not SSM.
-        # NOTE(strtgbb): Azure credentials don't exist in community workflow
-        if info.is_community_pr:
-            print(
-                "NOTE: No azure credentials provided for community PR - disable azure storage"
-            )
-            config_installs_args += " --no-azure"
-
-            # NOTE(strtgbb): With the above, some tests are still trying to use azure, try this:
-            os.environ["USE_AZURE_STORAGE_FOR_MERGE_TREE"] = "0"
-    else:
-        print("Disable azure for a local run")
-        config_installs_args += " --no-azure"
-
     if (is_azure_storage or is_s3_storage) and is_encrypted_storage:
         config_installs_args += " --encrypted-storage"
         runner_options += f" --encrypted-storage"

ci/jobs/scripts/clickhouse_proc.py

@@ -163,15 +163,30 @@ def start_minio(self, test_type):
         return False
 
     def start_azurite(self):
+        # Raise the open files limit before launching azurite-rs.
+        # Each concurrent test query opens a TCP connection plus an in-memory
+        # blob handle, and the default soft limit (1024) was exhausted under
+        # parallel load, causing `accept error: Too many open files`.
+        # Fall back to the hard limit if 1048576 cannot be set.
         command = (
-            f"cd {temp_dir} && azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --silent --inMemoryPersistence",
+            f"cd {temp_dir} && "
+            "(ulimit -n 1048576 2>/dev/null || ulimit -n $(ulimit -Hn)) && "
+            "azurite-rs --host 0.0.0.0 --blob-port 10000 --silent --in-memory"
         )
         with open(self.AZURITE_LOG, "w") as log_file:
             self.azurite_proc = subprocess.Popen(
                 command, stdout=log_file, stderr=subprocess.STDOUT, shell=True
             )
-        print(f"Started azurite asynchronously with PID {self.azurite_proc.pid}")
-        return True
+        print(f"Started azurite-rs asynchronously with PID {self.azurite_proc.pid}")
+
+        if Shell.check(
+            "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:10000/ | grep -qE '400|200'",
+            verbose=False,
+            retries=6,
+        ):
+            return True
+        print("Failed to start azurite-rs")
+        return False
 
     def start_kafka(self):
         command = [
@@ -1358,6 +1373,8 @@ def set_random_timezone():
             param = sys.argv[2]
             assert param in ["stateless"]
             res = ch.start_minio(param)
+        elif command == "start_azurite":
+            res = ch.start_azurite()
         else:
             raise ValueError(f"Unknown command: {command}")
     except Exception as e:

ci/jobs/stress_job.py

@@ -62,21 +62,6 @@ def get_additional_envs(info, check_name: str) -> List[str]:
     from ci.jobs.ci_utils import is_extended_run
 
     result = []
-    if not info.is_local_run:
-        # azure_connection_string = Shell.get_output(
-        #     f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
-        #     verbose=True,
-        #     strict=True,
-        # )
-        # result.append(f"AZURE_CONNECTION_STRING='{azure_connection_string}'")
-        # NOTE(strtgbb): We pass azure credentials through the docker command, not SSM.
-        # NOTE(strtgbb): Azure credentials don't exist in community workflow
-        if info.is_community_pr:
-            print(
-                "NOTE: No azure credentials provided for community PR - disable azure storage"
-            )
-            os.environ["USE_AZURE_STORAGE_FOR_MERGE_TREE"] = "0"
-
     # some cloud-specific features require feature flags enabled
     # so we need this ENV to be able to disable the randomization
     # of feature flags
@@ -121,6 +106,8 @@ def get_run_command(
         "docker run --cap-add=SYS_PTRACE "
         # For dmesg and sysctl
         "--privileged "
+        # azurite-rs (in-process Azure Blob Storage emulator) needs many fds under parallel load
+        "--ulimit nofile=1048576:1048576 "
         # a static link, don't use S3_URL or S3_DOWNLOAD
         "-e S3_URL='https://s3.amazonaws.com/clickhouse-datasets' "
         "--tmpfs /tmp/clickhouse:mode=1777 "

tests/config/config.d/azure_storage_conf.xml

@@ -6,10 +6,8 @@
                 <object_storage_type>azure</object_storage_type>
                 <skip_access_check>false</skip_access_check>
                 <max_single_part_upload_size>33554432</max_single_part_upload_size>
-                <storage_account_url from_env="AZURE_STORAGE_ACCOUNT_URL"/>
-                <container_name from_env="AZURE_CONTAINER_NAME"/>
-                <account_name from_env="AZURE_ACCOUNT_NAME"/>
-                <account_key from_env="AZURE_STORAGE_KEY"/>
+                <container_name>clickhouse-tests</container_name>
+                <connection_string>DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;</connection_string>
             </azure>
             <cached_azure>
                 <type>cache</type>

tests/config/install.sh

@@ -356,7 +356,7 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
 fi
 
 if [[ "$EXPORT_S3_STORAGE_POLICIES" == "1" ]]; then
-    if [[ "$NO_AZURE" != "1" ]] && [[ -v AZURE_STORAGE_KEY ]]; then
+    if [[ "$NO_AZURE" != "1" ]]; then
         ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/
     fi
 

tests/docker_scripts/stress_runner.sh

@@ -53,6 +53,7 @@ configure
 cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py logs_export_config || echo "ERROR: Failed to create log export config"
 
 cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_minio stateless || { echo "Failed to start minio"; exit 1; }
+cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; }
 
 start_server || { echo "Failed to start server"; exit 1; }
 

tests/docker_scripts/upgrade_runner.sh

@@ -21,7 +21,7 @@ source /repo/tests/docker_scripts/attach_gdb.lib
 # shellcheck source=../stateless/stress_tests.lib
 source /repo/tests/docker_scripts/stress_tests.lib
 
-azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
+cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; }
 cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_minio stateless || ( echo "Failed to start minio" && exit 1 ) # to have a proper environment
 
 echo "Get previous release tag"