[#1092] Probe the Docker container's health without binding as the root user

.github/workflows/build.yml

@@ -586,10 +586,10 @@ jobs:
           cleanup() { docker rm -f test_master $REPLICAS >/dev/null 2>&1 || true; docker network rm test_replication >/dev/null 2>&1 || true; }
           cleanup
           trap 'code=$?; for c in test_master $REPLICAS; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR
-          # every tool reads the root password from a file (#1084); dsreplication run with -n prints
+          # every tool reads the root password from a file (#1084, #1092); dsreplication run with -n prints
           # no command line, so a password put back on one would pass every check below
-          rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/replicate.sh || rc=$?
-          if [ $rc -ne 1 ]; then echo "::error::replicate.sh passes the root password on a command line, or grep could not read it"; false; fi
+          rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh || rc=$?
+          if [ $rc -ne 1 ]; then echo "::error::setup.sh or replicate.sh passes the root password on a command line, or grep could not read them"; false; fi
           # the password file goes to /dev/shm, off the writable layer of the container, and the mktemp of the image puts it there
           docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.' /opt/opendj/bootstrap/replicate.sh || { echo "::error::replicate.sh no longer puts the password file on /dev/shm"; false; }
           docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-replicate.4444.*) ;; *) exit 1;; esac' || { echo "::error::mktemp in the image does not create the password file on /dev/shm"; false; }
@@ -636,13 +636,14 @@ jobs:
           if [ $rc -ne 5 ] || grep -qE "trying again|initializing replication" <<<"$out"; then
             echo "$out"; echo "::error::a second replicate.sh exited with $rc, not with the 5 of its dsreplication enable, or went on after it"; false
           fi
-          # the root password shows in no container log, and the file replicate.sh passed it in is gone (#1084)
+          # the root password shows in no container log, and the files setup.sh and replicate.sh passed it in are gone (#1084, #1092)
           for c in test_master $REPLICAS; do
             if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi
           done
-          for c in $REPLICAS; do
-            # the JVM of the HEALTHCHECK's ldapsearch keeps its command line, root password included, in /tmp/hsperfdata_* while it runs
-            left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm | grep -v '^/tmp/hsperfdata_' || true)
+          for c in test_master $REPLICAS; do
+            # a JVM keeps its command line in /tmp/hsperfdata_* while it runs; the HEALTHCHECK no longer binds as root (#1092),
+            # so no process left running has the root password on it
+            left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true)
             if [ -n "$left" ]; then echo "::error::The root password is left in $left of $c"; false; fi
           done
           docker exec test_replica test -e /dev/shm/opendj-replicate.5444.other || { echo "::error::run.sh of test_replica removed the password file of another container"; false; }
@@ -705,6 +706,59 @@ jobs:
           done
           docker exec test_bootstrap test -e "$shm_other" || { echo "::error::run.sh removed $shm_other, the password file of another container"; false; }
           docker kill test_bootstrap test_bootstrap_shm
+      - name: Docker test health check
+        shell: bash
+        run: |
+          # the ERR trap below has to fire for a check failing inside stays_healthy too
+          set -o errtrace
+          # the containers are run without --rm, so that one whose bootstrap failed is still there for the trap to print
+          trap 'code=$?; for c in test_health test_health_bind; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; docker inspect --format="{{json .State.Health}}" $c 2>&1 || true; echo "::endgroup::"; done; docker rm -f test_health test_health_bind >/dev/null 2>&1 || true; exit $code' ERR
+          IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}
+          # a failed probe leaves a failing streak until the next probe passes, 5 s later at the
+          # earliest, so a container found "healthy 0" every 2 s for 45 s passed every probe since
+          stays_healthy() {
+            local end=$((SECONDS + 45))
+            while [ $SECONDS -lt $end ]; do
+              test "$(docker inspect --format='{{.State.Health.Status}} {{.State.Health.FailingStreak}}' "$1")" = "healthy 0"
+              sleep 2
+            done
+          }
+          # ROOT_PASSWORD is only the initial root password: changing it must not turn the container unhealthy
+          docker run -it -d --memory="512m" --health-interval=5s -e ROOT_PASSWORD=initial_password --name=test_health $IMAGE
+          timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_health | grep -q \"healthy\"; do sleep 10; done'
+          docker exec test_health /opt/opendj/bin/ldappasswordmodify --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword initial_password --currentPassword initial_password --newPassword rotated_password
+          # bind settings kept for the CLI in the home of the image user must not reach the probe
+          docker exec test_health sh -c 'mkdir -p /home/opendj/.opendj && printf "bindDN=cn=Directory Manager\nbindPassword=wrong_password\n" > /home/opendj/.opendj/tools.properties'
+          stays_healthy test_health
+          docker rm -f test_health
+          # an instance rejecting unauthenticated requests is probed with the account it is given, whose password is read from a file
+          # a password of its own, so the command lines below can be searched for it
+          printf hc_secret_1092 > "$RUNNER_TEMP/healthcheck_password"
+          chmod 644 "$RUNNER_TEMP/healthcheck_password"
+          docker run -it -d --memory="512m" --health-interval=5s -v "$RUNNER_TEMP/healthcheck_password:/tmp/healthcheck_password:ro" -e ROOT_PASSWORD=hc_secret_1092 -e HEALTHCHECK_BIND_DN="cn=Directory Manager" -e HEALTHCHECK_BIND_PASSWORD_FILE=/tmp/healthcheck_password --name=test_health_bind $IMAGE
+          timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_health_bind | grep -q \"healthy\"; do sleep 10; done'
+          docker exec test_health_bind /opt/opendj/bin/dsconfig set-global-configuration-prop --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPasswordFile /tmp/healthcheck_password --set reject-unauthenticated-requests:true --no-prompt --trustAll
+          # the setting has taken: the anonymous probe would now be refused
+          rc=0
+          docker exec test_health_bind /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --baseDN "" --searchScope base "(objectClass=*)" 1.1 || rc=$?
+          test "$rc" = 53
+          # and the probe reports that refusal when it has no account to bind with
+          rc=0
+          docker exec -e HEALTHCHECK_BIND_DN= test_health_bind /opt/opendj/healthcheck.sh || rc=$?
+          test "$rc" = 1
+          # a password file it cannot read is reported as such, whether it is missing or there but not
+          # readable: the image runs as its own user, not root, so mode 000 keeps the probe out
+          docker exec test_health_bind sh -c 'touch /tmp/unreadable_password && chmod 000 /tmp/unreadable_password'
+          for f in /nonexistent /tmp/unreadable_password; do
+            rc=0
+            out=$(docker exec -e HEALTHCHECK_BIND_PASSWORD_FILE=$f test_health_bind /opt/opendj/healthcheck.sh) || rc=$?
+            test "$rc" = 1
+            grep -q 'is not a readable file' <<< "$out"
+          done
+          stays_healthy test_health_bind
+          # the password never shows on a command line: sample every process's for two probe intervals
+          docker exec test_health_bind sh -c 'end=$(($(date +%s) + 12)); while [ "$(date +%s)" -lt "$end" ]; do for f in /proc/[0-9]*/cmdline; do tr "\0" " " < "$f" 2>/dev/null; echo; done | grep -q "[h]c_secret_1092" && exit 1; sleep 0.2; done; exit 0'
+          docker rm -f test_health_bind
       - name: Scan image for vulnerabilities (Trivy)
         # trivy resolves the image from the local Docker daemon, so only the runner's
         # linux/amd64 manifest is scanned; cache: false keeps the ~1GB trivy DBs from
@@ -894,10 +948,10 @@ jobs:
           cleanup() { docker rm -f test_master $REPLICAS >/dev/null 2>&1 || true; docker network rm test_replication >/dev/null 2>&1 || true; }
           cleanup
           trap 'code=$?; for c in test_master $REPLICAS; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR
-          # every tool reads the root password from a file (#1084); dsreplication run with -n prints
+          # every tool reads the root password from a file (#1084, #1092); dsreplication run with -n prints
           # no command line, so a password put back on one would pass every check below
-          rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/replicate.sh || rc=$?
-          if [ $rc -ne 1 ]; then echo "::error::replicate.sh passes the root password on a command line, or grep could not read it"; false; fi
+          rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh || rc=$?
+          if [ $rc -ne 1 ]; then echo "::error::setup.sh or replicate.sh passes the root password on a command line, or grep could not read them"; false; fi
           # the password file goes to /dev/shm, off the writable layer of the container, and the mktemp of the image puts it there
           docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.' /opt/opendj/bootstrap/replicate.sh || { echo "::error::replicate.sh no longer puts the password file on /dev/shm"; false; }
           docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-replicate.4444.*) ;; *) exit 1;; esac' || { echo "::error::mktemp in the image does not create the password file on /dev/shm"; false; }
@@ -944,13 +998,14 @@ jobs:
           if [ $rc -ne 5 ] || grep -qE "trying again|initializing replication" <<<"$out"; then
             echo "$out"; echo "::error::a second replicate.sh exited with $rc, not with the 5 of its dsreplication enable, or went on after it"; false
           fi
-          # the root password shows in no container log, and the file replicate.sh passed it in is gone (#1084)
+          # the root password shows in no container log, and the files setup.sh and replicate.sh passed it in are gone (#1084, #1092)
           for c in test_master $REPLICAS; do
             if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi
           done
-          for c in $REPLICAS; do
-            # the JVM of the HEALTHCHECK's ldapsearch keeps its command line, root password included, in /tmp/hsperfdata_* while it runs
-            left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm | grep -v '^/tmp/hsperfdata_' || true)
+          for c in test_master $REPLICAS; do
+            # a JVM keeps its command line in /tmp/hsperfdata_* while it runs; the HEALTHCHECK no longer binds as root (#1092),
+            # so no process left running has the root password on it
+            left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true)
             if [ -n "$left" ]; then echo "::error::The root password is left in $left of $c"; false; fi
           done
           docker exec test_replica test -e /dev/shm/opendj-replicate.5444.other || { echo "::error::run.sh of test_replica removed the password file of another container"; false; }
@@ -1013,6 +1068,59 @@ jobs:
           done
           docker exec test_bootstrap test -e "$shm_other" || { echo "::error::run.sh removed $shm_other, the password file of another container"; false; }
           docker kill test_bootstrap test_bootstrap_shm
+      - name: Docker test health check
+        shell: bash
+        run: |
+          # the ERR trap below has to fire for a check failing inside stays_healthy too
+          set -o errtrace
+          # the containers are run without --rm, so that one whose bootstrap failed is still there for the trap to print
+          trap 'code=$?; for c in test_health test_health_bind; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; docker inspect --format="{{json .State.Health}}" $c 2>&1 || true; echo "::endgroup::"; done; docker rm -f test_health test_health_bind >/dev/null 2>&1 || true; exit $code' ERR
+          IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine
+          # a failed probe leaves a failing streak until the next probe passes, 5 s later at the
+          # earliest, so a container found "healthy 0" every 2 s for 45 s passed every probe since
+          stays_healthy() {
+            local end=$((SECONDS + 45))
+            while [ $SECONDS -lt $end ]; do
+              test "$(docker inspect --format='{{.State.Health.Status}} {{.State.Health.FailingStreak}}' "$1")" = "healthy 0"
+              sleep 2
+            done
+          }
+          # ROOT_PASSWORD is only the initial root password: changing it must not turn the container unhealthy
+          docker run -it -d --memory="1g" --health-interval=5s -e ROOT_PASSWORD=initial_password --name=test_health $IMAGE
+          timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_health | grep -q \"healthy\"; do sleep 10; done'
+          docker exec test_health /opt/opendj/bin/ldappasswordmodify --hostname localhost --port 1636 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword initial_password --currentPassword initial_password --newPassword rotated_password
+          # bind settings kept for the CLI in the home of the image user must not reach the probe
+          docker exec test_health sh -c 'mkdir -p /home/opendj/.opendj && printf "bindDN=cn=Directory Manager\nbindPassword=wrong_password\n" > /home/opendj/.opendj/tools.properties'
+          stays_healthy test_health
+          docker rm -f test_health
+          # an instance rejecting unauthenticated requests is probed with the account it is given, whose password is read from a file
+          # a password of its own, so the command lines below can be searched for it
+          printf hc_secret_1092 > "$RUNNER_TEMP/healthcheck_password"
+          chmod 644 "$RUNNER_TEMP/healthcheck_password"
+          docker run -it -d --memory="1g" --health-interval=5s -v "$RUNNER_TEMP/healthcheck_password:/tmp/healthcheck_password:ro" -e ROOT_PASSWORD=hc_secret_1092 -e HEALTHCHECK_BIND_DN="cn=Directory Manager" -e HEALTHCHECK_BIND_PASSWORD_FILE=/tmp/healthcheck_password --name=test_health_bind $IMAGE
+          timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_health_bind | grep -q \"healthy\"; do sleep 10; done'
+          docker exec test_health_bind /opt/opendj/bin/dsconfig set-global-configuration-prop --hostname localhost --port 4444 --bindDN "cn=Directory Manager" --bindPasswordFile /tmp/healthcheck_password --set reject-unauthenticated-requests:true --no-prompt --trustAll
+          # the setting has taken: the anonymous probe would now be refused
+          rc=0
+          docker exec test_health_bind /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --useSsl --trustAll --baseDN "" --searchScope base "(objectClass=*)" 1.1 || rc=$?
+          test "$rc" = 53
+          # and the probe reports that refusal when it has no account to bind with
+          rc=0
+          docker exec -e HEALTHCHECK_BIND_DN= test_health_bind /opt/opendj/healthcheck.sh || rc=$?
+          test "$rc" = 1
+          # a password file it cannot read is reported as such, whether it is missing or there but not
+          # readable: the image runs as its own user, not root, so mode 000 keeps the probe out
+          docker exec test_health_bind sh -c 'touch /tmp/unreadable_password && chmod 000 /tmp/unreadable_password'
+          for f in /nonexistent /tmp/unreadable_password; do
+            rc=0
+            out=$(docker exec -e HEALTHCHECK_BIND_PASSWORD_FILE=$f test_health_bind /opt/opendj/healthcheck.sh) || rc=$?
+            test "$rc" = 1
+            grep -q 'is not a readable file' <<< "$out"
+          done
+          stays_healthy test_health_bind
+          # the password never shows on a command line: sample every process's for two probe intervals
+          docker exec test_health_bind sh -c 'end=$(($(date +%s) + 12)); while [ "$(date +%s)" -lt "$end" ]; do for f in /proc/[0-9]*/cmdline; do tr "\0" " " < "$f" 2>/dev/null; echo; done | grep -q "[h]c_secret_1092" && exit 1; sleep 0.2; done; exit 0'
+          docker rm -f test_health_bind
       - name: Scan image for vulnerabilities (Trivy)
         # trivy resolves the image from the local Docker daemon, so only the runner's
         # linux/amd64 manifest is scanned; cache: false keeps the ~1GB trivy DBs from

opendj-packages/opendj-docker/Dockerfile

@@ -65,8 +65,9 @@ RUN  printf 'Acquire::ForceIPv4 "true";\nAcquire::Retries "5";\n' > /etc/apt/apt
 # root. The scripts copied below are only read and run, so they just keep the same group.
 COPY --chown=$OPENDJ_USER:0 bootstrap/ /opt/opendj/bootstrap/
 COPY --chown=$OPENDJ_USER:0 run.sh /opt/opendj/run.sh
+COPY --chown=$OPENDJ_USER:0 healthcheck.sh /opt/opendj/healthcheck.sh
 
-RUN chmod +x /opt/opendj/run.sh /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh
+RUN chmod +x /opt/opendj/run.sh /opt/opendj/healthcheck.sh /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh
 
 EXPOSE $PORT/tcp $LDAPS_PORT/tcp $ADMIN_PORT/tcp
 
@@ -75,10 +76,11 @@ USER $OPENDJ_USER
 # "healthy" has to mean the instance is ready to serve, not just that it answers: setup
 # starts the server in the middle of the bootstrap, before the backend of BASE_DN is
 # created and its entries imported, so probing the root DSE alone reports ready while a
-# search of BASE_DN still fails with "No Such Entry". Testing the marker first also keeps
-# the probe from launching a JVM every interval until the bootstrap is through. The start
-# period is what a bootstrap importing SAMPLE_DATA into a small container can take; a
-# probe that succeeds ends it early, and a bootstrap that failed never writes the marker.
-HEALTHCHECK --interval=30s --timeout=30s --start-period=5m --retries=3 CMD test -f "$BOOTSTRAP_COMPLETE" && opendj/bin/ldapsearch --hostname localhost --port $LDAPS_PORT --bindDN "$ROOT_USER_DN" --bindPassword "${ROOT_PASSWORD:-password}" --useSsl --trustAll --baseDN "" --searchScope base "(objectClass=*)" 1.1 || exit 1
+# search of BASE_DN still fails with "No Such Entry". healthcheck.sh tests the marker
+# first, then searches the root DSE without binding as the root user, whose password the
+# operator is expected to change. The start period is what a bootstrap importing
+# SAMPLE_DATA into a small container can take; a probe that succeeds ends it early, and a
+# bootstrap that failed never writes the marker.
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5m --retries=3 CMD ["/opt/opendj/healthcheck.sh"]
 
 ENTRYPOINT ["/opt/opendj/run.sh"]