From eefafc4b58d4bc83e567a474ac6c9af1fbed55d5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 24 Aug 2026 10:32:13 -0700 Subject: [PATCH 01/32] ci: increase kokoro timeout to 360 mins for system tests --- .kokoro/continuous/system.cfg | 3 +++ .kokoro/presubmit/system.cfg | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.kokoro/continuous/system.cfg b/.kokoro/continuous/system.cfg index 789455bd6973..d5dc63b5a58f 100644 --- a/.kokoro/continuous/system.cfg +++ b/.kokoro/continuous/system.cfg @@ -1,5 +1,8 @@ # Format: //devtools/kokoro/config/proto/build.proto +# Increase timeout to 6 hours (default is 180 mins / 3 hours) +timeout_mins: 360 + # Only run this nox session. env_vars: { key: "NOX_SESSION" diff --git a/.kokoro/presubmit/system.cfg b/.kokoro/presubmit/system.cfg index 789455bd6973..d5dc63b5a58f 100644 --- a/.kokoro/presubmit/system.cfg +++ b/.kokoro/presubmit/system.cfg @@ -1,5 +1,8 @@ # Format: //devtools/kokoro/config/proto/build.proto +# Increase timeout to 6 hours (default is 180 mins / 3 hours) +timeout_mins: 360 + # Only run this nox session. env_vars: { key: "NOX_SESSION" From c19b0cb30af7aca7907ce804bc045b38c1bea0fb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 24 Aug 2026 15:00:22 -0700 Subject: [PATCH 02/32] added package timeouts --- .kokoro/system.sh | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 91d96e7a1497..bbfef0e7c9da 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -128,6 +128,7 @@ run_package_test() { reap_parallel_results() { local retval=0 local failed_count=0 + local timed_out_count=0 local succeeded_count=0 if [ -z "$LOG_DIR" ]; then @@ -142,6 +143,13 @@ reap_parallel_results() { fi done + # Count timed out packages by checking for .timed_out marker files + for timed_out in "$LOG_DIR"/*.timed_out; do + if [ -f "$timed_out" ]; then + timed_out_count=$((timed_out_count + 1)) + fi + done + local total_tested=${#PACKAGES_TO_TEST[@]} succeeded_count=$((total_tested - failed_count)) @@ -152,6 +160,9 @@ reap_parallel_results() { echo "Total Packages: $total_tested" echo "Succeeded: $succeeded_count" echo "Failed: $failed_count" + if [ "$timed_out_count" -gt 0 ]; then + echo "Timed Out: $timed_out_count" + fi echo "==================================================" local succeeded_packages=() @@ -168,14 +179,23 @@ reap_parallel_results() { # List failed packages for failed in "$LOG_DIR"/*.failed; do if [ -f "$failed" ]; then - basename "$failed" .failed + local pkg=$(basename "$failed" .failed) + if [ -f "$LOG_DIR/$pkg.timed_out" ]; then + echo "$pkg (TIMED OUT after ${PACKAGE_TEST_TIMEOUT})" + else + echo "$pkg" + fi fi done for failed in "$LOG_DIR"/*.failed; do if [ -f "$failed" ]; then local pkg=$(basename "$failed" .failed) echo "--------------------------------------------------" - echo "@PACKAGE (FAILED): $pkg" + if [ -f "$LOG_DIR/$pkg.timed_out" ]; then + echo "@PACKAGE (TIMED OUT after ${PACKAGE_TEST_TIMEOUT}): $pkg" + else + echo "@PACKAGE (FAILED): $pkg" + fi echo "--------------------------------------------------" if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" @@ -287,6 +307,7 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} +PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-35m}" # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) @@ -301,9 +322,10 @@ fi echo "==================================================" echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages" echo "Concurrency limit: ${MAX_JOBS}" +echo "Per-package timeout: ${PACKAGE_TEST_TIMEOUT}" echo "==================================================" -export LOG_DIR +export LOG_DIR PACKAGE_TEST_TIMEOUT export -f run_package_test export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR @@ -323,8 +345,22 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \ log_file="$LOG_DIR/$pkg.log" fi - # Run test; if it fails, create a .failed file to signal failure to the reaper - run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed" + # Run test with timeout guard; if it fails or times out, mark as failed + set +e + timeout --kill-after=1m "${PACKAGE_TEST_TIMEOUT}" bash -c '\''run_package_test "$0"'\'' "$pkg" > "$log_file" 2>&1 + status=$? + set -e + + if [ $status -eq 124 ] || [ $status -eq 137 ]; then + echo "" >> "$log_file" + echo "==================================================" >> "$log_file" + echo "ERROR: Package test timed out after ${PACKAGE_TEST_TIMEOUT}!" >> "$log_file" + echo "==================================================" >> "$log_file" + touch "$LOG_DIR/$pkg.timed_out" + touch "$LOG_DIR/$pkg.failed" + elif [ $status -ne 0 ]; then + touch "$LOG_DIR/$pkg.failed" + fi ' reap_parallel_results || RETVAL=1 From 85a4461a86159eeb498209cf3d79567b4f6b0008 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 24 Aug 2026 15:01:20 -0700 Subject: [PATCH 03/32] touch changelogs to trigger tests --- packages/bigquery-magics/CHANGELOG.md | 2 ++ packages/django-google-spanner/CHANGELOG.md | 2 ++ packages/gapic-generator/CHANGELOG.md | 2 ++ packages/google-auth/CHANGELOG.md | 2 ++ packages/google-cloud-access-approval/CHANGELOG.md | 2 ++ packages/google-cloud-automl/CHANGELOG.md | 2 ++ packages/google-cloud-bigquery-connection/CHANGELOG.md | 2 ++ packages/google-cloud-bigquery-datatransfer/CHANGELOG.md | 2 ++ packages/google-cloud-bigquery-reservation/CHANGELOG.md | 2 ++ packages/google-cloud-bigquery-storage/CHANGELOG.md | 2 ++ packages/google-cloud-bigquery/CHANGELOG.md | 2 ++ packages/google-cloud-bigtable/CHANGELOG.md | 2 ++ packages/google-cloud-compute/CHANGELOG.md | 2 ++ packages/google-cloud-container/CHANGELOG.md | 2 ++ packages/google-cloud-dataproc/CHANGELOG.md | 2 ++ packages/google-cloud-datastore/CHANGELOG.md | 2 ++ packages/google-cloud-dlp/CHANGELOG.md | 2 ++ packages/google-cloud-dns/CHANGELOG.md | 2 ++ packages/google-cloud-error-reporting/CHANGELOG.md | 2 ++ packages/google-cloud-firestore/CHANGELOG.md | 2 ++ packages/google-cloud-kms/CHANGELOG.md | 2 ++ packages/google-cloud-logging/CHANGELOG.md | 2 ++ packages/google-cloud-monitoring/CHANGELOG.md | 2 ++ packages/google-cloud-ndb/CHANGELOG.md | 2 ++ packages/google-cloud-os-config/CHANGELOG.md | 2 ++ packages/google-cloud-pubsub/CHANGELOG.md | 2 ++ packages/google-cloud-scheduler/CHANGELOG.md | 2 ++ packages/google-cloud-spanner-dbapi-driver/setup.py | 2 ++ packages/google-cloud-spanner/CHANGELOG.md | 2 ++ packages/google-cloud-speech/CHANGELOG.md | 2 ++ packages/google-cloud-storage/CHANGELOG.md | 2 ++ packages/google-cloud-tasks/CHANGELOG.md | 2 ++ packages/google-cloud-testutils/CHANGELOG.md | 2 ++ packages/google-cloud-texttospeech/CHANGELOG.md | 2 ++ packages/google-cloud-translate/CHANGELOG.md | 2 ++ packages/google-cloud-videointelligence/CHANGELOG.md | 2 ++ packages/google-cloud-vision/CHANGELOG.md | 2 ++ packages/google-resumable-media/CHANGELOG.md | 2 ++ packages/pandas-gbq/CHANGELOG.md | 2 ++ packages/sqlalchemy-bigquery/CHANGELOG.md | 2 ++ packages/sqlalchemy-spanner/CHANGELOG.md | 2 ++ 41 files changed, 82 insertions(+) diff --git a/packages/bigquery-magics/CHANGELOG.md b/packages/bigquery-magics/CHANGELOG.md index a1c4b1a94d0d..33e8c23fff10 100644 --- a/packages/bigquery-magics/CHANGELOG.md +++ b/packages/bigquery-magics/CHANGELOG.md @@ -192,3 +192,5 @@ * Clean up docs and let the nox pass ([#24](https://github.com/googleapis/python-bigquery-magics/issues/24)) ([275712f](https://github.com/googleapis/python-bigquery-magics/commit/275712f4e4b647cda2d253e1f6b7a2fa093ee7c1)) * Reset the changelog for the new package ([#22](https://github.com/googleapis/python-bigquery-magics/issues/22)) ([f7d9c14](https://github.com/googleapis/python-bigquery-magics/commit/f7d9c1445feac32e468a3e06ca55c9474a1ae548)) + + diff --git a/packages/django-google-spanner/CHANGELOG.md b/packages/django-google-spanner/CHANGELOG.md index f457528bc01c..981528a1bc39 100644 --- a/packages/django-google-spanner/CHANGELOG.md +++ b/packages/django-google-spanner/CHANGELOG.md @@ -328,3 +328,5 @@ * minor fixes to README.md ([#448](https://www.github.com/googleapis/python-spanner-django/issues/448)) ([f969000](https://www.github.com/googleapis/python-spanner-django/commit/f9690007603c94f4c99b244a92c639adfd360a8f)) * move test suite information to CONTRIBUTING.md ([#442](https://www.github.com/googleapis/python-spanner-django/issues/442)) ([05280ae](https://www.github.com/googleapis/python-spanner-django/commit/05280aecdcbe933e113616b5705f4e76303d9637)) + + diff --git a/packages/gapic-generator/CHANGELOG.md b/packages/gapic-generator/CHANGELOG.md index cc8db91f3c62..bf09511db9f9 100644 --- a/packages/gapic-generator/CHANGELOG.md +++ b/packages/gapic-generator/CHANGELOG.md @@ -2120,3 +2120,5 @@ ### Bug Fixes * update GOOGLE_API_USE_MTLS value ([#453](https://www.github.com/googleapis/gapic-generator-python/issues/453)) ([7449ad5](https://www.github.com/googleapis/gapic-generator-python/commit/7449ad5aad4a1fbbf9ca3796e097512fc80991e3)) + + diff --git a/packages/google-auth/CHANGELOG.md b/packages/google-auth/CHANGELOG.md index c2a95fc8057d..3bbd55d21306 100644 --- a/packages/google-auth/CHANGELOG.md +++ b/packages/google-auth/CHANGELOG.md @@ -1951,3 +1951,5 @@ Initial release with foundational functionality for cryptography and JWTs. - ``google.auth.crypt`` for creating and verifying cryptographic signatures. - ``google.auth.jwt`` for creating (encoding) and verifying (decoding) JSON Web tokens. + + diff --git a/packages/google-cloud-access-approval/CHANGELOG.md b/packages/google-cloud-access-approval/CHANGELOG.md index 75e88ce04461..28601a48d52a 100644 --- a/packages/google-cloud-access-approval/CHANGELOG.md +++ b/packages/google-cloud-access-approval/CHANGELOG.md @@ -467,3 +467,5 @@ ### Features * generate v1 ([88003fe](https://www.github.com/googleapis/python-access-approval/commit/88003fe05150ee653ba9a8ba072058b35d3f3c49)) + + diff --git a/packages/google-cloud-automl/CHANGELOG.md b/packages/google-cloud-automl/CHANGELOG.md index 3f450acd87b4..9a58c4886db7 100644 --- a/packages/google-cloud-automl/CHANGELOG.md +++ b/packages/google-cloud-automl/CHANGELOG.md @@ -713,3 +713,5 @@ ### New Features - Initial Release of AutoML v1beta1 + + diff --git a/packages/google-cloud-bigquery-connection/CHANGELOG.md b/packages/google-cloud-bigquery-connection/CHANGELOG.md index 296b3a5de90c..1200e8abf102 100644 --- a/packages/google-cloud-bigquery-connection/CHANGELOG.md +++ b/packages/google-cloud-bigquery-connection/CHANGELOG.md @@ -506,3 +506,5 @@ ### Features * generate v1 ([73b89dc](https://www.github.com/googleapis/python-bigquery-connection/commit/73b89dcb423026c4b4e537ff728d22be2cb5ff3f)) + + diff --git a/packages/google-cloud-bigquery-datatransfer/CHANGELOG.md b/packages/google-cloud-bigquery-datatransfer/CHANGELOG.md index ade076830fa1..0237fb289169 100644 --- a/packages/google-cloud-bigquery-datatransfer/CHANGELOG.md +++ b/packages/google-cloud-bigquery-datatransfer/CHANGELOG.md @@ -709,3 +709,5 @@ sources like Adwords, DoubleClick Campaign Manager, DoubleClick for Publishers and YouTube. PyPI: https://pypi.org/project/google-cloud-bigquery-datatransfer/0.1.0/ + + diff --git a/packages/google-cloud-bigquery-reservation/CHANGELOG.md b/packages/google-cloud-bigquery-reservation/CHANGELOG.md index bb8f77b10713..16327e6e1825 100644 --- a/packages/google-cloud-bigquery-reservation/CHANGELOG.md +++ b/packages/google-cloud-bigquery-reservation/CHANGELOG.md @@ -519,3 +519,5 @@ ### Features * generate v1 ([6293404](https://www.github.com/googleapis/python-bigquery-reservation/commit/6293404e47ca2efdcb5f702e248f43250060eb8c)) + + diff --git a/packages/google-cloud-bigquery-storage/CHANGELOG.md b/packages/google-cloud-bigquery-storage/CHANGELOG.md index 7b35fed906bd..8756f17ced6c 100644 --- a/packages/google-cloud-bigquery-storage/CHANGELOG.md +++ b/packages/google-cloud-bigquery-storage/CHANGELOG.md @@ -889,3 +889,5 @@ 11-29-2018 13:45 PST - Initial release of BigQuery Storage API client. + + diff --git a/packages/google-cloud-bigquery/CHANGELOG.md b/packages/google-cloud-bigquery/CHANGELOG.md index 02222faddffb..bdce16d6e473 100644 --- a/packages/google-cloud-bigquery/CHANGELOG.md +++ b/packages/google-cloud-bigquery/CHANGELOG.md @@ -2779,3 +2779,5 @@ PyPI: https://pypi.org/project/google-cloud-bigquery/0.27.0/ (#3598) PyPI: https://pypi.org/project/google-cloud-bigquery/0.26.0/ + + diff --git a/packages/google-cloud-bigtable/CHANGELOG.md b/packages/google-cloud-bigtable/CHANGELOG.md index f50fa92ad9dc..becc28723b1b 100644 --- a/packages/google-cloud-bigtable/CHANGELOG.md +++ b/packages/google-cloud-bigtable/CHANGELOG.md @@ -1226,3 +1226,5 @@ PyPI: https://pypi.org/project/google-cloud-bigtable/0.28.1/ on `google-api-core` (#4221, #4280) PyPI: https://pypi.org/project/google-cloud-bigtable/0.28.0/ + + diff --git a/packages/google-cloud-compute/CHANGELOG.md b/packages/google-cloud-compute/CHANGELOG.md index 80f3e311aa90..a81690d26f00 100644 --- a/packages/google-cloud-compute/CHANGELOG.md +++ b/packages/google-cloud-compute/CHANGELOG.md @@ -774,3 +774,5 @@ ### Features * generate v1 ([53f9a3d](https://www.github.com/googleapis/python-compute/commit/53f9a3d6f14ef45b5bc3e38a48e3fa17059591eb)) + + diff --git a/packages/google-cloud-container/CHANGELOG.md b/packages/google-cloud-container/CHANGELOG.md index 90d9704399ed..e3ca4f009dcf 100644 --- a/packages/google-cloud-container/CHANGELOG.md +++ b/packages/google-cloud-container/CHANGELOG.md @@ -1396,3 +1396,5 @@ resource efficiency, automated operations, and open source flexibility to accelerate your time to market. PyPI: https://pypi.org/project/google-cloud-container/0.1.0/ + + diff --git a/packages/google-cloud-dataproc/CHANGELOG.md b/packages/google-cloud-dataproc/CHANGELOG.md index a89333498759..263c4025cdd0 100644 --- a/packages/google-cloud-dataproc/CHANGELOG.md +++ b/packages/google-cloud-dataproc/CHANGELOG.md @@ -832,3 +832,5 @@ - Re-enable lint for tests, remove usage of pylint (#4921) - Normalize all setup.py files (#4909) + + diff --git a/packages/google-cloud-datastore/CHANGELOG.md b/packages/google-cloud-datastore/CHANGELOG.md index 901dbfca5794..412f26faf13f 100644 --- a/packages/google-cloud-datastore/CHANGELOG.md +++ b/packages/google-cloud-datastore/CHANGELOG.md @@ -821,3 +821,5 @@ on `google-api-core` (#4221, #4280) PyPI: https://pypi.org/project/google-cloud-datastore/1.4.0/ + + diff --git a/packages/google-cloud-dlp/CHANGELOG.md b/packages/google-cloud-dlp/CHANGELOG.md index dc4fc8f2c7a6..bf967aa1b391 100644 --- a/packages/google-cloud-dlp/CHANGELOG.md +++ b/packages/google-cloud-dlp/CHANGELOG.md @@ -919,3 +919,5 @@ ## 0.1.0 Initial release of the DLP (Data Loss Prevention) client library. (#4879) + + diff --git a/packages/google-cloud-dns/CHANGELOG.md b/packages/google-cloud-dns/CHANGELOG.md index 13b61213dcdf..65ef9cf94613 100644 --- a/packages/google-cloud-dns/CHANGELOG.md +++ b/packages/google-cloud-dns/CHANGELOG.md @@ -241,3 +241,5 @@ on `google-api-core` (#4221, #4280) PyPI: https://pypi.org/project/google-cloud-dns/0.28.0/ + + diff --git a/packages/google-cloud-error-reporting/CHANGELOG.md b/packages/google-cloud-error-reporting/CHANGELOG.md index 7aa4c7c198c2..501013dfd1b9 100644 --- a/packages/google-cloud-error-reporting/CHANGELOG.md +++ b/packages/google-cloud-error-reporting/CHANGELOG.md @@ -539,3 +539,5 @@ - Upgrading to `google-cloud-logging >= 1.4.0` (#4296) PyPI: https://pypi.org/project/google-cloud-error-reporting/0.28.0/ + + diff --git a/packages/google-cloud-firestore/CHANGELOG.md b/packages/google-cloud-firestore/CHANGELOG.md index 90a2ec524f0c..f9fff9d1f62e 100644 --- a/packages/google-cloud-firestore/CHANGELOG.md +++ b/packages/google-cloud-firestore/CHANGELOG.md @@ -1242,3 +1242,5 @@ on `google-api-core` (#4221, #4280) PyPI: https://pypi.org/project/google-cloud-firestore/0.28.0/ + + diff --git a/packages/google-cloud-kms/CHANGELOG.md b/packages/google-cloud-kms/CHANGELOG.md index 819709005334..a38ff41abdf5 100644 --- a/packages/google-cloud-kms/CHANGELOG.md +++ b/packages/google-cloud-kms/CHANGELOG.md @@ -831,3 +831,5 @@ ### New Features - KMS v1 + + diff --git a/packages/google-cloud-logging/CHANGELOG.md b/packages/google-cloud-logging/CHANGELOG.md index 73f9e621f9e4..94237a6206ec 100644 --- a/packages/google-cloud-logging/CHANGELOG.md +++ b/packages/google-cloud-logging/CHANGELOG.md @@ -917,3 +917,5 @@ `googleapis-common-protos` dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-logging/1.4.0/ + + diff --git a/packages/google-cloud-monitoring/CHANGELOG.md b/packages/google-cloud-monitoring/CHANGELOG.md index 91501e1a575d..c68c146d3875 100644 --- a/packages/google-cloud-monitoring/CHANGELOG.md +++ b/packages/google-cloud-monitoring/CHANGELOG.md @@ -856,3 +856,5 @@ on `google-api-core` (#4221, #4280) PyPI: https://pypi.org/project/google-cloud-monitoring/0.28.0/ + + diff --git a/packages/google-cloud-ndb/CHANGELOG.md b/packages/google-cloud-ndb/CHANGELOG.md index 943cbb2d7123..9dd5b695939e 100644 --- a/packages/google-cloud-ndb/CHANGELOG.md +++ b/packages/google-cloud-ndb/CHANGELOG.md @@ -704,3 +704,5 @@ ## 0.0.1dev1 Initial development release of NDB client library. + + diff --git a/packages/google-cloud-os-config/CHANGELOG.md b/packages/google-cloud-os-config/CHANGELOG.md index cddf3e057d3b..5672269bfb90 100644 --- a/packages/google-cloud-os-config/CHANGELOG.md +++ b/packages/google-cloud-os-config/CHANGELOG.md @@ -515,3 +515,5 @@ ### Features * generate v1 ([5d1f582](https://www.github.com/googleapis/python-os-config/commit/5d1f582b5b02d128ef44120d285941805d234ec7)) + + diff --git a/packages/google-cloud-pubsub/CHANGELOG.md b/packages/google-cloud-pubsub/CHANGELOG.md index 2ccdfaa246f2..6d6e5c6e393c 100644 --- a/packages/google-cloud-pubsub/CHANGELOG.md +++ b/packages/google-cloud-pubsub/CHANGELOG.md @@ -1732,3 +1732,5 @@ PyPI: https://pypi.org/project/google-cloud-pubsub/0.29.1/ `googleapis-common-protos` dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-pubsub/0.29.0/ + + diff --git a/packages/google-cloud-scheduler/CHANGELOG.md b/packages/google-cloud-scheduler/CHANGELOG.md index 4b15ea69cdce..baef4bb68490 100644 --- a/packages/google-cloud-scheduler/CHANGELOG.md +++ b/packages/google-cloud-scheduler/CHANGELOG.md @@ -559,3 +559,5 @@ ### New Features - Initial release of Cloud Scheduler library. ([#6482](https://github.com/googleapis/google-cloud-python/pull/6482)) + + diff --git a/packages/google-cloud-spanner-dbapi-driver/setup.py b/packages/google-cloud-spanner-dbapi-driver/setup.py index cfc0b06a229b..32f75eda757b 100644 --- a/packages/google-cloud-spanner-dbapi-driver/setup.py +++ b/packages/google-cloud-spanner-dbapi-driver/setup.py @@ -81,3 +81,5 @@ include_package_data=True, zip_safe=False, ) + +# trigger system tests diff --git a/packages/google-cloud-spanner/CHANGELOG.md b/packages/google-cloud-spanner/CHANGELOG.md index 5cb6ae365011..1d6aafc810a0 100644 --- a/packages/google-cloud-spanner/CHANGELOG.md +++ b/packages/google-cloud-spanner/CHANGELOG.md @@ -1705,3 +1705,5 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/ `googleapis-common-protos` dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-spanner/0.29.0/ + + diff --git a/packages/google-cloud-speech/CHANGELOG.md b/packages/google-cloud-speech/CHANGELOG.md index 82b21092fa3f..807f965f663a 100644 --- a/packages/google-cloud-speech/CHANGELOG.md +++ b/packages/google-cloud-speech/CHANGELOG.md @@ -935,3 +935,5 @@ This is the (hopefully) final release candidate before 1.0. `googleapis-common-protos`dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-speech/0.30.0/ + + diff --git a/packages/google-cloud-storage/CHANGELOG.md b/packages/google-cloud-storage/CHANGELOG.md index 7561b729589a..f4fb7788dd4c 100644 --- a/packages/google-cloud-storage/CHANGELOG.md +++ b/packages/google-cloud-storage/CHANGELOG.md @@ -1495,3 +1495,5 @@ Please consult the README for details on this major version release. - Requiring `google-resumable-media >= 0.3.1` (#4244) PyPI: https://pypi.org/project/google-cloud-storage/1.6.0/ + + diff --git a/packages/google-cloud-tasks/CHANGELOG.md b/packages/google-cloud-tasks/CHANGELOG.md index c1a4d276449b..1d4985dafe0e 100644 --- a/packages/google-cloud-tasks/CHANGELOG.md +++ b/packages/google-cloud-tasks/CHANGELOG.md @@ -734,3 +734,5 @@ ### New Features - Add v2beta2 endpoint for Tasks + + diff --git a/packages/google-cloud-testutils/CHANGELOG.md b/packages/google-cloud-testutils/CHANGELOG.md index dd7e48bbea63..e0e92d23b7bb 100644 --- a/packages/google-cloud-testutils/CHANGELOG.md +++ b/packages/google-cloud-testutils/CHANGELOG.md @@ -165,3 +165,5 @@ ### Features * add lower bound checker ([#8](https://www.github.com/googleapis/python-test-utils/issues/8)) ([5ebac9f](https://www.github.com/googleapis/python-test-utils/commit/5ebac9fb0ad005f8ea947c14dfca6de3c0d2cac9)) + + diff --git a/packages/google-cloud-texttospeech/CHANGELOG.md b/packages/google-cloud-texttospeech/CHANGELOG.md index d2b2fbb06fc8..d506cd6dc001 100644 --- a/packages/google-cloud-texttospeech/CHANGELOG.md +++ b/packages/google-cloud-texttospeech/CHANGELOG.md @@ -754,3 +754,5 @@ ### Interface additions - Added text-to-speech v1beta1. (#5049) + + diff --git a/packages/google-cloud-translate/CHANGELOG.md b/packages/google-cloud-translate/CHANGELOG.md index bf6c38e56d36..5ff37ea41b59 100644 --- a/packages/google-cloud-translate/CHANGELOG.md +++ b/packages/google-cloud-translate/CHANGELOG.md @@ -760,3 +760,5 @@ on `google-api-core` (#4221, #4280) PyPI: https://pypi.org/project/google-cloud-translate/1.3.0/ + + diff --git a/packages/google-cloud-videointelligence/CHANGELOG.md b/packages/google-cloud-videointelligence/CHANGELOG.md index b61657a11015..e138f866d4f2 100644 --- a/packages/google-cloud-videointelligence/CHANGELOG.md +++ b/packages/google-cloud-videointelligence/CHANGELOG.md @@ -717,3 +717,5 @@ change as the `v1` and `v1beta2` endpoints are identical. If you pinned to to `4 - Beta` (eb43849569556c6e47f11b8310864c5a280507f2) PyPI: https://pypi.org/project/google-cloud-videointelligence/0.28.0/ + + diff --git a/packages/google-cloud-vision/CHANGELOG.md b/packages/google-cloud-vision/CHANGELOG.md index cb4234a502b9..f433cfd43956 100644 --- a/packages/google-cloud-vision/CHANGELOG.md +++ b/packages/google-cloud-vision/CHANGELOG.md @@ -831,3 +831,5 @@ PyPI: https://pypi.org/project/google-cloud-vision/0.29.0/ `googleapis-common-protos`dependencies (#4096, #4098) PyPI: https://pypi.org/project/google-cloud-vision/0.28.0/ + + diff --git a/packages/google-resumable-media/CHANGELOG.md b/packages/google-resumable-media/CHANGELOG.md index 4e8ee5a9b596..262fb057be6f 100644 --- a/packages/google-resumable-media/CHANGELOG.md +++ b/packages/google-resumable-media/CHANGELOG.md @@ -568,3 +568,5 @@ might break the hypothetical usecase of downloading a blob marked with 2017-04-21 - Initial public release. + + diff --git a/packages/pandas-gbq/CHANGELOG.md b/packages/pandas-gbq/CHANGELOG.md index 718e7fd41cf5..d3a2288beb86 100644 --- a/packages/pandas-gbq/CHANGELOG.md +++ b/packages/pandas-gbq/CHANGELOG.md @@ -916,3 +916,5 @@ Includes patches since the 0.19.2 release on pandas with the following: [pandas-GH#14064](https://github.com/pandas-dev/pandas/pull/14064), and [pandas-GH#14305](https://github.com/pandas-dev/pandas/pull/14305) + + diff --git a/packages/sqlalchemy-bigquery/CHANGELOG.md b/packages/sqlalchemy-bigquery/CHANGELOG.md index 77a42c6139cf..4007bafcaef3 100644 --- a/packages/sqlalchemy-bigquery/CHANGELOG.md +++ b/packages/sqlalchemy-bigquery/CHANGELOG.md @@ -459,3 +459,5 @@ Older versions of this project were distributed as [pybigquery][0]. - Prefer explicitly provided dataset over default dataset in lookup. ([#53](https://github.com/mxmzdlv/pybigquery/pull/53)) - Use the provided `project_id` when using a service account. ([#52](https://github.com/mxmzdlv/pybigquery/pull/52)) + + diff --git a/packages/sqlalchemy-spanner/CHANGELOG.md b/packages/sqlalchemy-spanner/CHANGELOG.md index ec78fa081651..ad5891eff305 100644 --- a/packages/sqlalchemy-spanner/CHANGELOG.md +++ b/packages/sqlalchemy-spanner/CHANGELOG.md @@ -349,3 +349,5 @@ ### Miscellaneous Chores * setup release 1.0.0 ([#165](https://www.github.com/googleapis/python-spanner-sqlalchemy/issues/165)) ([37a415d](https://www.github.com/googleapis/python-spanner-sqlalchemy/commit/37a415d071d39e99f233a1c15c1c4b89bd436570)) + + From b1c3b825da771c9e234c84898c03c00a1ce90201 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 25 Aug 2026 21:14:32 -0700 Subject: [PATCH 04/32] add core_deps back as pre_submit --- .kokoro/presubmit/head.cfg | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .kokoro/presubmit/head.cfg diff --git a/.kokoro/presubmit/head.cfg b/.kokoro/presubmit/head.cfg new file mode 100644 index 000000000000..3f2987023513 --- /dev/null +++ b/.kokoro/presubmit/head.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "core_deps_from_source" +} From f785f59a476ba66dd3e149d7ef4569a098b6f195 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 13:51:52 -0700 Subject: [PATCH 05/32] Increase PACKAGE_TEST_TIMEOUT to 60 minutes --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index bbfef0e7c9da..48982e1cf601 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -307,7 +307,7 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} -PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-35m}" +PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-60m}" # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) From f24baf224015afdefa462f877fef88f95e5a06b1 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 16:04:54 -0700 Subject: [PATCH 06/32] renamed core_deps test --- .kokoro/presubmit/{head.cfg => core_deps.cfg} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .kokoro/presubmit/{head.cfg => core_deps.cfg} (100%) diff --git a/.kokoro/presubmit/head.cfg b/.kokoro/presubmit/core_deps.cfg similarity index 100% rename from .kokoro/presubmit/head.cfg rename to .kokoro/presubmit/core_deps.cfg From 30ae19519adb05ba052fb7c15c5902fedfdcba98 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 16:05:06 -0700 Subject: [PATCH 07/32] set timeout to 35 --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 48982e1cf601..0cc4cf733838 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -307,7 +307,7 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} -PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-60m}" +PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-40m}" # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) From 13fd7a2a98d96f3ff884f9d68d06f689004e95c5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 16:15:36 -0700 Subject: [PATCH 08/32] ignore case in assertion --- .../google-cloud-bigquery/tests/unit/test_client.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/google-cloud-bigquery/tests/unit/test_client.py b/packages/google-cloud-bigquery/tests/unit/test_client.py index d9c7fbbfbe5a..123d18359da1 100644 --- a/packages/google-cloud-bigquery/tests/unit/test_client.py +++ b/packages/google-cloud-bigquery/tests/unit/test_client.py @@ -1798,11 +1798,13 @@ def test_get_table_sets_user_agent(self): http.request.assert_called_once_with( url=mock.ANY, method="GET", - headers={ - "X-Goog-API-Client": expected_user_agent, - "Accept-Encoding": "gzip", - "User-Agent": expected_user_agent, - }, + headers=requests.structures.CaseInsensitiveDict( + { + "X-Goog-API-Client": expected_user_agent, + "Accept-Encoding": "gzip", + "User-Agent": expected_user_agent, + } + ), data=mock.ANY, timeout=DEFAULT_TIMEOUT, ) From c82cad18e725623444dca93c1842e4bce61ecd85 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 16:33:00 -0700 Subject: [PATCH 09/32] core_deps triggers upstream pacakages for dep changes --- .kokoro/system.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 0cc4cf733838..3848ba120f22 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -93,7 +93,7 @@ run_package_test() { PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_GFILE_DIR}/service-account.json" NOX_FILE="noxfile.py" - NOX_SESSION="system" + NOX_SESSION="${NOX_SESSION:-system}" ;; *) PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") @@ -283,6 +283,20 @@ for path in `find 'packages' \ files_to_check=("${package_path}") fi + # When testing core dependencies from source, also trigger on changes to core packages + if [[ "${NOX_SESSION}" == "core_deps_from_source" ]]; then + files_to_check+=( + "packages/google-api-core" + "packages/google-auth" + "packages/google-auth-httplib2" + "packages/google-auth-oauthlib" + "packages/google-cloud-core" + "packages/googleapis-common-protos" + "packages/grpc-google-iam-v1" + "packages/proto-plus" + ) + fi + set +e # Passing the array expanded as arguments to git diff. package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check[@]}" | wc -l) From 272c8f5f67666ed15157d742dc1e8333b684b535 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 16:37:22 -0700 Subject: [PATCH 10/32] assert updated header --- .../google-cloud-bigquery/tests/unit/test_client.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/google-cloud-bigquery/tests/unit/test_client.py b/packages/google-cloud-bigquery/tests/unit/test_client.py index 123d18359da1..f9e8a38b716e 100644 --- a/packages/google-cloud-bigquery/tests/unit/test_client.py +++ b/packages/google-cloud-bigquery/tests/unit/test_client.py @@ -1798,13 +1798,11 @@ def test_get_table_sets_user_agent(self): http.request.assert_called_once_with( url=mock.ANY, method="GET", - headers=requests.structures.CaseInsensitiveDict( - { - "X-Goog-API-Client": expected_user_agent, - "Accept-Encoding": "gzip", - "User-Agent": expected_user_agent, - } - ), + headers={ + "x-goog-api-client": expected_user_agent, + "Accept-Encoding": "gzip", + "User-Agent": expected_user_agent, + }, data=mock.ANY, timeout=DEFAULT_TIMEOUT, ) From fdcbd49996c634835e7f84536a0f5e2ba9484b78 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:13:31 -0700 Subject: [PATCH 11/32] revive core_deps GA test --- .github/workflows/unittest.yml | 54 ++++++++++++++++++++++++++++++++-- ci/get_package_shards.py | 47 ++++++++++++++++++++++++++--- ci/run_single_test.sh | 12 +++++--- 3 files changed, 102 insertions(+), 11 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 6c2206518bc3..bc61c9fc864c 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -156,12 +156,55 @@ jobs: path: .coverage.${{ matrix.python }}.* include-hidden-files: true + core-deps: + needs: initialize + if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != '' + runs-on: ubuntu-22.04 + strategy: + fail-fast: true + matrix: + python: ["3.14"] + package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }} + name: ${{ matrix.package_shard.is_sharded && format('core-deps ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps ({0})', matrix.python) }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` + # See https://github.com/googleapis/google-cloud-python/issues/12013 + # and https://github.com/actions/checkout#checkout-head. + with: + fetch-depth: 2 + persist-credentials: false + - name: Setup Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ matrix.python }} + cache: 'pip' + allow-prereleases: true + - name: Setup uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + enable-cache: true + cache-dependency-glob: 'packages/**/testing/constraints*.txt' + - name: Install nox and nox-uv + run: | + uv pip install --system nox nox-uv + - name: Run core_deps_from_source for ${{ matrix.package_shard.description }} + env: + BUILD_TYPE: presubmit + TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} + TEST_TYPE: core_deps_from_source + PY_VERSION: ${{ matrix.python }} + PACKAGE_LIST: ${{ matrix.package_shard.packages }} + run: | + ci/run_conditional_tests.sh + all-tests: - needs: [initialize, unit] + needs: [initialize, unit, core-deps] if: always() runs-on: ubuntu-latest steps: - - name: Check unit test results + - name: Check test results run: | # 1. Check initialize job if [[ "${{ needs.initialize.result }}" != "success" ]]; then @@ -173,7 +216,12 @@ jobs: echo "Unit tests failed" exit 1 fi - echo "All unit tests passed or were skipped" + # 3. Check core dependencies test shards + if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then + echo "Core dependencies tests failed" + exit 1 + fi + echo "All unit and core dependencies tests passed or were skipped" cover: if: always() && !cancelled() && needs.all-tests.result == 'success' diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index 015adfc2bacd..4ade18d64309 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -22,12 +22,12 @@ preview-packages/foo) are kept aligned in the exact same shard. """ -import os -import subprocess +import collections import json import math +import os +import subprocess import sys -import collections def get_package_directories(): @@ -88,6 +88,29 @@ def get_packages(): return packages_map +def get_handwritten_packages(): + """Returns a dictionary of packages that are handwritten / non-GAPIC_AUTO. + + Parses each package's .repo-metadata.json to identify non-GAPIC_AUTO libraries. + """ + all_packages = get_packages() + handwritten = collections.defaultdict(list) + for pkg_name, paths in all_packages.items(): + for path in paths: + metadata_file = os.path.join(path, ".repo-metadata.json") + if os.path.exists(metadata_file): + try: + with open(metadata_file) as f: + data = json.load(f) + if data.get("library_type") != "GAPIC_AUTO" or pkg_name.startswith("google-cloud-compute"): + handwritten[pkg_name] = paths + break + except Exception: + handwritten[pkg_name] = paths + break + return dict(handwritten) + + def get_packages_to_test(): """Determines the dictionary of package names to directory paths that need to be tested. @@ -132,6 +155,22 @@ def get_packages_to_test(): if full_path not in to_test_paths[pkg_name]: to_test_paths[pkg_name].append(full_path) + # Core dependency packages whose changes require testing across downstream handwritten packages + core_packages = { + "google-api-core", + "google-auth", + "google-auth-httplib2", + "google-auth-oauthlib", + "google-cloud-core", + "googleapis-common-protos", + "grpc-google-iam-v1", + "proto-plus", + "google-crc32c", + } + if any(pkg in core_packages for pkg in to_test_paths): + # When a core package changes, test all handwritten packages (non-GAPIC_AUTO) + return get_handwritten_packages() + return dict(to_test_paths) @@ -173,7 +212,7 @@ def group_packages(packages_map): # Pack packages alphabetically by package name. for name, paths, weight in pkg_items: - # If adding this package would exceed target weight AND we haven't reached the + # If adding this package would exceed target weight AND we haven't reached the # shard limit, start a new shard. Otherwise, keep "stuffing" the current one. if current_shard_items and (current_shard_weight + weight > target_weight) and len(shards_list) < max_shards - 1: shards_list.append(current_shard_items) diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 0ea0b969770d..46bf2f4df335 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -19,7 +19,7 @@ # `PY_VERSION` should be one of ["3.10", "3.11", "3.12", "3.13"] # This script is called by the `ci/run_conditional_tests.sh` script. -# A specific `nox` session will be run, depending on the value of +# A specific `nox` session will be run, depending on the value of # `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is # `lint`, the `nox -s lint` session will be run. @@ -68,6 +68,10 @@ case ${TEST_TYPE} in nox -s prerelease_deps-3.14 retval=$? ;; + core_deps_from_source) + nox --stop-on-first-error -s core_deps_from_source + retval=$? + ;; unit) case ${PY_VERSION} in "3.10") @@ -131,12 +135,12 @@ case ${TEST_TYPE} in source .venv-profiler/bin/activate export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 python -m pip install --upgrade pip setuptools - + PROFILER_TEMP_DIR=$(mktemp -d) cp ../../scripts/import_profiler/profiler.py "${PROFILER_TEMP_DIR}/profiler.py" PROFILER_SCRIPT="${PROFILER_TEMP_DIR}/profiler.py" BASELINE_CSV="${PROFILER_TEMP_DIR}/baseline_${PACKAGE_NAME}.csv" - + if [ -n "${TARGET_BRANCH}" ]; then # Fetch history for the target branch without --depth=1 in case it was shallowly fetched if [ -f "$(git rev-parse --git-dir)/shallow" ]; then @@ -180,7 +184,7 @@ case ${TEST_TYPE} in echo "Could not find baseline commit for ${TARGET_BRANCH:-main}. Skipping baseline generation." fi fi - + # TODO(https://github.com/googleapis/google-cloud-python/issues/18035): # Clean up this fallback once Python 3.15 is officially released and upstream binary wheels are available on PyPI. # On pre-release Python versions, packages with complex C/Rust dependencies (e.g. bigframes) fail during pip install due to missing pre-built wheels. From 4985d1b754997ce0c6bba8820220b713af3bd67a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:16:15 -0700 Subject: [PATCH 12/32] support either case --- .../tests/unit/test_client.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/google-cloud-bigquery/tests/unit/test_client.py b/packages/google-cloud-bigquery/tests/unit/test_client.py index f9e8a38b716e..5cce574d3ff1 100644 --- a/packages/google-cloud-bigquery/tests/unit/test_client.py +++ b/packages/google-cloud-bigquery/tests/unit/test_client.py @@ -1798,14 +1798,20 @@ def test_get_table_sets_user_agent(self): http.request.assert_called_once_with( url=mock.ANY, method="GET", - headers={ - "x-goog-api-client": expected_user_agent, - "Accept-Encoding": "gzip", - "User-Agent": expected_user_agent, - }, + headers=mock.ANY, data=mock.ANY, timeout=DEFAULT_TIMEOUT, ) + _, kwargs = http.request.call_args + actual_headers = {k.lower(): v for k, v in kwargs["headers"].items()} + self.assertEqual( + actual_headers, + { + "x-goog-api-client": expected_user_agent, + "accept-encoding": "gzip", + "user-agent": expected_user_agent, + }, + ) self.assertIn("my-application/1.2.3", expected_user_agent) def test_get_iam_policy(self): From 108f609b2ab0a65f557f124485475712b5cd6cdd Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:20:44 -0700 Subject: [PATCH 13/32] change backend --- .github/workflows/unittest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index bc61c9fc864c..5d6f48b81c1f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -196,6 +196,7 @@ jobs: TEST_TYPE: core_deps_from_source PY_VERSION: ${{ matrix.python }} PACKAGE_LIST: ${{ matrix.package_shard.packages }} + NOX_DEFAULT_VENV_BACKEND: "virtualenv" run: | ci/run_conditional_tests.sh From 6ce8e8165821ebf563ecbb6411826648031b2ea8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:25:30 -0700 Subject: [PATCH 14/32] force the python version --- .github/workflows/unittest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 5d6f48b81c1f..ee91e14b1407 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -197,6 +197,7 @@ jobs: PY_VERSION: ${{ matrix.python }} PACKAGE_LIST: ${{ matrix.package_shard.packages }} NOX_DEFAULT_VENV_BACKEND: "virtualenv" + NOXFORCEPYTHON: ${{ matrix.python }} run: | ci/run_conditional_tests.sh From 8a981bc5e6c22d363689ab0d239bc851264cddda Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:35:46 -0700 Subject: [PATCH 15/32] boost timeout --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 3848ba120f22..79b04401b677 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -321,7 +321,7 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} -PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-40m}" +PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-90m}" # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) From 9a643520ba171b76a9f812b0ef150843f4bf99f4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:46:55 -0700 Subject: [PATCH 16/32] improved handwritten lib detection --- ci/get_package_shards.py | 44 ++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index 4ade18d64309..fd26332648a3 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -70,9 +70,11 @@ def get_package_weights(): return weights -def get_packages(): +def get_packages(handwritten_only=False): """Lists all package directory paths in the repository grouped by package name. + If handwritten_only is True, includes only non-GAPIC_AUTO libraries. + Returns: dict: A dictionary mapping package_name -> list of relative directory paths. """ @@ -83,34 +85,22 @@ def get_packages(): continue for d in os.listdir(subdir): full_path = os.path.join(subdir, d) + '/' - if os.path.isdir(full_path): - packages_map[d].append(full_path) + if not os.path.isdir(full_path): + continue + if handwritten_only: + meta_file = os.path.join(full_path, ".repo-metadata.json") + if os.path.exists(meta_file): + try: + with open(meta_file) as f: + data = json.load(f) + if data.get("library_type") == "GAPIC_AUTO": + continue + except Exception: + pass + packages_map[d].append(full_path) return packages_map -def get_handwritten_packages(): - """Returns a dictionary of packages that are handwritten / non-GAPIC_AUTO. - - Parses each package's .repo-metadata.json to identify non-GAPIC_AUTO libraries. - """ - all_packages = get_packages() - handwritten = collections.defaultdict(list) - for pkg_name, paths in all_packages.items(): - for path in paths: - metadata_file = os.path.join(path, ".repo-metadata.json") - if os.path.exists(metadata_file): - try: - with open(metadata_file) as f: - data = json.load(f) - if data.get("library_type") != "GAPIC_AUTO" or pkg_name.startswith("google-cloud-compute"): - handwritten[pkg_name] = paths - break - except Exception: - handwritten[pkg_name] = paths - break - return dict(handwritten) - - def get_packages_to_test(): """Determines the dictionary of package names to directory paths that need to be tested. @@ -169,7 +159,7 @@ def get_packages_to_test(): } if any(pkg in core_packages for pkg in to_test_paths): # When a core package changes, test all handwritten packages (non-GAPIC_AUTO) - return get_handwritten_packages() + return get_packages(handwritten_only=True) return dict(to_test_paths) From 431e71bed99b6c7de90b639340455b5a795ded04 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 17:48:01 -0700 Subject: [PATCH 17/32] skip preview package for now; update noxfiles --- ci/run_single_test.sh | 4 ++++ .../gapic/templates/noxfile.py.j2 | 19 +++++++++++-------- .../integration/goldens/asset/noxfile.py | 12 +++++++----- .../goldens/credentials/noxfile.py | 12 +++++++----- .../integration/goldens/eventarc/noxfile.py | 12 +++++++----- .../integration/goldens/logging/noxfile.py | 12 +++++++----- .../goldens/logging_internal/noxfile.py | 12 +++++++----- .../integration/goldens/redis/noxfile.py | 12 +++++++----- .../goldens/redis_selective/noxfile.py | 12 +++++++----- .../goldens/storagebatchoperations/noxfile.py | 12 +++++++----- 10 files changed, 71 insertions(+), 48 deletions(-) diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 46bf2f4df335..8f888923e3b2 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -69,6 +69,10 @@ case ${TEST_TYPE} in retval=$? ;; core_deps_from_source) + if [[ "$(pwd)" == */preview-packages/* ]]; then + echo "Skipping core_deps_from_source for preview package $(pwd)" + exit 0 + fi nox --stop-on-first-error -s core_deps_from_source retval=$? ;; diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index d4176bdf4222..e5c1982017ba 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -163,7 +163,7 @@ def lint(session): "ruff", "format", "--check", f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", + "--line-length=88", *LINT_PATHS, ) @@ -179,7 +179,7 @@ def lint(session): def blacken(session): """(Deprecated) Legacy session. Please use 'nox -s format'.""" session.log("WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future.") - + # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports) session.install(RUFF_VERSION) session.run( @@ -518,14 +518,14 @@ def prerelease_deps(session, protobuf_implementation): # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") parsed_deps = { - dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) for dep in prerel_deps } # Dynamically sort local packages vs PyPI dependencies local_paths = [] pypi_deps = [] - + for dep, pkg_name in parsed_deps.items(): if (deps_dir / pkg_name).exists(): local_paths.append(str(deps_dir / pkg_name)) @@ -624,13 +624,16 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] - + session.install(*dep_paths, "--no-deps", "--ignore-installed") print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") diff --git a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py index 0098c6989836..e876bb4f2cfd 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py index 0744db4ab69b..0e277e5dfd2b 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py index f163aac8e963..738850b16b99 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py index 1510142f6d7d..df1eb716d12c 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py index 1510142f6d7d..df1eb716d12c 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py index 8b1386af4435..555a4745454b 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py index 8b1386af4435..555a4745454b 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py index 75fded704a26..1ac80d2800c0 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py @@ -17,9 +17,8 @@ import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox @@ -616,9 +615,12 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent + # Locate the monorepo 'packages' directory containing core dependencies + deps_dir = next( + p / "packages" + for p in CURRENT_DIRECTORY.parents + if (p / "packages").is_dir() + ) # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] From 225e5d12fa05078badb2d1df541b7f42997fb12c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 18:47:01 -0700 Subject: [PATCH 18/32] added weights for handwritten --- .github/workflows/unittest.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index ee91e14b1407..2cd269a4ec42 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -42,13 +42,24 @@ jobs: PACKAGE_WEIGHTS: | google-ai-generativelanguage: 4 google-auth: 5 + google-cloud-bigquery: 6 + google-cloud-bigtable: 4 google-cloud-compute: 12 google-cloud-compute-v1beta: 12 google-cloud-dialogflow: 6 google-cloud-dialogflow-cx: 6 google-cloud-discoveryengine: 8 + google-cloud-firestore: 6 + google-cloud-logging: 4 + google-cloud-monitoring: 4 + google-cloud-pubsub: 4 google-cloud-retail: 5 + google-cloud-spanner: 6 + google-cloud-storage: 6 google-shopping-merchant-accounts: 4 + pandas-gbq: 3 + sqlalchemy-bigquery: 3 + sqlalchemy-spanner: 3 steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 From 194d71e7f037df200c37fd6566e811a997c892b6 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 18:47:16 -0700 Subject: [PATCH 19/32] updated package name --- .github/workflows/unittest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 2cd269a4ec42..75706398ec19 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -176,7 +176,7 @@ jobs: matrix: python: ["3.14"] package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }} - name: ${{ matrix.package_shard.is_sharded && format('core-deps ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps ({0})', matrix.python) }} + name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }} steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 From 1740b75c0495f87990fe028c661dfadc7bda2aff Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 18:47:46 -0700 Subject: [PATCH 20/32] fixed import order --- packages/gapic-generator/gapic/templates/noxfile.py.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index e5c1982017ba..47205dd39819 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -6,9 +6,8 @@ import os import pathlib import re import shutil - -from typing import Dict, List import warnings +from typing import Dict, List import nox From 8f8d00005a5eb3b3f2a574da909b0f5aa88b44cb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 18:55:48 -0700 Subject: [PATCH 21/32] addressed gemini comments --- .kokoro/system.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 79b04401b677..366494358672 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -130,6 +130,9 @@ reap_parallel_results() { local failed_count=0 local timed_out_count=0 local succeeded_count=0 + local failed + local timed_out + local pkg if [ -z "$LOG_DIR" ]; then echo "Error: LOG_DIR is not set." @@ -179,7 +182,7 @@ reap_parallel_results() { # List failed packages for failed in "$LOG_DIR"/*.failed; do if [ -f "$failed" ]; then - local pkg=$(basename "$failed" .failed) + pkg=$(basename "$failed" .failed) if [ -f "$LOG_DIR/$pkg.timed_out" ]; then echo "$pkg (TIMED OUT after ${PACKAGE_TEST_TIMEOUT})" else @@ -189,7 +192,7 @@ reap_parallel_results() { done for failed in "$LOG_DIR"/*.failed; do if [ -f "$failed" ]; then - local pkg=$(basename "$failed" .failed) + pkg=$(basename "$failed" .failed) echo "--------------------------------------------------" if [ -f "$LOG_DIR/$pkg.timed_out" ]; then echo "@PACKAGE (TIMED OUT after ${PACKAGE_TEST_TIMEOUT}): $pkg" From a74c454a7a7f42888175fc68eb0bd057d2a6d261 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 20:11:34 -0700 Subject: [PATCH 22/32] removed uv step --- .github/workflows/unittest.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 75706398ec19..1f033b502ea4 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -192,14 +192,9 @@ jobs: python-version: ${{ matrix.python }} cache: 'pip' allow-prereleases: true - - name: Setup uv - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 - with: - enable-cache: true - cache-dependency-glob: 'packages/**/testing/constraints*.txt' - - name: Install nox and nox-uv + - name: Install nox run: | - uv pip install --system nox nox-uv + pip install nox - name: Run core_deps_from_source for ${{ matrix.package_shard.description }} env: BUILD_TYPE: presubmit From 76da0b725121b17b016d350d91c29e0e868f8661 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 20:11:46 -0700 Subject: [PATCH 23/32] updated weights --- .github/workflows/unittest.yml | 41 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 1f033b502ea4..68ccc4969d7f 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -40,26 +40,33 @@ jobs: # Each weight is roughly 1 minute of expected execution time # Default for unset packages is 1 PACKAGE_WEIGHTS: | - google-ai-generativelanguage: 4 - google-auth: 5 - google-cloud-bigquery: 6 + django-google-spanner: 2 + gapic-generator: 3 + google-ai-generativelanguage: 3 + google-api-core: 2 + google-auth: 2 + google-cloud-bigquery: 4 + google-cloud-bigquery-storage: 2 google-cloud-bigtable: 4 google-cloud-compute: 12 google-cloud-compute-v1beta: 12 - google-cloud-dialogflow: 6 - google-cloud-dialogflow-cx: 6 - google-cloud-discoveryengine: 8 - google-cloud-firestore: 6 - google-cloud-logging: 4 - google-cloud-monitoring: 4 - google-cloud-pubsub: 4 - google-cloud-retail: 5 - google-cloud-spanner: 6 - google-cloud-storage: 6 - google-shopping-merchant-accounts: 4 - pandas-gbq: 3 - sqlalchemy-bigquery: 3 - sqlalchemy-spanner: 3 + google-cloud-datastore: 3 + google-cloud-dialogflow: 4 + google-cloud-dialogflow-cx: 4 + google-cloud-discoveryengine: 4 + google-cloud-firestore: 3 + google-cloud-logging: 3 + google-cloud-monitoring: 3 + google-cloud-ndb: 2 + google-cloud-pubsub: 2 + google-cloud-retail: 3 + google-cloud-spanner: 4 + google-cloud-storage: 4 + google-shopping-merchant-accounts: 3 + pandas-gbq: 2 + proto-plus: 1 + sqlalchemy-bigquery: 2 + sqlalchemy-spanner: 2 steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 From fb96ae625ef86495acd90c07f338acd877258e2e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 20:13:39 -0700 Subject: [PATCH 24/32] updated noxfiles to use local packages instead of over network --- packages/db-dtypes/noxfile.py | 21 ++++++++++++------- .../noxfile.py | 21 ++++++++++++------- packages/google-cloud-ndb/noxfile.py | 21 ++++++++++++------- packages/googleapis-common-protos/noxfile.py | 21 ++++++++++++------- packages/pandas-gbq/noxfile.py | 21 ++++++++++++------- packages/proto-plus/noxfile.py | 16 ++++++++------ packages/sqlalchemy-bigquery/noxfile.py | 21 ++++++++++++------- packages/sqlalchemy-spanner/noxfile.py | 21 ++++++++++++------- 8 files changed, 101 insertions(+), 62 deletions(-) diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index b26f72f7a285..5f99ca23a5c2 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -512,16 +512,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 18a537cb9bda..1502ae07a03b 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -536,16 +536,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index cb957a674ed1..7a90848f63c2 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -504,16 +504,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index 78fd0cbf4ff9..79587afeca21 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -558,16 +558,21 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index 0021226416cd..ed751fe7e104 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -584,16 +584,21 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 5f56955863c8..63cfe15796a6 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -167,14 +167,18 @@ def core_deps_from_source(session, implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - # dependency of google-api-core - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core", + "googleapis-common-protos", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") # TODO(https://github.com/googleapis/google-cloud-python/issues/15115): Install protobuf from source at HEAD session.install("--pre", "--upgrade", "protobuf") diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 506a5bfcfb8f..5e7d20291107 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -706,16 +706,21 @@ def core_deps_from_source(session, protobuf_implementation): session.install("-e", install_target, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index df709c96e50a..84a348c76e61 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -547,16 +547,21 @@ def core_deps_from_source(session, protobuf_implementation): session.install(".") core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") tests_path = os.path.join("tests", "unit") session.run( From b1a2694227ff853b92fa90f6bfdc6a2862f58832 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 20:25:33 -0700 Subject: [PATCH 25/32] increase timeout --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 366494358672..3a7d5b059f31 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -324,7 +324,7 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} -PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-90m}" +PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-120m}" # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX) From 6cf9b879a24d46fe7fa891aaa055e01fd968fdd6 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 20:55:10 -0700 Subject: [PATCH 26/32] fixed lint --- packages/db-dtypes/noxfile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index 5f99ca23a5c2..d7fc9b977ee3 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -523,10 +523,16 @@ def core_deps_from_source(session, protobuf_implementation): while deps_dir.name != "packages" and deps_dir.parent != deps_dir: deps_dir = deps_dir.parent - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] + local_paths = [ + str(deps_dir / dep) + for dep in core_dependencies_from_source + if (deps_dir / dep).exists() + ] if local_paths: session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + print( + f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" + ) tests_path = os.path.join("tests", "unit") session.run( From 90ce674a8b1a0e4453ac253d6a3ce7bd95e3870a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 20:59:28 -0700 Subject: [PATCH 27/32] move sqlalchemy compliance tests into new runner --- .kokoro/presubmit/sqlalchemy_compliance.cfg | 10 ++++++++++ .kokoro/presubmit/system.cfg | 6 ++++++ .kokoro/system.sh | 6 ++++++ 3 files changed, 22 insertions(+) create mode 100644 .kokoro/presubmit/sqlalchemy_compliance.cfg diff --git a/.kokoro/presubmit/sqlalchemy_compliance.cfg b/.kokoro/presubmit/sqlalchemy_compliance.cfg new file mode 100644 index 000000000000..1b48a0f49197 --- /dev/null +++ b/.kokoro/presubmit/sqlalchemy_compliance.cfg @@ -0,0 +1,10 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Increase timeout to 3 hours +timeout_mins: 180 + +# Only run compliance nox session. +env_vars: { + key: "NOX_SESSION" + value: "compliance" +} diff --git a/.kokoro/presubmit/system.cfg b/.kokoro/presubmit/system.cfg index d5dc63b5a58f..c6920f22aaeb 100644 --- a/.kokoro/presubmit/system.cfg +++ b/.kokoro/presubmit/system.cfg @@ -8,3 +8,9 @@ env_vars: { key: "NOX_SESSION" value: "system-3.12" } + +# Skip compliance tests in system job since they run in dedicated compliance job +env_vars: { + key: "RUN_COMPLIANCE_TESTS" + value: "false" +} diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 3a7d5b059f31..da08d4a2a98d 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -300,6 +300,12 @@ for path in `find 'packages' \ ) fi + # When running compliance tests, only test packages that have compliance suites + if [[ "${NOX_SESSION}" == "compliance"* && "${package_name}" != "sqlalchemy-"* ]]; then + printf "SKIP %-20s %-40s %s\n" "[not_applicable]" "${package_name}" "${commit_hash:-HEAD}" + continue + fi + set +e # Passing the array expanded as arguments to git diff. package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check[@]}" | wc -l) From 7381511bc096bef677e00cbedddcdb344ac5a94d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 Aug 2026 21:17:10 -0700 Subject: [PATCH 28/32] reverted core_deps changes --- .github/workflows/unittest.yml | 83 +++---------------- ci/get_package_shards.py | 35 +------- ci/run_single_test.sh | 8 -- packages/db-dtypes/noxfile.py | 27 ++---- .../gapic/templates/noxfile.py.j2 | 22 +++-- .../integration/goldens/asset/noxfile.py | 12 ++- .../goldens/credentials/noxfile.py | 12 ++- .../integration/goldens/eventarc/noxfile.py | 12 ++- .../integration/goldens/logging/noxfile.py | 12 ++- .../goldens/logging_internal/noxfile.py | 12 ++- .../integration/goldens/redis/noxfile.py | 12 ++- .../goldens/redis_selective/noxfile.py | 12 ++- .../goldens/storagebatchoperations/noxfile.py | 12 ++- .../tests/unit/test_client.py | 16 ++-- .../noxfile.py | 21 ++--- packages/google-cloud-ndb/noxfile.py | 21 ++--- .../setup.py | 2 - packages/googleapis-common-protos/noxfile.py | 21 ++--- packages/pandas-gbq/noxfile.py | 21 ++--- packages/proto-plus/noxfile.py | 16 ++-- packages/sqlalchemy-bigquery/noxfile.py | 21 ++--- packages/sqlalchemy-spanner/noxfile.py | 21 ++--- 22 files changed, 130 insertions(+), 301 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 68ccc4969d7f..6c2206518bc3 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -40,33 +40,15 @@ jobs: # Each weight is roughly 1 minute of expected execution time # Default for unset packages is 1 PACKAGE_WEIGHTS: | - django-google-spanner: 2 - gapic-generator: 3 - google-ai-generativelanguage: 3 - google-api-core: 2 - google-auth: 2 - google-cloud-bigquery: 4 - google-cloud-bigquery-storage: 2 - google-cloud-bigtable: 4 + google-ai-generativelanguage: 4 + google-auth: 5 google-cloud-compute: 12 google-cloud-compute-v1beta: 12 - google-cloud-datastore: 3 - google-cloud-dialogflow: 4 - google-cloud-dialogflow-cx: 4 - google-cloud-discoveryengine: 4 - google-cloud-firestore: 3 - google-cloud-logging: 3 - google-cloud-monitoring: 3 - google-cloud-ndb: 2 - google-cloud-pubsub: 2 - google-cloud-retail: 3 - google-cloud-spanner: 4 - google-cloud-storage: 4 - google-shopping-merchant-accounts: 3 - pandas-gbq: 2 - proto-plus: 1 - sqlalchemy-bigquery: 2 - sqlalchemy-spanner: 2 + google-cloud-dialogflow: 6 + google-cloud-dialogflow-cx: 6 + google-cloud-discoveryengine: 8 + google-cloud-retail: 5 + google-shopping-merchant-accounts: 4 steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -174,52 +156,12 @@ jobs: path: .coverage.${{ matrix.python }}.* include-hidden-files: true - core-deps: - needs: initialize - if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != '' - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - python: ["3.14"] - package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }} - name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }} - steps: - - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base` - # See https://github.com/googleapis/google-cloud-python/issues/12013 - # and https://github.com/actions/checkout#checkout-head. - with: - fetch-depth: 2 - persist-credentials: false - - name: Setup Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: ${{ matrix.python }} - cache: 'pip' - allow-prereleases: true - - name: Install nox - run: | - pip install nox - - name: Run core_deps_from_source for ${{ matrix.package_shard.description }} - env: - BUILD_TYPE: presubmit - TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }} - TEST_TYPE: core_deps_from_source - PY_VERSION: ${{ matrix.python }} - PACKAGE_LIST: ${{ matrix.package_shard.packages }} - NOX_DEFAULT_VENV_BACKEND: "virtualenv" - NOXFORCEPYTHON: ${{ matrix.python }} - run: | - ci/run_conditional_tests.sh - all-tests: - needs: [initialize, unit, core-deps] + needs: [initialize, unit] if: always() runs-on: ubuntu-latest steps: - - name: Check test results + - name: Check unit test results run: | # 1. Check initialize job if [[ "${{ needs.initialize.result }}" != "success" ]]; then @@ -231,12 +173,7 @@ jobs: echo "Unit tests failed" exit 1 fi - # 3. Check core dependencies test shards - if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then - echo "Core dependencies tests failed" - exit 1 - fi - echo "All unit and core dependencies tests passed or were skipped" + echo "All unit tests passed or were skipped" cover: if: always() && !cancelled() && needs.all-tests.result == 'success' diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index fd26332648a3..c797617889c2 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -70,11 +70,9 @@ def get_package_weights(): return weights -def get_packages(handwritten_only=False): +def get_packages(): """Lists all package directory paths in the repository grouped by package name. - If handwritten_only is True, includes only non-GAPIC_AUTO libraries. - Returns: dict: A dictionary mapping package_name -> list of relative directory paths. """ @@ -85,19 +83,8 @@ def get_packages(handwritten_only=False): continue for d in os.listdir(subdir): full_path = os.path.join(subdir, d) + '/' - if not os.path.isdir(full_path): - continue - if handwritten_only: - meta_file = os.path.join(full_path, ".repo-metadata.json") - if os.path.exists(meta_file): - try: - with open(meta_file) as f: - data = json.load(f) - if data.get("library_type") == "GAPIC_AUTO": - continue - except Exception: - pass - packages_map[d].append(full_path) + if os.path.isdir(full_path): + packages_map[d].append(full_path) return packages_map @@ -145,22 +132,6 @@ def get_packages_to_test(): if full_path not in to_test_paths[pkg_name]: to_test_paths[pkg_name].append(full_path) - # Core dependency packages whose changes require testing across downstream handwritten packages - core_packages = { - "google-api-core", - "google-auth", - "google-auth-httplib2", - "google-auth-oauthlib", - "google-cloud-core", - "googleapis-common-protos", - "grpc-google-iam-v1", - "proto-plus", - "google-crc32c", - } - if any(pkg in core_packages for pkg in to_test_paths): - # When a core package changes, test all handwritten packages (non-GAPIC_AUTO) - return get_packages(handwritten_only=True) - return dict(to_test_paths) diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 8f888923e3b2..2c6ad6b2fb7c 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -68,14 +68,6 @@ case ${TEST_TYPE} in nox -s prerelease_deps-3.14 retval=$? ;; - core_deps_from_source) - if [[ "$(pwd)" == */preview-packages/* ]]; then - echo "Skipping core_deps_from_source for preview package $(pwd)" - exit 0 - fi - nox --stop-on-first-error -s core_deps_from_source - retval=$? - ;; unit) case ${PY_VERSION} in "3.10") diff --git a/packages/db-dtypes/noxfile.py b/packages/db-dtypes/noxfile.py index d7fc9b977ee3..b26f72f7a285 100644 --- a/packages/db-dtypes/noxfile.py +++ b/packages/db-dtypes/noxfile.py @@ -512,27 +512,16 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [ - str(deps_dir / dep) - for dep in core_dependencies_from_source - if (deps_dir / dep).exists() - ] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print( - f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}" - ) + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index 47205dd39819..d4176bdf4222 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -6,8 +6,9 @@ import os import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -162,7 +163,7 @@ def lint(session): "ruff", "format", "--check", f"--target-version=py{ALL_PYTHON[0].replace('.', '')}", - "--line-length=88", + "--line-length=88", *LINT_PATHS, ) @@ -178,7 +179,7 @@ def lint(session): def blacken(session): """(Deprecated) Legacy session. Please use 'nox -s format'.""" session.log("WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future.") - + # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports) session.install(RUFF_VERSION) session.run( @@ -517,14 +518,14 @@ def prerelease_deps(session, protobuf_implementation): # Extract the base package name, safely ignoring version bounds and spaces # (e.g., "grpcio>=1.75.1" becomes "grpcio") parsed_deps = { - dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) for dep in prerel_deps } # Dynamically sort local packages vs PyPI dependencies local_paths = [] pypi_deps = [] - + for dep, pkg_name in parsed_deps.items(): if (deps_dir / pkg_name).exists(): local_paths.append(str(deps_dir / pkg_name)) @@ -623,16 +624,13 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] - + session.install(*dep_paths, "--no-deps", "--ignore-installed") print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") diff --git a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py index e876bb4f2cfd..0098c6989836 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py index 0e277e5dfd2b..0744db4ab69b 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py index 738850b16b99..f163aac8e963 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py index df1eb716d12c..1510142f6d7d 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py index df1eb716d12c..1510142f6d7d 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py index 555a4745454b..8b1386af4435 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py index 555a4745454b..8b1386af4435 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py index 1ac80d2800c0..75fded704a26 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py @@ -17,8 +17,9 @@ import pathlib import re import shutil -import warnings + from typing import Dict, List +import warnings import nox @@ -615,12 +616,9 @@ def core_deps_from_source(session, protobuf_implementation): "proto-plus", ] - # Locate the monorepo 'packages' directory containing core dependencies - deps_dir = next( - p / "packages" - for p in CURRENT_DIRECTORY.parents - if (p / "packages").is_dir() - ) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent # Batch the pip installation to avoid sequential overhead dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] diff --git a/packages/google-cloud-bigquery/tests/unit/test_client.py b/packages/google-cloud-bigquery/tests/unit/test_client.py index 5cce574d3ff1..d9c7fbbfbe5a 100644 --- a/packages/google-cloud-bigquery/tests/unit/test_client.py +++ b/packages/google-cloud-bigquery/tests/unit/test_client.py @@ -1798,20 +1798,14 @@ def test_get_table_sets_user_agent(self): http.request.assert_called_once_with( url=mock.ANY, method="GET", - headers=mock.ANY, + headers={ + "X-Goog-API-Client": expected_user_agent, + "Accept-Encoding": "gzip", + "User-Agent": expected_user_agent, + }, data=mock.ANY, timeout=DEFAULT_TIMEOUT, ) - _, kwargs = http.request.call_args - actual_headers = {k.lower(): v for k, v in kwargs["headers"].items()} - self.assertEqual( - actual_headers, - { - "x-goog-api-client": expected_user_agent, - "accept-encoding": "gzip", - "user-agent": expected_user_agent, - }, - ) self.assertIn("my-application/1.2.3", expected_user_agent) def test_get_iam_policy(self): diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index 1502ae07a03b..18a537cb9bda 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -536,21 +536,16 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index 7a90848f63c2..cb957a674ed1 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -504,21 +504,16 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/google-cloud-spanner-dbapi-driver/setup.py b/packages/google-cloud-spanner-dbapi-driver/setup.py index 32f75eda757b..cfc0b06a229b 100644 --- a/packages/google-cloud-spanner-dbapi-driver/setup.py +++ b/packages/google-cloud-spanner-dbapi-driver/setup.py @@ -81,5 +81,3 @@ include_package_data=True, zip_safe=False, ) - -# trigger system tests diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index 79587afeca21..78fd0cbf4ff9 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -558,21 +558,16 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + f"{CURRENT_DIRECTORY}/../googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") session.run( "py.test", diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index ed751fe7e104..0021226416cd 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -584,21 +584,16 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 63cfe15796a6..5f56955863c8 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -167,18 +167,14 @@ def core_deps_from_source(session, implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "google-api-core", - "googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + # dependency of google-api-core + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") # TODO(https://github.com/googleapis/google-cloud-python/issues/15115): Install protobuf from source at HEAD session.install("--pre", "--upgrade", "protobuf") diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 5e7d20291107..506a5bfcfb8f 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -706,21 +706,16 @@ def core_deps_from_source(session, protobuf_implementation): session.install("-e", install_target, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") tests_path = os.path.join("tests", "unit") session.run( diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index 84a348c76e61..df709c96e50a 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -547,21 +547,16 @@ def core_deps_from_source(session, protobuf_implementation): session.install(".") core_dependencies_from_source = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "proto-plus", + "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", + "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", + "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", ] - deps_dir = CURRENT_DIRECTORY.parent - while deps_dir.name != "packages" and deps_dir.parent != deps_dir: - deps_dir = deps_dir.parent - - local_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source if (deps_dir / dep).exists()] - if local_paths: - session.install(*local_paths, "--no-deps", "--ignore-installed") - print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") + for dep in core_dependencies_from_source: + session.install(dep, "--no-deps", "--ignore-installed") + print(f"Installed {dep}") tests_path = os.path.join("tests", "unit") session.run( From 97e3bd1c8637691aee87d3ff4ac81a4eac42cd03 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 27 Aug 2026 09:46:57 -0700 Subject: [PATCH 29/32] revert core_deps changes --- .kokoro/continuous/system.cfg | 3 --- .kokoro/presubmit/core_deps.cfg | 7 ------- .kokoro/presubmit/system.cfg | 3 --- .kokoro/system.sh | 14 -------------- ci/get_package_shards.py | 8 ++++---- ci/run_single_test.sh | 8 ++++---- 6 files changed, 8 insertions(+), 35 deletions(-) delete mode 100644 .kokoro/presubmit/core_deps.cfg diff --git a/.kokoro/continuous/system.cfg b/.kokoro/continuous/system.cfg index d5dc63b5a58f..789455bd6973 100644 --- a/.kokoro/continuous/system.cfg +++ b/.kokoro/continuous/system.cfg @@ -1,8 +1,5 @@ # Format: //devtools/kokoro/config/proto/build.proto -# Increase timeout to 6 hours (default is 180 mins / 3 hours) -timeout_mins: 360 - # Only run this nox session. env_vars: { key: "NOX_SESSION" diff --git a/.kokoro/presubmit/core_deps.cfg b/.kokoro/presubmit/core_deps.cfg deleted file mode 100644 index 3f2987023513..000000000000 --- a/.kokoro/presubmit/core_deps.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "core_deps_from_source" -} diff --git a/.kokoro/presubmit/system.cfg b/.kokoro/presubmit/system.cfg index c6920f22aaeb..f01fa3c46e72 100644 --- a/.kokoro/presubmit/system.cfg +++ b/.kokoro/presubmit/system.cfg @@ -1,8 +1,5 @@ # Format: //devtools/kokoro/config/proto/build.proto -# Increase timeout to 6 hours (default is 180 mins / 3 hours) -timeout_mins: 360 - # Only run this nox session. env_vars: { key: "NOX_SESSION" diff --git a/.kokoro/system.sh b/.kokoro/system.sh index da08d4a2a98d..10ec4cbf40de 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -286,20 +286,6 @@ for path in `find 'packages' \ files_to_check=("${package_path}") fi - # When testing core dependencies from source, also trigger on changes to core packages - if [[ "${NOX_SESSION}" == "core_deps_from_source" ]]; then - files_to_check+=( - "packages/google-api-core" - "packages/google-auth" - "packages/google-auth-httplib2" - "packages/google-auth-oauthlib" - "packages/google-cloud-core" - "packages/googleapis-common-protos" - "packages/grpc-google-iam-v1" - "packages/proto-plus" - ) - fi - # When running compliance tests, only test packages that have compliance suites if [[ "${NOX_SESSION}" == "compliance"* && "${package_name}" != "sqlalchemy-"* ]]; then printf "SKIP %-20s %-40s %s\n" "[not_applicable]" "${package_name}" "${commit_hash:-HEAD}" diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py index c797617889c2..015adfc2bacd 100644 --- a/ci/get_package_shards.py +++ b/ci/get_package_shards.py @@ -22,12 +22,12 @@ preview-packages/foo) are kept aligned in the exact same shard. """ -import collections -import json -import math import os import subprocess +import json +import math import sys +import collections def get_package_directories(): @@ -173,7 +173,7 @@ def group_packages(packages_map): # Pack packages alphabetically by package name. for name, paths, weight in pkg_items: - # If adding this package would exceed target weight AND we haven't reached the + # If adding this package would exceed target weight AND we haven't reached the # shard limit, start a new shard. Otherwise, keep "stuffing" the current one. if current_shard_items and (current_shard_weight + weight > target_weight) and len(shards_list) < max_shards - 1: shards_list.append(current_shard_items) diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 2c6ad6b2fb7c..0ea0b969770d 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -19,7 +19,7 @@ # `PY_VERSION` should be one of ["3.10", "3.11", "3.12", "3.13"] # This script is called by the `ci/run_conditional_tests.sh` script. -# A specific `nox` session will be run, depending on the value of +# A specific `nox` session will be run, depending on the value of # `TEST_TYPE` and `PY_VERSION`. For example, if `TEST_TYPE` is # `lint`, the `nox -s lint` session will be run. @@ -131,12 +131,12 @@ case ${TEST_TYPE} in source .venv-profiler/bin/activate export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 python -m pip install --upgrade pip setuptools - + PROFILER_TEMP_DIR=$(mktemp -d) cp ../../scripts/import_profiler/profiler.py "${PROFILER_TEMP_DIR}/profiler.py" PROFILER_SCRIPT="${PROFILER_TEMP_DIR}/profiler.py" BASELINE_CSV="${PROFILER_TEMP_DIR}/baseline_${PACKAGE_NAME}.csv" - + if [ -n "${TARGET_BRANCH}" ]; then # Fetch history for the target branch without --depth=1 in case it was shallowly fetched if [ -f "$(git rev-parse --git-dir)/shallow" ]; then @@ -180,7 +180,7 @@ case ${TEST_TYPE} in echo "Could not find baseline commit for ${TARGET_BRANCH:-main}. Skipping baseline generation." fi fi - + # TODO(https://github.com/googleapis/google-cloud-python/issues/18035): # Clean up this fallback once Python 3.15 is officially released and upstream binary wheels are available on PyPI. # On pre-release Python versions, packages with complex C/Rust dependencies (e.g. bigframes) fail during pip install due to missing pre-built wheels. From 9b99e1bb3bf9d53872d1f0a6fff1df04f9a5ab0c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 27 Aug 2026 09:51:18 -0700 Subject: [PATCH 30/32] removed extra timeout config --- .kokoro/presubmit/sqlalchemy_compliance.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/.kokoro/presubmit/sqlalchemy_compliance.cfg b/.kokoro/presubmit/sqlalchemy_compliance.cfg index 1b48a0f49197..891deec4aa4e 100644 --- a/.kokoro/presubmit/sqlalchemy_compliance.cfg +++ b/.kokoro/presubmit/sqlalchemy_compliance.cfg @@ -1,8 +1,5 @@ # Format: //devtools/kokoro/config/proto/build.proto -# Increase timeout to 3 hours -timeout_mins: 180 - # Only run compliance nox session. env_vars: { key: "NOX_SESSION" From 533533146c684c5d17627e81e82e0fd38e3bedba Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 27 Aug 2026 09:54:42 -0700 Subject: [PATCH 31/32] removed compliance test logic --- .kokoro/presubmit/sqlalchemy_compliance.cfg | 7 ------- .kokoro/presubmit/system.cfg | 6 ------ .kokoro/system.sh | 6 ------ 3 files changed, 19 deletions(-) delete mode 100644 .kokoro/presubmit/sqlalchemy_compliance.cfg diff --git a/.kokoro/presubmit/sqlalchemy_compliance.cfg b/.kokoro/presubmit/sqlalchemy_compliance.cfg deleted file mode 100644 index 891deec4aa4e..000000000000 --- a/.kokoro/presubmit/sqlalchemy_compliance.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run compliance nox session. -env_vars: { - key: "NOX_SESSION" - value: "compliance" -} diff --git a/.kokoro/presubmit/system.cfg b/.kokoro/presubmit/system.cfg index f01fa3c46e72..789455bd6973 100644 --- a/.kokoro/presubmit/system.cfg +++ b/.kokoro/presubmit/system.cfg @@ -5,9 +5,3 @@ env_vars: { key: "NOX_SESSION" value: "system-3.12" } - -# Skip compliance tests in system job since they run in dedicated compliance job -env_vars: { - key: "RUN_COMPLIANCE_TESTS" - value: "false" -} diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 10ec4cbf40de..38d75a18f5ac 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -286,12 +286,6 @@ for path in `find 'packages' \ files_to_check=("${package_path}") fi - # When running compliance tests, only test packages that have compliance suites - if [[ "${NOX_SESSION}" == "compliance"* && "${package_name}" != "sqlalchemy-"* ]]; then - printf "SKIP %-20s %-40s %s\n" "[not_applicable]" "${package_name}" "${commit_hash:-HEAD}" - continue - fi - set +e # Passing the array expanded as arguments to git diff. package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- "${files_to_check[@]}" | wc -l) From 71fe5e45f0cc66a9789b97da91f237dc250840fd Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 27 Aug 2026 10:03:11 -0700 Subject: [PATCH 32/32] reduced per-package timeout --- .kokoro/system.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/system.sh b/.kokoro/system.sh index 38d75a18f5ac..d1184540bda1 100755 --- a/.kokoro/system.sh +++ b/.kokoro/system.sh @@ -310,7 +310,7 @@ done # Parallel Execution Logic MAX_JOBS=${MAX_JOBS:-4} -PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-120m}" +PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-60m}" # Temporary directory for clean log segregation LOG_DIR=$(mktemp -d -t test-logs-XXXXXX)