GH-3708: Inline parquet.thrift

README.md

@@ -62,6 +62,23 @@ brew install thrift
 export PATH="/usr/local/opt/thrift@0.23.0/bin:$PATH"
 ```
 
+### Update Parquet Thrift Definitions
+
+Parquet-Java uses an inlined version of the `parquet.thrift` IDL, located at
+`parquet-format-structures/src/main/thrift`. To update this definition to include
+changes from the canonical definition in [parquet-format](https://github.com/apache/parquet-format), you can use:
+
+```
+# Update to a parquet-format release
+> ./dev/update-parquet-thrift.sh apache-parquet-format-2.13.0
+
+# Update to an arbitrary parquet-format commit (can be unreleased)
+> ./dev/update-parquet-thrift.sh <commit hash>
+
+# Update to a commit from a parquet-format fork (e.g. for a reference implementation of an in-progress spec change)
+> PARQUET_FORMAT_REPO="https://github.com/divjotarora/parquet-format" ./dev/update-parquet-thrift.sh <commit hash>
+```
+
 ### Build Parquet with Maven
 
 Once protobuf and thrift are available in your path, you can build the project by running:

dev/update-parquet-thrift.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Updates the inlined parquet.thrift and sidecar to a specific parquet-format
+# commit or tag.
+#
+# Usage:
+#   update-parquet-thrift.sh <ref>
+#
+#   <ref> is a full 40-char parquet-format commit SHA, or a parquet-format tag
+#   (e.g. apache-parquet-format-2.13.0).
+#
+#   Set PARQUET_FORMAT_REPO to pull from a fork instead of apache/parquet-format,
+#   e.g. PARQUET_FORMAT_REPO=https://github.com/<user>/parquet-format.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+THRIFT_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet.thrift"
+SIDECAR_FILE="${REPO_ROOT}/parquet-format-structures/src/main/thrift/parquet-format.version"
+
+# Source repo, overridable via the PARQUET_FORMAT_REPO env var to pull from a fork.
+# The raw-download host is derived from it (assumes a GitHub-hosted repo).
+PARQUET_FORMAT_REPO="${PARQUET_FORMAT_REPO:-https://github.com/apache/parquet-format}"
+PARQUET_FORMAT_RAW="${PARQUET_FORMAT_REPO/github.com/raw.githubusercontent.com}"
+THRIFT_PATH_IN_FORMAT="src/main/thrift/parquet.thrift"
+
+usage() {
+  cat <<EOF
+Usage: $0 <ref>
+  <ref>  a full 40-char parquet-format commit SHA, or a parquet-format tag
+         (e.g. apache-parquet-format-2.13.0)
+EOF
+  exit 1
+}
+
+[[ $# -ne 1 ]] && usage
+
+ref="$1"
+
+# Resolve ref to a full commit SHA.
+if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then
+  resolved_sha="$ref"
+else
+  echo "Resolving ${ref} ..."
+  # ^{} refers to the unwrapped commit of an annotated tag, if present
+  ls_out="$(git ls-remote "$PARQUET_FORMAT_REPO" "$ref" "${ref}^{}")" || {
+    echo "ERROR: git ls-remote failed for '${ref}' in ${PARQUET_FORMAT_REPO}" >&2
+    exit 1
+  }
+  if [[ -z "$ls_out" ]]; then
+    echo "ERROR: ref '${ref}' not found in ${PARQUET_FORMAT_REPO}" >&2
+    exit 1
+  fi
+  # The ^{} peeled commit (annotated tags) is listed last, so tail -1 selects it.
+  resolved_sha="$(printf '%s\n' "$ls_out" | tail -1 | cut -f1)"
+  if [[ -z "$resolved_sha" ]]; then
+    echo "ERROR: could not resolve commit SHA for '${ref}'" >&2
+    exit 1
+  fi
+  echo "  -> commit: ${resolved_sha}"
+fi
+
+# Read the old commit from the sidecar for the summary.
+old_commit="(none)"
+if [[ -f "$SIDECAR_FILE" ]]; then
+  old_commit="$(grep '^parquet-format.commit=' "$SIDECAR_FILE" | cut -d= -f2 || true)"
+fi
+
+# Fetch upstream parquet.thrift to a temp file. Only overwrite the inlined file after fetch and
+# validation succeed, so a failure never overwrites the tracked IDL.
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+
+echo "Fetching parquet.thrift at ${resolved_sha} ..."
+url="${PARQUET_FORMAT_RAW}/${resolved_sha}/${THRIFT_PATH_IN_FORMAT}"
+if ! curl -fsSL -o "$tmp" "$url"; then
+  echo "ERROR: could not fetch parquet.thrift at ${resolved_sha}" >&2
+  echo "       The tracked files were not modified." >&2
+  exit 1
+fi
+
+if [[ ! -s "$tmp" ]]; then
+  echo "ERROR: fetched parquet.thrift is empty" >&2
+  exit 1
+fi
+if ! grep -q 'namespace java org.apache.parquet.format' "$tmp"; then
+  echo "ERROR: fetched file does not look like parquet.thrift (missing namespace declaration)" >&2
+  exit 1
+fi
+
+# Write the validated thrift file, then write the sidecar.
+mv "$tmp" "$THRIFT_FILE"
+chmod 644 "$THRIFT_FILE"
+echo "  -> written to ${THRIFT_FILE}"
+
+cat > "$SIDECAR_FILE" <<SIDECAR
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=${resolved_sha}
+SIDECAR
+echo "  -> sidecar updated: ${SIDECAR_FILE}"
+
+# Summary
+echo ""
+echo "Update complete:"
+echo "  old commit: ${old_commit}"
+echo "  new commit: ${resolved_sha}"
+echo ""
+echo "Next steps: rebuild parquet-format-structures and review the diff:"
+echo "  ./mvnw -pl parquet-format-structures -am install -DskipTests"
+echo "  git diff parquet-format-structures/src/main/thrift/parquet.thrift"

parquet-format-structures/pom.xml

@@ -34,44 +34,14 @@
   <url>https://parquet.apache.org/</url>
   <description>Parquet-mr related java classes to use the parquet-format thrift structures.</description>
 
-  <properties>
-    <parquet.thrift.path>${project.build.directory}/parquet-format-thrift</parquet.thrift.path>
-  </properties>
-
   <build>
     <plugins>
-      <!-- Getting the parquet-format thrift file -->
-       <plugin>
-         <groupId>org.apache.maven.plugins</groupId>
-         <artifactId>maven-dependency-plugin</artifactId>
-         <executions>
-           <execution>
-             <id>unpack</id>
-             <phase>initialize</phase>
-             <goals>
-               <goal>unpack</goal>
-             </goals>
-             <configuration>
-               <artifactItems>
-                 <artifactItem>
-                   <groupId>org.apache.parquet</groupId>
-                   <artifactId>parquet-format</artifactId>
-                   <version>${parquet.format.version}</version>
-                   <type>jar</type>
-                 </artifactItem>
-               </artifactItems>
-               <includes>parquet.thrift</includes>
-               <outputDirectory>${parquet.thrift.path}</outputDirectory>
-             </configuration>
-           </execution>
-         </executions>
-       </plugin>
       <!-- thrift -->
       <plugin>
         <groupId>org.apache.thrift</groupId>
         <artifactId>thrift-maven-plugin</artifactId>
         <configuration>
-          <thriftSourceRoot>${parquet.thrift.path}</thriftSourceRoot>
+          <thriftSourceRoot>${project.basedir}/src/main/thrift</thriftSourceRoot>
           <thriftExecutable>${format.thrift.executable}</thriftExecutable>
         </configuration>
         <executions>

parquet-format-structures/src/main/thrift/parquet-format.version

@@ -0,0 +1,2 @@
+# Provenance of the inlined parquet.thrift. Maintained by dev/update-parquet-thrift.sh.
+parquet-format.commit=c47e2a66e88943fc46fde1b028a9432f14fdf5c0