diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java index 20fbb13e..1a3de467 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java @@ -8,8 +8,14 @@ import com.google.gson.JsonParseException; import com.google.gson.JsonSerializationContext; import com.google.gson.JsonSerializer; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; import com.google.gson.annotations.JsonAdapter; import com.google.gson.annotations.SerializedName; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; import com.nimbusds.jose.JOSEException; import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.JWSHeader; @@ -30,6 +36,7 @@ import org.erdtman.jcs.JsonCanonicalizer; import java.io.IOException; +import java.io.StringReader; import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; @@ -38,6 +45,7 @@ import java.security.interfaces.RSAPublicKey; import java.security.cert.X509Certificate; import java.text.ParseException; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Base64; import java.util.List; @@ -118,6 +126,203 @@ public int hashCode() { } } + /** + * A compact, append-only {@link Segment} list. + *

+ * A plain {@code ArrayList} costs roughly 176 bytes per segment, which + * is several gigabytes for a multi-terabyte payload. Every segment of a TDF but + * the last has the same sizes, and every segment hash is the same fixed length, + * so all this stores is the hash characters packed into fixed-stride chunks + * plus the two size pairs — about 24 bytes per segment, with no array-growth + * copies and no {@code Integer.MAX_VALUE} ceiling on the backing store. + *

+ * {@link #append} refuses anything that does not fit those assumptions (a + * manifest from another writer may not honor them); callers fall back to an + * {@code ArrayList}. + *

+ * ponytail: the remaining ceiling is {@link #aggregate}, which materializes one + * byte array of {@code count * hashLength} — about 134M segments (256 TiB at + * 2 MiB segments), or ~100M if the TDF carries assertions, whose signature is + * base64 of that array. Going past that needs a streaming Mac and streaming + * base64, and a wire-format change for assertions. + */ + static final class Segments extends AbstractList { + private static final int SEGMENTS_PER_CHUNK = 4096; + /** + * A chunk is {@code stride * SEGMENTS_PER_CHUNK} bytes, so an absurd hash in an + * untrusted manifest would size the very first allocation. The longest hash any + * writer produces is 128 characters (hex-encoded HS256); anything past this is + * left to the {@code ArrayList} fallback, which only ever costs what the input + * itself costs. + */ + private static final int MAX_HASH_LENGTH = 1024; + + private final List chunks = new ArrayList<>(); + private int count; + /** Length of every hash, in characters; established by the first append. */ + private int stride; + /** Trailing '=' in every hash, so decoded hashes are also fixed length. */ + private int padding; + private long defaultSegmentSize; + private long defaultEncryptedSegmentSize; + private long lastSegmentSize; + private long lastEncryptedSegmentSize; + + /** + * @return false if this segment cannot be represented compactly, in which + * case the segment was not appended and the caller must fall back + */ + boolean append(String hash, long segmentSize, long encryptedSegmentSize) { + if (hash == null) { + return false; + } + if (count == 0) { + if (hash.isEmpty() || hash.length() > MAX_HASH_LENGTH) { + return false; + } + stride = hash.length(); + padding = trailingPadding(hash); + defaultSegmentSize = segmentSize; + defaultEncryptedSegmentSize = encryptedSegmentSize; + } else if (hash.length() != stride + || trailingPadding(hash) != padding + // the previously appended segment is no longer the last one, so it + // has to match the defaults from here on + || lastSegmentSize != defaultSegmentSize + || lastEncryptedSegmentSize != defaultEncryptedSegmentSize) { + return false; + } + + if (count % SEGMENTS_PER_CHUNK == 0) { + chunks.add(new byte[stride * SEGMENTS_PER_CHUNK]); + } + byte[] chunk = chunks.get(count / SEGMENTS_PER_CHUNK); + int offset = (count % SEGMENTS_PER_CHUNK) * stride; + for (int i = 0; i < stride; i++) { + char c = hash.charAt(i); + if (c > 0x7f) { // non-ASCII would not survive the byte-per-character packing + return false; + } + chunk[offset + i] = (byte) c; + } + + lastSegmentSize = segmentSize; + lastEncryptedSegmentSize = encryptedSegmentSize; + count++; + return true; + } + + private static int trailingPadding(String hash) { + int padding = 0; + while (padding < hash.length() && hash.charAt(hash.length() - 1 - padding) == '=') { + padding++; + } + return padding; + } + + /** + * Every segment hash concatenated, base64-decoded if {@code decodeBase64} + * and taken as raw ASCII otherwise. This is the aggregate hash that the root + * signature and the assertion signatures are computed over. + */ + byte[] aggregate(boolean decodeBase64) { + if (count == 0) { + return new byte[0]; + } + byte[] scratch = new byte[stride]; + if (!decodeBase64) { + byte[] aggregate = new byte[Math.multiplyExact(count, stride)]; + for (int i = 0; i < count; i++) { + copyHash(i, aggregate, i * stride); + } + return aggregate; + } + + copyHash(0, scratch, 0); + int decodedStride = Base64.getDecoder().decode(scratch).length; + byte[] aggregate = new byte[Math.multiplyExact(count, decodedStride)]; + byte[] decoded = new byte[decodedStride]; + for (int i = 0; i < count; i++) { + copyHash(i, scratch, 0); + Base64.getDecoder().decode(scratch, decoded); + System.arraycopy(decoded, 0, aggregate, i * decodedStride, decodedStride); + } + return aggregate; + } + + private void copyHash(int index, byte[] destination, int destinationOffset) { + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + System.arraycopy(chunk, (index % SEGMENTS_PER_CHUNK) * stride, destination, destinationOffset, stride); + } + + @Override + public Segment get(int index) { + Objects.checkIndex(index, count); + byte[] chunk = chunks.get(index / SEGMENTS_PER_CHUNK); + Segment segment = new Segment(); + segment.hash = new String(chunk, (index % SEGMENTS_PER_CHUNK) * stride, stride, StandardCharsets.US_ASCII); + boolean last = index == count - 1; + segment.segmentSize = last ? lastSegmentSize : defaultSegmentSize; + segment.encryptedSegmentSize = last ? lastEncryptedSegmentSize : defaultEncryptedSegmentSize; + return segment; + } + + @Override + public int size() { + return count; + } + } + + /** + * Reads {@code segments} into a {@link Segments} when the manifest allows it, + * and into a plain {@code ArrayList} when it does not. Writing delegates to the + * reflective {@link Segment} adapter, so the JSON is unchanged either way. + */ + static final class SegmentsAdapterFactory implements TypeAdapterFactory { + @Override + @SuppressWarnings("unchecked") + public TypeAdapter create(Gson gson, TypeToken type) { + TypeAdapter element = gson.getAdapter(Segment.class); + return (TypeAdapter) new TypeAdapter>() { + @Override + public void write(JsonWriter out, List value) throws IOException { + if (value == null) { + out.nullValue(); + return; + } + out.beginArray(); + for (Segment segment : value) { + element.write(out, segment); + } + out.endArray(); + } + + @Override + public List read(JsonReader in) throws IOException { + if (in.peek() == JsonToken.NULL) { + in.nextNull(); + return null; + } + in.beginArray(); + Segments compact = new Segments(); + List fallback = null; + while (in.hasNext()) { + Segment segment = element.read(in); + if (fallback != null) { + fallback.add(segment); + } else if (segment == null + || !compact.append(segment.hash, segment.segmentSize, segment.encryptedSegmentSize)) { + fallback = new ArrayList<>(compact); + fallback.add(segment); + } + } + in.endArray(); + return fallback == null ? compact : fallback; + } + }; + } + } + static public class RootSignature { @SerializedName(value = "alg") public String algorithm; @@ -145,6 +350,7 @@ static public class IntegrityInformation { public String segmentHashAlg; public int segmentSizeDefault; public int encryptedSegmentSizeDefault; + @JsonAdapter(SegmentsAdapterFactory.class) public List segments; @Override @@ -559,6 +765,14 @@ public AssertionConfig.Statement deserialize(JsonElement json, Type typeOfT, Jso public Payload payload; public List assertions = new ArrayList<>(); protected static Manifest readManifest(String manifestJson) { + return readManifest(new StringReader(manifestJson)); + } + + /** + * Parses a manifest without materializing it as a {@link String}, which a + * manifest with tens of millions of segments cannot be. + */ + protected static Manifest readManifest(java.io.Reader manifestJson) { Manifest result = gson.fromJson(manifestJson, Manifest.class); if (result.assertions == null) { result.assertions = new ArrayList<>(); @@ -583,9 +797,13 @@ protected static Manifest readManifest(String manifestJson) { throw new IllegalArgumentException("Manifest with null policy"); } - for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { - if (segment == null || segment.hash == null) { - throw new IllegalArgumentException("Invalid integrity segment"); + // Segments rejects null segments and hashes as it is built, so only the + // fallback representation needs checking here. + if (!(result.encryptionInformation.integrityInformation.segments instanceof Segments)) { + for (Manifest.Segment segment : result.encryptionInformation.integrityInformation.segments) { + if (segment == null || segment.hash == null) { + throw new IllegalArgumentException("Invalid integrity segment"); + } } } for (Manifest.KeyAccess keyAccess : result.encryptionInformation.keyAccessObj) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java index 5e903498..272bf8be 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/SDK.java @@ -185,8 +185,9 @@ public static boolean isTDF(SeekableByteChannel channel) { */ public static Manifest readManifest(SeekableByteChannel tdfBytes) throws SDKException, IOException { TDFReader reader = new TDFReader(tdfBytes); - String manifestJson = reader.manifest(); - return Manifest.readManifest(manifestJson); + try (var manifestJson = reader.manifest()) { + return Manifest.readManifest(manifestJson); + } } /** diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java index b54901b9..775ae819 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDF.java @@ -15,10 +15,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.security.*; @@ -81,6 +83,13 @@ private static byte[] tdfECKeySaltCompute() { private static final Gson gson = new GsonBuilder().create(); + /** + * The manifest is serialized directly into the zip entry, and + * {@link ZipWriter#stream} updates its CRC on every {@code write} call, so the + * character encoder needs a buffer in front of it. + */ + private static final int MANIFEST_WRITE_BUFFER_SIZE = 1 << 16; + /** * A self-imposed ceiling on the number of AES-GCM authenticated-encryption * invocations under a single payload key. One invocation is spent on the @@ -472,6 +481,23 @@ public PolicyObject readPolicyObject() { } } + /** + * Concatenates every segment hash, which is what the root signature and the + * assertion signatures are computed over. + */ + private static byte[] aggregateSegmentHashes(List segments, boolean isEncrypted) { + if (segments instanceof Manifest.Segments) { + return ((Manifest.Segments) segments).aggregate(isEncrypted); + } + var aggregateHash = new ByteArrayOutputStream(); + for (Manifest.Segment segment : segments) { + aggregateHash.writeBytes(isEncrypted + ? Base64.getDecoder().decode(segment.hash) + : segment.hash.getBytes()); + } + return aggregateHash.toByteArray(); + } + private static byte[] calculateSignature(byte[] data, byte[] secret, Config.IntegrityAlgorithm algorithm) { if (algorithm == Config.IntegrityAlgorithm.HS256) { return CryptoUtils.CalculateSHA256Hmac(secret, data); @@ -501,11 +527,11 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo long encryptedSegmentSize = (long) tdfConfig.defaultSegmentSize + kGcmIvSize + AesGcm.GCM_TAG_LENGTH; TDFWriter tdfWriter = new TDFWriter(outputStream); - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); byte[] readBuf = new byte[tdfConfig.defaultSegmentSize]; IvCounter payloadIv = IvCounter.forPayload(); - tdfObject.manifest.encryptionInformation.integrityInformation.segments = new ArrayList<>(); + var segments = new Manifest.Segments(); + tdfObject.manifest.encryptionInformation.integrityInformation.segments = segments; boolean finished; try (var payloadOutput = tdfWriter.payload()) { do { @@ -519,7 +545,6 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo byte[] cipherData; byte[] segmentSig; - Manifest.Segment segmentInfo = new Manifest.Segment(); // encrypt cipherData = tdfObject.aesGcm.encrypt(payloadIv.next(), AesGcm.GCM_TAG_LENGTH, @@ -530,19 +555,23 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo if (tdfConfig.hexEncodeRootAndSegmentHashes) { segmentSig = Hex.encodeHexString(segmentSig).getBytes(StandardCharsets.UTF_8); } - segmentInfo.hash = Base64.getEncoder().encodeToString(segmentSig); - aggregateHash.write(segmentSig); - segmentInfo.segmentSize = readThisLoop; - segmentInfo.encryptedSegmentSize = cipherData.length; - - tdfObject.manifest.encryptionInformation.integrityInformation.segments.add(segmentInfo); + // Every segment we write is full except the last one, and every hash has the + // same length, so the compact representation always accepts them. + if (!segments.append(Base64.getEncoder().encodeToString(segmentSig), readThisLoop, + cipherData.length)) { + throw new SDKException("unable to record segment " + segments.size() + " in the manifest"); + } } while (!finished); } + // Materialized once and reused by the root signature and every assertion below; + // it is proportional to the number of segments. + byte[] aggregateHash = segments.aggregate(true); + Manifest.RootSignature rootSignature = new Manifest.RootSignature(); - byte[] rootSig = calculateSignature(aggregateHash.toByteArray(), tdfObject.payloadKey, + byte[] rootSig = calculateSignature(aggregateHash, tdfObject.payloadKey, tdfConfig.integrityAlgorithm); byte[] encodedRootSig = tdfConfig.hexEncodeRootAndSegmentHashes ? Hex.encodeHexString(rootSig).getBytes(StandardCharsets.UTF_8) @@ -595,9 +624,9 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo throw new SDKException("error decoding assertion hash", e); } } - byte[] completeHash = new byte[aggregateHash.size() + assertionHash.length]; - System.arraycopy(aggregateHash.toByteArray(), 0, completeHash, 0, aggregateHash.size()); - System.arraycopy(assertionHash, 0, completeHash, aggregateHash.size(), assertionHash.length); + byte[] completeHash = new byte[aggregateHash.length + assertionHash.length]; + System.arraycopy(aggregateHash, 0, completeHash, 0, aggregateHash.length); + System.arraycopy(assertionHash, 0, completeHash, aggregateHash.length, assertionHash.length); var encodedHash = Base64.getEncoder().encodeToString(completeHash); @@ -618,9 +647,13 @@ TDFObject createTDF(InputStream payload, OutputStream outputStream, Config.TDFCo } tdfObject.manifest.assertions = signedAssertions; - String manifestAsStr = gson.toJson(tdfObject.manifest); - tdfWriter.appendManifest(manifestAsStr); + // Serialize straight into the zip entry: a manifest with tens of millions of segments + // exceeds the maximum size of a Java String. + try (var manifestOutput = new BufferedWriter( + new OutputStreamWriter(tdfWriter.manifest(), StandardCharsets.UTF_8), MANIFEST_WRITE_BUFFER_SIZE)) { + gson.toJson(tdfObject.manifest, manifestOutput); + } tdfObject.size = tdfWriter.finish(); return tdfObject; @@ -657,9 +690,11 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig, Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throws SDKException, IOException { TDFReader tdfReader = new TDFReader(tdf); - String manifestJson = tdfReader.manifest(); // use Manifest.readManifest in order to validate the Manifest input - Manifest manifest = Manifest.readManifest(manifestJson); + Manifest manifest; + try (var manifestJson = tdfReader.manifest()) { + manifest = Manifest.readManifest(manifestJson); + } byte[] payloadKey = new byte[GCM_KEY_SIZE]; String unencryptedMetadata = null; @@ -744,15 +779,8 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) String rootAlgorithm = manifest.encryptionInformation.integrityInformation.rootSignature.algorithm; String rootSignature = manifest.encryptionInformation.integrityInformation.rootSignature.signature; - ByteArrayOutputStream aggregateHash = new ByteArrayOutputStream(); - for (Manifest.Segment segment : manifest.encryptionInformation.integrityInformation.segments) { - if (manifest.payload.isEncrypted) { - byte[] decodedHash = Base64.getDecoder().decode(segment.hash); - aggregateHash.write(decodedHash); - } else { - aggregateHash.write(segment.hash.getBytes()); - } - } + byte[] aggregateHash = aggregateSegmentHashes( + manifest.encryptionInformation.integrityInformation.segments, manifest.payload.isEncrypted); String rootSigValue; boolean isLegacyTdf = manifest.tdfVersion == null || manifest.tdfVersion.isEmpty(); @@ -762,7 +790,7 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) sigAlg = Config.IntegrityAlgorithm.GMAC; } - var sig = calculateSignature(aggregateHash.toByteArray(), payloadKey, sigAlg); + var sig = calculateSignature(aggregateHash, payloadKey, sigAlg); if (isLegacyTdf) { sig = Hex.encodeHexString(sig).getBytes(); } @@ -775,7 +803,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new IllegalStateException("error getting instance of SHA-256 digest", e); } - rootSigValue = Base64.getEncoder().encodeToString(digest.digest(aggregateHash.toString().getBytes())); + // the round trip through the platform default charset is a no-op for the + // hex hashes this branch sees, and is kept so the digest is unchanged + rootSigValue = Base64.getEncoder().encodeToString(digest.digest(new String(aggregateHash).getBytes())); } if (rootSignature.compareTo(rootSigValue) != 0) { @@ -790,7 +820,6 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) "segment size mismatch. encrypted segment size differs from plaintext segment size. the TDF is invalid"); } - var aggregateHashByteArrayBytes = aggregateHash.toByteArray(); // Validate assertions for (var assertion : manifest.assertions) { // Skip assertion verification if disabled @@ -830,9 +859,9 @@ Reader loadTDF(SeekableByteChannel tdf, Config.TDFReaderConfig tdfReaderConfig) throw new SDKException("error decoding assertion hash", e); } } - var signature = new byte[aggregateHashByteArrayBytes.length + hashOfAssertion.length]; - System.arraycopy(aggregateHashByteArrayBytes, 0, signature, 0, aggregateHashByteArrayBytes.length); - System.arraycopy(hashOfAssertion, 0, signature, aggregateHashByteArrayBytes.length, hashOfAssertion.length); + var signature = new byte[aggregateHash.length + hashOfAssertion.length]; + System.arraycopy(aggregateHash, 0, signature, 0, aggregateHash.length); + System.arraycopy(hashOfAssertion, 0, signature, aggregateHash.length, hashOfAssertion.length); var encodeSignature = Base64.getEncoder().encodeToString(signature); if (!Objects.equals(encodeSignature, hashValues.getSignature())) { diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java index 6e9f32d2..590a17c3 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFReader.java @@ -1,8 +1,10 @@ package io.opentdf.platform.sdk; -import java.io.ByteArrayOutputStream; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -18,6 +20,8 @@ */ public class TDFReader { + private static final int MANIFEST_BUFFER_SIZE = 1 << 16; + private final ZipReader.Entry manifestEntry; private final InputStream payload; @@ -37,15 +41,18 @@ public TDFReader(SeekableByteChannel tdf) throws SDKException, IOException { payload = entries.get(TDF_PAYLOAD_FILE_NAME).getData(); } - String manifest() { - var out = new ByteArrayOutputStream(); + /** + * The manifest entry as a character stream; the caller must close it. Returned + * as a stream rather than a String because a manifest with tens of millions of + * segments exceeds the maximum size of a Java String. + */ + Reader manifest() { try { - manifestEntry.getData().transferTo(out); + return new BufferedReader( + new InputStreamReader(manifestEntry.getData(), StandardCharsets.UTF_8), MANIFEST_BUFFER_SIZE); } catch (IOException e) { throw new SDKException("error retrieving manifest from zip file", e); } - - return out.toString(StandardCharsets.UTF_8); } int readPayloadBytes(byte[] buf) { @@ -62,8 +69,10 @@ int readPayloadBytes(byte[] buf) { } PolicyObject readPolicyObject() { - String manifestJson = manifest(); - Manifest manifest = Manifest.readManifest(manifestJson); - return Manifest.decodePolicyObject(manifest); + try (Reader manifestRaw = manifest()) { + return Manifest.decodePolicyObject(Manifest.readManifest(manifestRaw)); + } catch (IOException e) { + throw new SDKException("error reading manifest from zip file", e); + } } } diff --git a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java index 7137c232..b64a39f1 100644 --- a/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java +++ b/sdk/src/main/java/io/opentdf/platform/sdk/TDFWriter.java @@ -24,8 +24,24 @@ public TDFWriter(OutputStream destination) { this.archiveWriter = new ZipWriter(destination, maxNonZip64Value); } + /** + * @deprecated use {@link #manifest()}. A manifest with tens of millions of segments + * exceeds the maximum size of a Java {@link String}, so this cannot + * express every manifest the SDK writes. It produces the same zip entry. + */ + @Deprecated public void appendManifest(String manifest) throws IOException { - this.archiveWriter.data(TDF_MANIFEST_FILE_NAME, manifest.getBytes(StandardCharsets.UTF_8)); + try (OutputStream output = manifest()) { + output.write(manifest.getBytes(StandardCharsets.UTF_8)); + } + } + + /** + * Opens the manifest entry for writing. The returned stream must be closed before + * {@link #finish()} is called, otherwise the entry never makes it into the central directory. + */ + public OutputStream manifest() throws IOException { + return this.archiveWriter.stream(TDF_MANIFEST_FILE_NAME); } public OutputStream payload() throws IOException { diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java new file mode 100644 index 00000000..3c5155ee --- /dev/null +++ b/sdk/src/test/java/io/opentdf/platform/sdk/SegmentsTest.java @@ -0,0 +1,227 @@ +package io.opentdf.platform.sdk; + +import com.google.gson.Gson; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.io.IOException; +import java.io.Reader; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Covers the compact {@link Manifest.Segments} representation: it has to + * serialize exactly like a plain list, fall back when a manifest does not fit + * its assumptions, and let a manifest larger than a Java String round trip. + */ +public class SegmentsTest { + + private static final Gson GSON = new Gson(); + private static final int SEGMENT_SIZE = 16384; + private static final int ENCRYPTED_SEGMENT_SIZE = SEGMENT_SIZE + 28; + + private static String hash(int i) { + var bytes = new byte[16]; + bytes[0] = (byte) i; + bytes[1] = (byte) (i >> 8); + bytes[2] = (byte) (i >> 16); + return Base64.getEncoder().encodeToString(bytes); + } + + private static Manifest.Segment segment(String hash, long segmentSize, long encryptedSegmentSize) { + var segment = new Manifest.Segment(); + segment.hash = hash; + segment.segmentSize = segmentSize; + segment.encryptedSegmentSize = encryptedSegmentSize; + return segment; + } + + private static String manifestJson(List segments) { + return "{\"schemaVersion\":\"4.3.0\",\"encryptionInformation\":{\"type\":\"split\"," + // no '=' padding: Gson HTML-escapes it, and this literal is compared verbatim + + "\"policy\":\"eyJib2R5Ijp7fX0K\",\"keyAccess\":[{\"type\":\"wrapped\",\"url\":\"http://kas\"," + + "\"protocol\":\"kas\",\"wrappedKey\":\"AAAA\"}]," + + "\"method\":{\"algorithm\":\"AES-256-GCM\",\"iv\":\"AAAAAAAAAAAA\",\"isStreamable\":true}," + + "\"integrityInformation\":{\"rootSignature\":{\"alg\":\"GMAC\",\"sig\":\"AAAA\"}," + + "\"segmentHashAlg\":\"GMAC\",\"segmentSizeDefault\":" + SEGMENT_SIZE + + ",\"encryptedSegmentSizeDefault\":" + ENCRYPTED_SEGMENT_SIZE + ",\"segments\":" + + segments.stream().map(GSON::toJson).collect(Collectors.joining(",", "[", "]")) + + "}},\"payload\":{\"type\":\"reference\",\"url\":\"0.payload\",\"protocol\":\"zip\"," + + "\"mimeType\":\"application/octet-stream\",\"isEncrypted\":true},\"assertions\":[]}"; + } + + /** A run of uniform segments with a short final one, i.e. what this SDK writes. */ + private static List uniformSegments(int count) { + return IntStream.range(0, count) + .mapToObj(i -> segment(hash(i), + i == count - 1 ? 7 : SEGMENT_SIZE, + i == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)) + .collect(Collectors.toList()); + } + + @Test + void uniformSegmentsAreStoredCompactlyAndSerializeUnchanged() { + var segments = uniformSegments(5); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void irregularSegmentSizesFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + // a non-final segment that disagrees with the defaults cannot be stored compactly + segments.set(1, segment(hash(1), SEGMENT_SIZE / 2, ENCRYPTED_SEGMENT_SIZE / 2)); + var json = manifestJson(segments); + + var manifest = Manifest.readManifest(json); + var parsed = manifest.encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + assertThat(Manifest.toJson(manifest)).isEqualTo(json); + } + + @Test + void segmentsWithDifferingHashLengthsFallBackToAPlainList() { + var segments = new ArrayList<>(uniformSegments(3)); + segments.set(1, segment("deadbeef", SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + var json = manifestJson(segments); + + var parsed = Manifest.readManifest(json).encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + } + + @Test + void anAbsurdlyLongHashFallsBackInsteadOfSizingAChunkFromIt() { + // a chunk is stride * 4096 bytes, so without a bound this one hash would ask for + // gigabytes before the fallback ever ran + var segments = new ArrayList<>(uniformSegments(2)); + segments.set(0, segment("A".repeat(100_000), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)); + + var parsed = Manifest.readManifest(manifestJson(segments)) + .encryptionInformation.integrityInformation.segments; + + assertThat(parsed).isNotInstanceOf(Manifest.Segments.class); + assertThat(parsed).containsExactlyElementsOf(segments); + } + + @Test + void aggregateConcatenatesEveryHash() { + var segments = new Manifest.Segments(); + for (int i = 0; i < 3; i++) { + assertThat(segments.append(hash(i), SEGMENT_SIZE, ENCRYPTED_SEGMENT_SIZE)).isTrue(); + } + + var decoded = new byte[48]; + for (int i = 0; i < 3; i++) { + System.arraycopy(Base64.getDecoder().decode(hash(i)), 0, decoded, i * 16, 16); + } + assertThat(segments.aggregate(true)).isEqualTo(decoded); + assertThat(segments.aggregate(false)) + .isEqualTo((hash(0) + hash(1) + hash(2)).getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + @Test + void emptySegmentsAggregateToNothing() { + assertThat(new Manifest.Segments().aggregate(true)).isEmpty(); + assertThat(new Manifest.Segments().aggregate(false)).isEmpty(); + } + + /** + * A manifest whose JSON is larger than {@code Integer.MAX_VALUE} cannot be held + * as a String, so both directions have to stream. Opt in with + * {@code -Dtdf.hugeSegments=true -Xmx2g}; it needs roughly a minute and 1 GiB. + */ + @Test + @EnabledIfSystemProperty(named = "tdf.hugeSegments", matches = "true") + void aManifestTooLargeToBeAStringRoundTrips() throws IOException { + // ~25M segments at ~99 JSON bytes each is ~2.5 GB, past the String ceiling + final int count = 25_000_000; + + Manifest manifest; + try (Reader json = new GeneratedManifestReader(count)) { + manifest = Manifest.readManifest(json); + } + var segments = manifest.encryptionInformation.integrityInformation.segments; + assertThat(segments).isInstanceOf(Manifest.Segments.class); + assertThat(segments.size()).isEqualTo(count); + assertThat(segments.get(count - 1).hash).isEqualTo(hash(count - 1)); + + var counting = new CountingWriter(); + new Gson().toJson(manifest, counting); + assertThat(counting.written).isGreaterThan(Integer.MAX_VALUE); + } + + /** Renders a manifest with {@code count} segments without ever storing it. */ + private static final class GeneratedManifestReader extends Reader { + private final int count; + private final String suffix; + private int next; + private String pending; + private int pendingOffset; + + GeneratedManifestReader(int count) { + this.count = count; + var template = manifestJson(uniformSegments(1)); + this.pending = template.substring(0, template.indexOf("\"segments\":[") + "\"segments\":[".length()); + this.suffix = template.substring(template.indexOf("]}},\"payload\"")); + } + + @Override + public int read(char[] buffer, int offset, int length) { + if (pendingOffset == pending.length()) { + if (next > count) { + return -1; + } + pending = next == count ? suffix + : (next == 0 ? "" : ",") + GSON.toJson(segment(hash(next), + next == count - 1 ? 7 : SEGMENT_SIZE, + next == count - 1 ? 35 : ENCRYPTED_SEGMENT_SIZE)); + pendingOffset = 0; + next++; + if (pending.isEmpty()) { + return read(buffer, offset, length); + } + } + int n = Math.min(length, pending.length() - pendingOffset); + pending.getChars(pendingOffset, pendingOffset + n, buffer, offset); + pendingOffset += n; + return n; + } + + @Override + public void close() { + } + } + + private static final class CountingWriter extends Writer { + private long written; + + @Override + public void write(char[] buffer, int offset, int length) { + written += length; + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + } +} diff --git a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java index 2f5d22fc..84433a88 100644 --- a/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java +++ b/sdk/src/test/java/io/opentdf/platform/sdk/TDFWriterTest.java @@ -5,13 +5,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.StringWriter; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.StandardOpenOption; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; public class TDFWriterTest { @Test @@ -64,14 +69,39 @@ void simpleTDFCreate() throws IOException { " }\n" + "}"; String payload = "Hello, world!"; - FileOutputStream fileOutStream = new FileOutputStream("sample.tdf"); - TDFWriter writer = new TDFWriter(fileOutStream); - try (var p = writer.payload()) { - new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + var tdfFile = File.createTempFile("sample", ".tdf"); + tdfFile.deleteOnExit(); + try (var fileOutStream = new FileOutputStream(tdfFile)) { + TDFWriter writer = new TDFWriter(fileOutStream); + try (var p = writer.payload()) { + new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); + } + try (var m = writer.manifest()) { + m.write(kManifestJsonFromTDF.getBytes(StandardCharsets.UTF_8)); + } + writer.finish(); + } + + // our own reader + try (var channel = FileChannel.open(tdfFile.toPath(), StandardOpenOption.READ)) { + var entries = new ZipReader(channel).getEntries().stream() + .collect(Collectors.toMap(ZipReader.Entry::getName, e -> e)); + assertEquals(kManifestJsonFromTDF, + new String(entries.get(TDFWriter.TDF_MANIFEST_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + assertEquals(payload, + new String(entries.get(TDFWriter.TDF_PAYLOAD_FILE_NAME).getData().readAllBytes(), + StandardCharsets.UTF_8)); + } + + // an independent central-directory based reader, as a stand-in for the other SDKs + try (var zipFile = new ZipFile(tdfFile)) { + var manifestEntry = zipFile.getEntry(TDFWriter.TDF_MANIFEST_FILE_NAME); + assertNotNull(manifestEntry); + try (var in = zipFile.getInputStream(manifestEntry)) { + assertEquals(kManifestJsonFromTDF, new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } } - writer.appendManifest(kManifestJsonFromTDF); - writer.finish(); - fileOutStream.close(); } /** @@ -89,12 +119,18 @@ void readsBackAManifestWrittenPastTheZip64Boundary() throws IOException { try (var p = writer.payload()) { new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8)).transferTo(p); } - writer.appendManifest(manifest); + try (var m = writer.manifest()) { + m.write(manifest.getBytes(StandardCharsets.UTF_8)); + } writer.finish(); try (var chan = new SeekableInMemoryByteChannel(out.toByteArray())) { var reader = new TDFReader(chan); - assertEquals(manifest, reader.manifest()); + var readBack = new StringWriter(); + try (var m = reader.manifest()) { + m.transferTo(readBack); + } + assertEquals(manifest, readBack.toString()); var payloadBytes = new byte[payload.length()]; assertEquals(payload.length(), reader.readPayloadBytes(payloadBytes));