diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/WriteHostBatcherTest.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/WriteHostBatcherTest.java index 3ffd17613..acf69d7a7 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/WriteHostBatcherTest.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/WriteHostBatcherTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.functionaltests; @@ -33,6 +33,12 @@ import java.io.*; import java.util.*; import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; @@ -1415,229 +1421,56 @@ public void run() { clearDB(port); } - // Multiple threads writing to same WHB object with unique uri's and with - // thread count =10 and txsize =3 but with small number of docs - // Failing intermittently with rewriteWHB branch - @Disabled + @Test public void testAddMultiThreadedLessDocsSuccess() throws Exception { - System.out.println("In testAddMultiThreadedLessDocsSuccess method"); - - final String query1 = "fn:count(fn:doc())"; - final AtomicInteger count = new AtomicInteger(0); - - ihbMT = dmManager.newWriteBatcher(); - ihbMT.withBatchSize(99); - ihbMT.withThreadCount(10); - // ihbMT.withTransactionSize(3); - - ihbMT.onBatchSuccess(batch -> { - - }).onBatchFailure((batch, throwable) -> { - throwable.printStackTrace(); - - }); - dmManager.startJob(ihbMT); - - class MyRunnable implements Runnable { - - @Override - public void run() { - - for (int j = 0; j < 15; j++) { - String uri = "/local/json-" + j + "-" + Thread.currentThread().getId(); - System.out.println("Thread name: " + Thread.currentThread().getName() + " URI:" + uri); - ihbMT.add(uri, fileHandle); - } - ihbMT.flushAndWait(); - } - - } - - class CountRunnable implements Runnable { - - @Override - public void run() { - try { - Thread.currentThread().sleep(15000L); - } catch (InterruptedException e) { - e.printStackTrace(); - } - Set threads = Thread.getAllStackTraces().keySet(); - Iterator iter = threads.iterator(); - while (iter.hasNext()) { - Thread t = iter.next(); - if (t.getName().contains("pool-1-thread-")) - System.out.println(t.getName()); - count.incrementAndGet(); - } - - } - - } - Thread countT; - countT = new Thread(new CountRunnable()); - - Thread t1, t2, t3; - t1 = new Thread(new MyRunnable()); - t2 = new Thread(new MyRunnable()); - t3 = new Thread(new MyRunnable()); - - countT.start(); - t1.start(); - t2.start(); - t3.start(); - - countT.join(); - - t1.join(); - t2.join(); - t3.join(); - - // assertTrue(count.intValue()==10); - assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 45); - clearDB(port); - } - - // Multiple threads writing to same WHB object with duplicate uri's and with - // thread count =10 and txsize =3 - // currently causing deadlock and at the completion of the test, clearDB() - // causes forests to go to middle closing state when run against reWriteHB - // branch - // Git Issue # 62 - @Disabled - public void testAddMultiThreadedwithThreadCountFailure() throws Exception { - System.out.println("In testAddMultiThreadedwithThreadCountFailure method"); - - final AtomicInteger count = new AtomicInteger(0); - final AtomicInteger eventCount = new AtomicInteger(0); - - ihbMT = dmManager.newWriteBatcher(); - ihbMT.withBatchSize(99); - ihbMT.withThreadCount(10); - // ihbMT.withTransactionSize(3); - - ihbMT.onBatchSuccess(batch -> { - synchronized (eventCount) { - eventCount.getAndAdd(batch.getItems().length); - } - - }).onBatchFailure((batch, throwable) -> { - throwable.printStackTrace(); - synchronized (eventCount) { - eventCount.getAndAdd(batch.getItems().length); - } - - }); - dmManager.startJob(ihbMT); - - class MyRunnable implements Runnable { - - @Override - public void run() { - - for (int j = 0; j < 5000; j++) { - String uri = "/local/json-" + j; - ihbMT.add(uri, fileHandle); - } - ihbMT.flushAndWait(); - } - - } - - class CountRunnable implements Runnable { - - @Override - public void run() { - try { - Thread.currentThread().sleep(15000L); - } catch (InterruptedException e) { - e.printStackTrace(); - } - Set threads = Thread.getAllStackTraces().keySet(); - Iterator iter = threads.iterator(); - while (iter.hasNext()) { - Thread t = iter.next(); - if (t.getName().contains("pool-1-thread-")) - count.incrementAndGet(); - - } - - } - - } - Thread countT; - countT = new Thread(new CountRunnable()); - - Thread t1, t2, t3; - t1 = new Thread(new MyRunnable()); - t2 = new Thread(new MyRunnable()); - t3 = new Thread(new MyRunnable()); - - countT.start(); - t1.start(); - t2.start(); - t3.start(); - - countT.join(); - - t1.join(); - t2.join(); - t3.join(); - - assertTrue(count.intValue() == 15000); - assertTrue(eventCount.intValue() == 10); - - } - - // ISSUE # 58 - @Disabled - public void testTransactionSize() throws Exception { - System.out.println("In testTransactionSize method"); - + int documentsPerProducer = 15; + int producerCount = 3; + AtomicInteger successCount = new AtomicInteger(); + AtomicInteger failureCount = new AtomicInteger(); + Queue failures = new ConcurrentLinkedQueue<>(); + WriteBatcher batcher = dmManager.newWriteBatcher() + .withBatchSize(99) + .withThreadCount(10) + .onBatchSuccess(batch -> successCount.addAndGet(batch.getItems().length)) + .onBatchFailure((batch, throwable) -> { + failureCount.addAndGet(batch.getItems().length); + failures.add(throwable); + }); + ExecutorService producers = Executors.newFixedThreadPool(producerCount); + CountDownLatch start = new CountDownLatch(1); + JobTicket ticket = dmManager.startJob(batcher); try { - final String query1 = "fn:count(fn:doc())"; - - final AtomicInteger successCount = new AtomicInteger(0); - - final AtomicBoolean failState = new AtomicBoolean(false); - final AtomicInteger failCount = new AtomicInteger(0); - - WriteBatcher ihb2 = dmManager.newWriteBatcher(); - ihb2.withBatchSize(480); - // ihb2.withTransactionSize(5); - dmManager.startJob(ihb2); - - ihb2.onBatchSuccess(batch -> { - - successCount.getAndAdd(batch.getItems().length); - System.out.println("Success Batch size " + batch.getItems().length); - for (WriteEvent w : batch.getItems()) { - System.out.println("Success " + w.getTargetUri()); - } - - }).onBatchFailure((batch, throwable) -> { - throwable.printStackTrace(); - System.out.println("Failure Batch size " + batch.getItems().length); - for (WriteEvent w : batch.getItems()) { - System.out.println("Failure " + w.getTargetUri()); - } - failState.set(true); - failCount.getAndAdd(batch.getItems().length); - }); - for (int j = 0; j < 500; j++) { - String uri = "/local/ABC-" + j; - ihb2.add(uri, stringHandle); + List> results = new ArrayList<>(); + for (int i = 0; i < producerCount; i++) { + int producer = i; + results.add(producers.submit(() -> { + start.await(); + for (int j = 0; j < documentsPerProducer; j++) { + String uri = "/local/json-" + j + "-" + producer; + batcher.add(uri, new FileHandle(fileJson).withFormat(Format.JSON)); + } + batcher.flushAsync(); + assertTrue(batcher.awaitCompletion(60, TimeUnit.SECONDS), "Concurrent flush timed out"); + return null; + })); + } + start.countDown(); + for (Future result : results) { + result.get(90, TimeUnit.SECONDS); + } + assertTrue(batcher.awaitCompletion(60, TimeUnit.SECONDS), "Write job timed out"); + assertTrue(failures.isEmpty(), () -> "Unexpected batch failures: " + failures); + assertEquals(0, failureCount.get()); + assertEquals(producerCount * documentsPerProducer, successCount.get(), + "Every submitted write must receive a success callback"); + assertEquals(producerCount * documentsPerProducer, getDocumentCount(dbName)); + } finally { + try { + dmManager.stopJob(ticket); + } finally { + producers.shutdownNow(); + assertTrue(producers.awaitTermination(30, TimeUnit.SECONDS), "Producer threads did not terminate"); } - - ihb2.flushAndWait(); - - System.out.println("Fail : " + failCount.intValue()); - System.out.println("Success : " + successCount.intValue()); - System.out - .println("Count : " + dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue()); - assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 500); - } catch (Exception e) { - e.printStackTrace(); } } diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTempMetaValues.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTempMetaValues.java index a7393ed10..0799add17 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTempMetaValues.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTempMetaValues.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.functionaltest; @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.marklogic.client.DatabaseClient; +import com.marklogic.client.FailedRequestException; import com.marklogic.client.Transaction; import com.marklogic.client.bitemporal.TemporalDocumentManager.ProtectionLevel; import com.marklogic.client.document.DocumentManager.Metadata; @@ -918,11 +919,11 @@ public void testProtectUpdateInTransaction() throws Exception { } } - @Disabled + @Test /* - * Test bitemporal protections - NOUPDATE With different transactions. Write - * doc in T1. Protect in T2. Update doc in T1 within 30 sec duration. TODO - * Wait for Git #542 + * Test bitemporal protections - NOUPDATE With different transactions. Commit + * the document in T1 and protection in T2, then use a new transaction to verify + * updates are rejected within 30 sec and succeed after protection expires. */ public void testProtectUpdateInDiffTransactions() throws Exception { @@ -932,9 +933,6 @@ public void testProtectUpdateInDiffTransactions() throws Exception { Calendar insertTime = DatatypeConverter.parseDateTime("2005-01-01T00:00:01"); Calendar updateTime = DatatypeConverter.parseDateTime("2005-01-01T00:00:11"); - Transaction t1 = writerClient.openTransaction(); - Transaction t2 = writerClient.openTransaction(); - String docId = "javaSingleJSONDoc.json"; JacksonDatabindHandle handle = getJSONDocumentHandle("2001-01-01T00:00:00", "2011-12-31T23:59:59", @@ -944,48 +942,61 @@ public void testProtectUpdateInDiffTransactions() throws Exception { JSONDocumentManager docMgr = writerClient.newJSONDocumentManager(); - docMgr.write("javaSingleJSONDocV1.json", docId, null, handle, null, t1, temporalLsqtCollectionName, insertTime); - - // Protect document for 30 sec from delete and update. Use Duration. Use T2 - docMgr.protect(docId, temporalLsqtCollectionName, ProtectionLevel.NOUPDATE, DatatypeFactory.newInstance().newDuration("PT30S"), t2); JacksonDatabindHandle handleUpd = getJSONDocumentHandle( "2003-01-01T00:00:00", "2008-12-31T23:59:59", "1999 Skyway Park - Updated - JSON", docId); - StringBuilder str = new StringBuilder(); + Transaction t1 = writerClient.openTransaction(); + Transaction t2 = null; try { - // Use t1 to write - docMgr.write(docId, null, handleUpd, null, t1, temporalLsqtCollectionName, updateTime); - } catch (Exception ex) { - str.append(ex.getMessage()); - System.out.println("Exception when update within 30 sec is " + str.toString()); - } - // TODO Yet to know what exception message will be. #542 - // assertTrue("Doc should not be updated", - // str.toString().contains("The document javaSingleJSONDoc.json is protected noUpdate")); - - // Sleep for 40 secs and try to update the same docId. - Thread.sleep(40000); - docMgr.write(docId, null, handleUpd, null, t1, temporalLsqtCollectionName, updateTime); - Thread.sleep(5000); + docMgr.write("javaSingleJSONDocV1.json", docId, null, handle, null, t1, temporalLsqtCollectionName, insertTime); + // Release the write locks before protecting the document in another transaction. + t1.commit(); + t1 = null; - JSONDocumentManager jsonDocMgr = writerClient.newJSONDocumentManager(); - DocumentPage readResults = jsonDocMgr.read(t1, docId); - System.out.println("Number of results = " + readResults.size()); - assertEquals(1, readResults.size()); + t2 = writerClient.openTransaction(); + docMgr.protect(docId, temporalLsqtCollectionName, ProtectionLevel.NOUPDATE, DatatypeFactory.newInstance().newDuration("PT30S"), t2); + t2.commit(); + t2 = null; + + Transaction updateTransaction = writerClient.openTransaction(); + // Preserve the original failure if rolling back also fails. + try (AutoCloseable rollback = updateTransaction::rollback) { + FailedRequestException ex = assertThrows(FailedRequestException.class, + () -> docMgr.write(docId, null, handleUpd, null, updateTransaction, temporalLsqtCollectionName, updateTime)); + assertTrue(ex.getMessage().contains("The document javaSingleJSONDoc.json is protected noUpdate")); + + // Sleep for 40 secs and try to update the same docId. + Thread.sleep(40000); + docMgr.write(docId, null, handleUpd, null, updateTransaction, temporalLsqtCollectionName, updateTime); + Thread.sleep(5000); - QueryManager queryMgr = writerClient.newQueryManager(); + JSONDocumentManager jsonDocMgr = writerClient.newJSONDocumentManager(); + try (DocumentPage readResults = jsonDocMgr.read(updateTransaction, docId)) { + System.out.println("Number of results = " + readResults.size()); + assertEquals(1, readResults.size()); + } - StructuredQueryBuilder sqb = queryMgr.newStructuredQueryBuilder(); - StructuredQueryDefinition termQuery = sqb.collection(temporalLsqtCollectionName); + QueryManager queryMgr = writerClient.newQueryManager(); + StructuredQueryBuilder sqb = queryMgr.newStructuredQueryBuilder(); + // Query only this document's versions, excluding the collection's LSQT document. + StructuredQueryDefinition termQuery = sqb.collection(docId); - long start = 1; - DocumentPage termQueryResults = docMgr.search(termQuery, start, t1); - System.out - .println("Number of results = " + termQueryResults.getTotalSize()); - assertEquals(4, termQueryResults.getTotalSize()); - t1.rollback(); - t2.rollback(); + long start = 1; + try (DocumentPage termQueryResults = docMgr.search(termQuery, start, updateTransaction)) { + System.out.println("Number of results = " + termQueryResults.size()); + assertEquals(4, termQueryResults.size()); + } + } + } finally { + try { + if (t2 != null) + t2.rollback(); + } finally { + if (t1 != null) + t1.rollback(); + } + } } @Test diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteBatcherTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteBatcherTest.java index a42bc2eda..679e61b43 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteBatcherTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteBatcherTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.datamovement; @@ -17,7 +17,6 @@ import com.marklogic.client.test.Common; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +26,7 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -1104,22 +1104,47 @@ public void run() { } } - // TODO: uncomment this after fixing https://github.com/marklogic/java-client-api/issues/646 - @Disabled + // Regression for https://github.com/marklogic/java-client-api/issues/646 + @Test public void testIssue646() throws Exception { - - WriteBatcher ihb2 = moveMgr.newWriteBatcher() - .withBatchSize(10); - - ihb2.onBatchFailure( (batch, throwable) -> throwable.printStackTrace() ); - - for (int j =0 ;j < 21; j++){ - String uri ="/local/string-"+ j; - DocumentMetadataHandle meta6 = new DocumentMetadataHandle().withProperty("docMeta-1", "true"); - ihb2.addAs(uri , meta6, "test"); + TextDocumentManager documentManager = client.newTextDocumentManager(); + AtomicInteger successCount = new AtomicInteger(); + AtomicReference failure = new AtomicReference<>(); + DocumentMetadataHandle metadata = new DocumentMetadataHandle() + .withCollections(whbTestCollection) + .withProperty("docMeta-1", "true") + .withQuality(1); + WriteBatcher batcher = moveMgr.newWriteBatcher() + .withBatchSize(10) + .withThreadCount(3) + .onBatchSuccess(batch -> successCount.addAndGet(batch.getItems().length)) + .onBatchFailure((batch, throwable) -> failure.compareAndSet(null, throwable)); + String[] uris = new String[21]; + for (int j = 0; j < uris.length; j++) { + uris[j] = "/local/issue646/string-" + j; + } + JobTicket ticket = moveMgr.startJob(batcher); + try { + for (String uri : uris) { + batcher.addAs(uri, metadata, "test"); + } + batcher.flushAsync(); + assertTrue(batcher.awaitCompletion(60, TimeUnit.SECONDS), "Metadata write job timed out"); + assertNull(failure.get(), () -> "Metadata write failed: " + failure.get()); + assertEquals(uris.length, successCount.get()); + for (String uri : uris) { + DocumentMetadataHandle actual = new DocumentMetadataHandle(); + assertEquals("test", documentManager.read(uri, actual, new StringHandle()).get()); + assertEquals("true", actual.getProperties().get("docMeta-1")); + assertEquals(1, actual.getQuality()); + } + } finally { + try { + moveMgr.stopJob(ticket); + } finally { + documentManager.delete(uris); + } } - - ihb2.flushAndWait(); } @Test diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java index a2d5087eb..a4c0d2b73 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java @@ -33,6 +33,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.util.ArrayList; import java.util.List; @@ -219,26 +220,27 @@ public void testMetadataPropertiesExtraction() { // testing https://github.com/marklogic/java-client-api/issues/783 -// @Test - @Disabled + @Test public void testStack20170725() throws IOException { XMLDocumentManager documentManager = Common.client.newXMLDocumentManager(); - Transaction transaction = Common.client.openTransaction(); - DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle(); - documentManager.writeAs("all_well.xml", new FileHandle( - new File("../marklogic-client-api-functionaltests/src/test/java/com" + - "/marklogic/client/functionaltest/data/all_well.xml"))); - InputStreamHandle handle = documentManager.read("all_well.xml", metadataHandle, new InputStreamHandle(), transaction); + String uri = "/test/issue783/all_well.xml"; + File file = new File("../marklogic-client-api-functionaltests/src/test/java/com" + + "/marklogic/client/functionaltest/data/all_well.xml"); + documentManager.write(uri, new DocumentMetadataHandle().withProperty("issue", "783"), new FileHandle(file)); try { - InputStream stream = handle.get(); + Transaction transaction = Common.client.openTransaction(); try { - while ( stream.read() > -1 ) {} + DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle(); + try (InputStreamHandle handle = documentManager.read(uri, metadataHandle, new InputStreamHandle(), transaction); + InputStream stream = handle.get()) { + assertEquals("783", metadataHandle.getProperties().get("issue")); + assertTrue(stream.transferTo(OutputStream.nullOutputStream()) > 0, "The complete stream must be readable"); + } } finally { - stream.close(); + transaction.rollback(); } } finally { - handle.close(); + documentManager.delete(uri); } - // if we ran without throwing any exceptions, then this test passes } }