From 8524cb993b2089df9f67e571378df3cf7519884e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 10 Oct 2025 14:17:16 -0400 Subject: [PATCH 01/70] Bumped to 8.0-SNAPSHOT --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index dc8a5f9c0..ce21d2cfa 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=8.0.0 +version=8.0-SNAPSHOT publishUrl=file:../marklogic-java/releases okhttpVersion=5.2.0 From 360db5eba8b9ee9a815ca69d6323b392527c2c63 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 13 Oct 2025 10:03:01 -0400 Subject: [PATCH 02/70] MLE-24717 Bumping logback to 1.5.19 Minor CVE thing. Getting rid of the PR template as well as it's not needed and GitKraken keeps trying to use it. --- .github/PULL_REQUEST_TEMPLATE.md | 9 --------- marklogic-client-api-functionaltests/build.gradle | 2 +- marklogic-client-api/build.gradle | 2 +- test-app/build.gradle | 2 +- 4 files changed, 3 insertions(+), 12 deletions(-) delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 623c3f1bd..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -So we can incorporate your pull request, please share the following: -* What issue are you addressing with this pull request? -* Are you modifying the correct branch? (See CONTRIBUTING.md) -* Have you run unit tests? (See CONTRIBUTING.md) -* Version of MarkLogic Java Client API (see Readme.txt) -* Version of MarkLogic Server (see admin gui on port 8001) -* Java version (`java -version`) -* OS and version -* What Changed: What happened before this change? What happens without this change? diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index d9cebfa63..e8d18b2c3 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -20,7 +20,7 @@ dependencies { exclude module: "marklogic-client-api" } - testImplementation 'ch.qos.logback:logback-classic:1.5.18' + testImplementation 'ch.qos.logback:logback-classic:1.5.19' testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index a8c48a096..cd3ace4a0 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -56,7 +56,7 @@ dependencies { testImplementation "com.squareup.okhttp3:mockwebserver3:5.1.0" testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" - testImplementation 'ch.qos.logback:logback-classic:1.5.18' + testImplementation 'ch.qos.logback:logback-classic:1.5.19' // Using this to avoid a schema validation issue with the regular xercesImpl testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' diff --git a/test-app/build.gradle b/test-app/build.gradle index a06a500b5..284d12d64 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -12,7 +12,7 @@ dependencies { implementation "io.undertow:undertow-core:2.3.19.Final" implementation "io.undertow:undertow-servlet:2.3.19.Final" implementation 'org.slf4j:slf4j-api:2.0.17' - implementation 'ch.qos.logback:logback-classic:1.5.18' + implementation 'ch.qos.logback:logback-classic:1.5.19' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } From a398a93163fb726fbd928a7f13136491444247dd Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 15 Oct 2025 15:00:03 -0400 Subject: [PATCH 03/70] MLE-24747 Bumping ml-gradle to 6.1.0 Removed hack in BitemporalTest for bug that is fixed. And bumped mockito, realized it can go to 5 safely now. --- .../build.gradle | 9 +++------ marklogic-client-api/build.gradle | 14 ++++++-------- .../marklogic/client/test/BitemporalTest.java | 17 ----------------- test-app/build.gradle | 2 +- 4 files changed, 10 insertions(+), 32 deletions(-) diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index e8d18b2c3..4a4d2e029 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -15,18 +15,15 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.19.0' - // Allows talking to the Manage API. - testImplementation("com.marklogic:ml-app-deployer:6.0.1") { - exclude module: "marklogic-client-api" - } + testImplementation "com.marklogic:ml-app-deployer:6.1.0" testImplementation 'ch.qos.logback:logback-classic:1.5.19' - testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' + testImplementation 'org.junit.jupiter:junit-jupiter:5.14.0' testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.13.4" + testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0" } tasks.withType(Test).configureEach { diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index cd3ace4a0..46b724103 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -46,14 +46,12 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.19.0' // Allows talking to the Manage API. - testImplementation("com.marklogic:ml-app-deployer:6.0.1") { - exclude module: "marklogic-client-api" - } + testImplementation "com.marklogic:ml-app-deployer:6.1.0" + + testImplementation "org.mockito:mockito-core:5.20.0" + testImplementation "org.mockito:mockito-inline:5.20.0" - // Starting with mockito 5.x, Java 11 is required, so sticking with 4.x as we have to support Java 8. - testImplementation "org.mockito:mockito-core:4.11.0" - testImplementation "org.mockito:mockito-inline:4.11.0" - testImplementation "com.squareup.okhttp3:mockwebserver3:5.1.0" + testImplementation "com.squareup.okhttp3:mockwebserver3:5.2.0" testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" testImplementation 'ch.qos.logback:logback-classic:1.5.19' @@ -73,7 +71,7 @@ dependencies { // https://docs.gradle.org/current/userguide/upgrading_version_8.html#test_framework_implementation_dependencies // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.13.4" + testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0" } // Ensure that mlHost and mlPassword can override the defaults of localhost/admin if they've been modified diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java index 9d447d098..97bb22d9d 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java @@ -15,8 +15,6 @@ import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.*; import com.marklogic.client.query.StructuredQueryBuilder.TemporalOperator; -import com.marklogic.mgmt.ManageClient; -import com.marklogic.mgmt.resource.temporal.TemporalCollectionLSQTManager; import jakarta.xml.bind.DatatypeConverter; import org.custommonkey.xmlunit.exceptions.XpathException; import org.junit.jupiter.api.AfterEach; @@ -163,21 +161,6 @@ void writeTwoVersionsOfFourDocuments() throws XpathException { @Test void lsqtTest() { - // Due to bug MLE-24511 where LSQT properties aren't updated correctly in ml-gradle 6.0.0, we need to manually - // deploy them for this test. - ManageClient manageClient = Common.newManageClient(); - TemporalCollectionLSQTManager mgr = new TemporalCollectionLSQTManager(manageClient, "java-unittest", "temporal-collection"); - String payload = """ - { - "lsqt-enabled": true, - "automation": { - "enabled": true, - "period": 5000 - } - } - """; - mgr.save(payload); - String version1 = """ %s version1 diff --git a/test-app/build.gradle b/test-app/build.gradle index 284d12d64..d9dc86e10 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -4,7 +4,7 @@ plugins { id "net.saliman.properties" version "1.5.2" - id 'com.marklogic.ml-gradle' version '6.0.1' + id 'com.marklogic.ml-gradle' version '6.1.0' id "com.github.psxpaul.execfork" version "0.2.2" } From 1775b8111bb184f6b071d9b1d825415316571ee2 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 16 Oct 2025 09:15:20 -0400 Subject: [PATCH 04/70] MLE-12708 Re-enabling test Bug was fixed in the server. --- .../java/com/marklogic/client/test/SPARQLManagerTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java index 9e37efa19..7287c639c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java @@ -30,7 +30,8 @@ import static org.junit.jupiter.api.Assertions.*; -public class SPARQLManagerTest { +class SPARQLManagerTest { + private static String graphUri = "http://marklogic.com/java/SPARQLManagerTest"; private static String triple1 = " ."; private static String triple2 = " ."; @@ -362,10 +363,6 @@ public void testSPARQLWithBindings() throws Exception { @Test public void testPagination() { - if (Common.getMarkLogicVersion().getMajor() >= 12) { - // Disabled until MLE-12708 is fixed. - return; - } SPARQLQueryDefinition qdef1 = smgr.newQueryDefinition( "SELECT ?s ?p ?o FROM <" + graphUri + "> { ?s ?p ?o }"); qdef1.setIncludeDefaultRulesets(false); From 7566d49d258fa177740944f9fe6da17ed31affde Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 16 Oct 2025 12:06:04 -0400 Subject: [PATCH 05/70] MLE-24747 Fixing typo in Gradle file Don't know why this didn't cause a failure initially, but did cause failures in nightly regressions. --- marklogic-client-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 46b724103..a70d9d4b9 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -49,7 +49,7 @@ dependencies { testImplementation "com.marklogic:ml-app-deployer:6.1.0" testImplementation "org.mockito:mockito-core:5.20.0" - testImplementation "org.mockito:mockito-inline:5.20.0" + testImplementation "org.mockito:mockito-inline:5.2.0" testImplementation "com.squareup.okhttp3:mockwebserver3:5.2.0" From 6f321dd17dea9fbb1ead5eb4c2adef478853aa4b Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 22 Oct 2025 14:04:35 -0400 Subject: [PATCH 06/70] MLE-24826 Bumping Spring, undertow for CVEs Also bumped an old version of xmlunit. Pretty sure the junit:junit stuff can be easily removed next. --- build.gradle | 1 - marklogic-client-api-functionaltests/build.gradle | 2 +- marklogic-client-api/build.gradle | 4 ++-- ml-development-tools/build.gradle | 2 +- test-app/build.gradle | 4 ++-- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 6ad238fe4..b71510d5f 100644 --- a/build.gradle +++ b/build.gradle @@ -23,7 +23,6 @@ subprojects { } repositories { - mavenLocal() mavenCentral() // Needed so that ml-development-tools can resolve snapshots of marklogic-client-api. diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index 4a4d2e029..0bfccada1 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -15,7 +15,7 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.19.0' - testImplementation "com.marklogic:ml-app-deployer:6.1.0" + testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" testImplementation 'ch.qos.logback:logback-classic:1.5.19' testImplementation 'org.junit.jupiter:junit-jupiter:5.14.0' diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index a70d9d4b9..8d0aca524 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -36,7 +36,7 @@ dependencies { compileOnly 'org.dom4j:dom4j:2.2.0' compileOnly 'com.google.code.gson:gson:2.13.2' - testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' + testImplementation 'org.junit.jupiter:junit-jupiter:5.14.0' // Forcing junit version to avoid vulnerability with older version in xmlunit testImplementation 'junit:junit:4.13.2' @@ -46,7 +46,7 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.19.0' // Allows talking to the Manage API. - testImplementation "com.marklogic:ml-app-deployer:6.1.0" + testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" testImplementation "org.mockito:mockito-core:5.20.0" testImplementation "org.mockito:mockito-inline:5.2.0" diff --git a/ml-development-tools/build.gradle b/ml-development-tools/build.gradle index f7bfb277f..9b0f67f85 100644 --- a/ml-development-tools/build.gradle +++ b/ml-development-tools/build.gradle @@ -32,7 +32,7 @@ dependencies { // Not yet migrating this project to JUnit 5. Will reconsider it once we have a reason to enhance // this project. testImplementation 'junit:junit:4.13.2' - testImplementation 'xmlunit:xmlunit:1.6' + testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' testCompileOnly gradleTestKit() testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" diff --git a/test-app/build.gradle b/test-app/build.gradle index d9dc86e10..f60f7b413 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -9,8 +9,8 @@ plugins { } dependencies { - implementation "io.undertow:undertow-core:2.3.19.Final" - implementation "io.undertow:undertow-servlet:2.3.19.Final" + implementation "io.undertow:undertow-core:2.3.20.Final" + implementation "io.undertow:undertow-servlet:2.3.20.Final" implementation 'org.slf4j:slf4j-api:2.0.17' implementation 'ch.qos.logback:logback-classic:1.5.19' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" From 1c51f56ccadff2410f1c8babf333143914b9e2ff Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 27 Oct 2025 10:57:16 -0400 Subject: [PATCH 07/70] MLE-24892 Cleaning up test dependencies Unfortunately can't upgrade to JUnit5 in ml-development-tools, far too many breaking changes to fix for now. --- marklogic-client-api/build.gradle | 2 -- .../com/marklogic/client/test/rows/TransformDocTest.java | 2 -- ml-development-tools/build.gradle | 8 ++------ 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 8d0aca524..11855b87b 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -38,8 +38,6 @@ dependencies { testImplementation 'org.junit.jupiter:junit-jupiter:5.14.0' - // Forcing junit version to avoid vulnerability with older version in xmlunit - testImplementation 'junit:junit:4.13.2' testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' testImplementation project(':examples') diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java index f7ed8233b..9e18896f5 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java @@ -13,9 +13,7 @@ import com.marklogic.client.io.marker.AbstractWriteHandle; import com.marklogic.client.row.RowRecord; import com.marklogic.client.test.Common; -import com.marklogic.client.test.MarkLogicVersion; import com.marklogic.client.test.junit5.RequiresML11; -import org.junit.Before; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/ml-development-tools/build.gradle b/ml-development-tools/build.gradle index 9b0f67f85..37836848e 100644 --- a/ml-development-tools/build.gradle +++ b/ml-development-tools/build.gradle @@ -29,13 +29,9 @@ dependencies { // Sticking with this older version for now as the latest 1.x version introduces breaking changes. implementation 'com.networknt:json-schema-validator:1.0.88' - // Not yet migrating this project to JUnit 5. Will reconsider it once we have a reason to enhance - // this project. + // Sticking with JUnit 4 as there are no vulnerabilities with it, and shifting to JUnit 5 in this module will be + // a significant and tedious effort. testImplementation 'junit:junit:4.13.2' - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' - testCompileOnly gradleTestKit() - - testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } // Added to avoid problem where processResources fails because - somehow - the plugin properties file is getting From ad497c336f1fb44242fbbf9348711a005c3638c5 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 18 Nov 2025 06:58:13 -0500 Subject: [PATCH 08/70] MLE-24892 Parameterizing image tags in Jenkinsfile --- Jenkinsfile | 73 ++++++++++++++++++----------------------------------- 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 080b15cf3..9226f1613 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -171,6 +171,7 @@ pipeline { parameters { booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') string(name: 'JAVA_VERSION', defaultValue: 'JAVA17', description: 'Either JAVA17 or JAVA21') + string(name: 'MARKLOGIC_IMAGE_TAGS', defaultValue: 'marklogic-server-ubi:latest-11,marklogic-server-ubi:latest-12', description: 'Comma-delimited list of MarkLogic image tags including variant (e.g., marklogic-server-ubi:latest-11,marklogic-server-ubi-rootless:11.3.2). The registry/org (ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic) path will be prepended automatically.') } environment { @@ -237,60 +238,36 @@ pipeline { } } - stage('regressions-11') { + stage('regressions') { when { allOf { branch 'develop' expression { return params.regressions } } } - steps { - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") - } - post { - always { - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() - } - } - } - - // Latest run had 87 errors, which have been added to MLE-24523 for later research. -// stage('regressions-12-reverseProxy') { -// when { -// allOf { -// branch 'develop' -// expression {return params.regressions} -// } -// } -// steps { -// runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") -// } -// post { -// always { -// junit '**/build/**/TEST*.xml' -// updateWorkspacePermissions() -// tearDownDocker() -// } -// } -// } - - stage('regressions-12') { - when { - allOf { - branch 'develop' - expression { return params.regressions } - } - } - steps { - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") - } - post { - always { - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() + steps { + script { + def imageTags = params.MARKLOGIC_IMAGE_TAGS.split(',') + def imagePrefix = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/' + + def parallelStages = [:] + imageTags.each { tag -> + def fullImage = imagePrefix + tag.trim() + def stageName = "regressions-${tag.trim().replace(':', '-')}" + + parallelStages[stageName] = { + stage(stageName) { + try { + runTests(fullImage) + } finally { + junit '**/build/**/TEST*.xml' + updateWorkspacePermissions() + tearDownDocker() + } + } + } + } + parallel parallelStages } } } From 3791e8a4b4657fb42779d7ddbbcb1ff81d8a3171 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 18 Nov 2025 08:29:03 -0500 Subject: [PATCH 09/70] MLE-24892 Fixing issue with parallel config --- Jenkinsfile | 45 ++++++++++++++++++++++++++----------------- test-app/build.gradle | 16 ++++++++++++++- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9226f1613..ef1af8976 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,12 +20,13 @@ def setupDockerMarkLogic(String image) { echo "Using image: "''' + image + ''' docker pull ''' + image + ''' MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build - echo "Waiting for MarkLogic server to initialize." - sleep 60s export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - ./gradlew mlTestConnections + export PATH=$JAVA_HOME/bin:$PATH + ./gradlew -i mlWaitTillReady + sleep 3 + ./gradlew -i mlWaitTillReady + ./gradlew mlTestConnections ./gradlew -i mlDeploy mlReloadSchemas ''' } @@ -161,7 +162,7 @@ def tearDownDocker() { } pipeline { - agent { label 'javaClientLinuxPool' } + agent none options { checkoutToSubdirectory 'java-client-api' @@ -184,6 +185,7 @@ pipeline { stages { stage('pull-request-tests') { + agent { label 'javaClientLinuxPool' } when { not { expression { return params.regressions } @@ -219,7 +221,9 @@ pipeline { } } } + stage('publish') { + agent { label 'javaClientLinuxPool' } when { branch 'develop' not { @@ -245,28 +249,33 @@ pipeline { expression { return params.regressions } } } - steps { - script { - def imageTags = params.MARKLOGIC_IMAGE_TAGS.split(',') - def imagePrefix = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/' - def parallelStages = [:] - imageTags.each { tag -> + steps { + script { + def imageTags = params.MARKLOGIC_IMAGE_TAGS.split(',') + def imagePrefix = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/' + + def parallelStages = [:] + + imageTags.each { tag -> def fullImage = imagePrefix + tag.trim() def stageName = "regressions-${tag.trim().replace(':', '-')}" parallelStages[stageName] = { - stage(stageName) { - try { - runTests(fullImage) - } finally { - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() + node('javaClientLinuxPool') { + stage(stageName) { + try { + runTests(fullImage) + } finally { + junit '**/build/**/TEST*.xml' + updateWorkspacePermissions() + tearDownDocker() + } } } } } + parallel parallelStages } } diff --git a/test-app/build.gradle b/test-app/build.gradle index f60f7b413..80a907488 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -2,12 +2,26 @@ * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ +buildscript { + repositories { + mavenCentral() + // Needed for ml-gradle 6.2-SNAPSHOT + maven { + url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" + } + } + dependencies { + classpath "com.marklogic:ml-gradle:6.2-SNAPSHOT" + } +} + plugins { id "net.saliman.properties" version "1.5.2" - id 'com.marklogic.ml-gradle' version '6.1.0' id "com.github.psxpaul.execfork" version "0.2.2" } +apply plugin: "com.marklogic.ml-gradle" + dependencies { implementation "io.undertow:undertow-core:2.3.20.Final" implementation "io.undertow:undertow-servlet:2.3.20.Final" From 57ad84e5873cfe07d7df1cd61384f23dd009c948 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 18 Nov 2025 09:09:33 -0500 Subject: [PATCH 10/70] MLE-24892 Backing off parallel config for now Having issues on Jenkins, just want to get dynamic stages in place for now. --- Jenkinsfile | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ef1af8976..5225054e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -162,7 +162,7 @@ def tearDownDocker() { } pipeline { - agent none + agent { label 'javaClientLinuxPool' } options { checkoutToSubdirectory 'java-client-api' @@ -185,7 +185,6 @@ pipeline { stages { stage('pull-request-tests') { - agent { label 'javaClientLinuxPool' } when { not { expression { return params.regressions } @@ -223,7 +222,6 @@ pipeline { } stage('publish') { - agent { label 'javaClientLinuxPool' } when { branch 'develop' not { @@ -255,28 +253,20 @@ pipeline { def imageTags = params.MARKLOGIC_IMAGE_TAGS.split(',') def imagePrefix = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/' - def parallelStages = [:] - imageTags.each { tag -> def fullImage = imagePrefix + tag.trim() def stageName = "regressions-${tag.trim().replace(':', '-')}" - parallelStages[stageName] = { - node('javaClientLinuxPool') { - stage(stageName) { - try { - runTests(fullImage) - } finally { - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() - } - } + stage(stageName) { + try { + runTests(fullImage) + } finally { + junit '**/build/**/TEST*.xml' + updateWorkspacePermissions() + tearDownDocker() } } } - - parallel parallelStages } } } From a035b8ee348a560d55a4b6368529f429acef8a55 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 10 Dec 2025 11:16:34 -0500 Subject: [PATCH 11/70] MLE-25782 Some refactoring of BatchWriter Wanted to move this into a separate class so it's easier / cleaner to prototype an incremental check listener. Also made BatchWriteSet nicer by making a bunch of fields final. No change in functionality, just cleaning up code. --- .../datamovement/impl/BatchWriteSet.java | 198 ++++++++---------- .../client/datamovement/impl/BatchWriter.java | 81 +++++++ .../datamovement/impl/WriteBatcherImpl.java | 80 +------ 3 files changed, 171 insertions(+), 188 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java index ce4426563..8cd7593a6 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java @@ -3,121 +3,95 @@ */ package com.marklogic.client.datamovement.impl; -import java.util.function.Consumer; - import com.marklogic.client.DatabaseClient; -import com.marklogic.client.document.DocumentWriteSet; -import com.marklogic.client.document.ServerTransform; import com.marklogic.client.datamovement.WriteBatch; import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.datamovement.WriteEvent; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.document.ServerTransform; + +import java.util.function.Consumer; -public class BatchWriteSet { - private WriteBatcher batcher; - private DocumentWriteSet writeSet; - private long batchNumber; - private long itemsSoFar; - private DatabaseClient client; - private ServerTransform transform; - private String temporalCollection; - private Runnable onSuccess; - private Consumer onFailure; - private Runnable onBeforeWrite; - - public BatchWriteSet(WriteBatcher batcher, DocumentWriteSet writeSet, DatabaseClient client, - ServerTransform transform, String temporalCollection) - { - this.batcher = batcher; - this.writeSet = writeSet; - this.client = client; - this.transform = transform; - this.temporalCollection = temporalCollection; - } - - public DocumentWriteSet getWriteSet() { - return writeSet; - } - - public void setWriteSet(DocumentWriteSet writeSet) { - this.writeSet = writeSet; - } - - public long getBatchNumber() { - return batchNumber; - } - - public void setBatchNumber(long batchNumber) { - this.batchNumber = batchNumber; - } - - public void setItemsSoFar(long itemsSoFar) { - this.itemsSoFar = itemsSoFar; - } - - public DatabaseClient getClient() { - return client; - } - - public void setClient(DatabaseClient client) { - this.client = client; - } - - public ServerTransform getTransform() { - return transform; - } - - public void setTransform(ServerTransform transform) { - this.transform = transform; - } - - public String getTemporalCollection() { - return temporalCollection; - } - - public void setTemporalCollection(String temporalCollection) { - this.temporalCollection = temporalCollection; - } - - public Runnable getOnSuccess() { - return onSuccess; - } - - public void onSuccess(Runnable onSuccess) { - this.onSuccess = onSuccess; - } - - public Consumer getOnFailure() { - return onFailure; - } - - public void onFailure(Consumer onFailure) { - this.onFailure = onFailure; - } - - public Runnable getOnBeforeWrite() { - return onBeforeWrite; - } - - public void onBeforeWrite(Runnable onBeforeWrite) { - this.onBeforeWrite = onBeforeWrite; - } - - public WriteBatch getBatchOfWriteEvents() { - WriteBatchImpl batch = new WriteBatchImpl() - .withBatcher(batcher) - .withClient(client) - .withJobBatchNumber(batchNumber) - .withJobWritesSoFar(itemsSoFar) - .withJobTicket(batcher.getJobTicket()); - WriteEvent[] writeEvents = getWriteSet().stream() - .map(writeOperation -> - new WriteEventImpl() - .withTargetUri(writeOperation.getUri()) - .withContent(writeOperation.getContent()) - .withMetadata(writeOperation.getMetadata()) - ) - .toArray(WriteEventImpl[]::new); - batch.withItems(writeEvents); - return batch; - } +class BatchWriteSet { + + private final WriteBatcher batcher; + private final DocumentWriteSet writeSet; + private final long batchNumber; + private final DatabaseClient client; + private final ServerTransform transform; + private final String temporalCollection; + + private long itemsSoFar; + private Runnable onSuccess; + private Consumer onFailure; + + BatchWriteSet(WriteBatcher batcher, DatabaseClient hostClient, ServerTransform transform, String temporalCollection, long batchNumber) { + this.batcher = batcher; + this.writeSet = hostClient.newDocumentManager().newWriteSet(); + this.client = hostClient; + this.transform = transform; + this.temporalCollection = temporalCollection; + this.batchNumber = batchNumber; + } + + public DocumentWriteSet getWriteSet() { + return writeSet; + } + + public long getBatchNumber() { + return batchNumber; + } + + public void setItemsSoFar(long itemsSoFar) { + this.itemsSoFar = itemsSoFar; + } + + public DatabaseClient getClient() { + return client; + } + + public ServerTransform getTransform() { + return transform; + } + + public String getTemporalCollection() { + return temporalCollection; + } + + public Runnable getOnSuccess() { + return onSuccess; + } + + public void onSuccess(Runnable onSuccess) { + this.onSuccess = onSuccess; + } + + public Consumer getOnFailure() { + return onFailure; + } + + public void onFailure(Consumer onFailure) { + this.onFailure = onFailure; + } + + public WriteBatch getBatchOfWriteEvents() { + WriteBatchImpl batch = new WriteBatchImpl() + .withBatcher(batcher) + .withClient(client) + .withJobBatchNumber(batchNumber) + .withJobWritesSoFar(itemsSoFar) + .withJobTicket(batcher.getJobTicket()); + + WriteEvent[] writeEvents = getWriteSet().stream() + .map(writeOperation -> + new WriteEventImpl() + .withTargetUri(writeOperation.getUri()) + .withContent(writeOperation.getContent()) + .withMetadata(writeOperation.getMetadata()) + ) + .toArray(WriteEventImpl[]::new); + + batch.withItems(writeEvents); + return batch; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java new file mode 100644 index 000000000..c3d1e6ffe --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.impl; + +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.io.Format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.util.function.Consumer; + +class BatchWriter implements Runnable { + + private static Logger logger = LoggerFactory.getLogger(WriteBatcherImpl.class); + + private final BatchWriteSet writeSet; + + BatchWriter(BatchWriteSet writeSet) { + if (writeSet.getWriteSet().size() == 0) { + throw new IllegalStateException("Attempt to write an empty batch"); + } + this.writeSet = writeSet; + } + + @Override + public void run() { + try { + logger.trace("begin write batch {} to forest on host \"{}\"", writeSet.getBatchNumber(), writeSet.getClient().getHost()); + if (writeSet.getTemporalCollection() == null) { + writeSet.getClient().newDocumentManager().write( + writeSet.getWriteSet(), writeSet.getTransform(), null + ); + } else { + // to get access to the TemporalDocumentManager write overload we need to instantiate + // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content + // format, so we'll set the default content format to unknown + XMLDocumentManager docMgr = writeSet.getClient().newXMLDocumentManager(); + docMgr.setContentFormat(Format.UNKNOWN); + docMgr.write( + writeSet.getWriteSet(), writeSet.getTransform(), null, writeSet.getTemporalCollection() + ); + } + closeAllHandles(); + Runnable onSuccess = writeSet.getOnSuccess(); + if (onSuccess != null) { + onSuccess.run(); + } + } catch (Throwable t) { + logger.trace("failed batch sent to forest on host \"{}\"", writeSet.getClient().getHost()); + Consumer onFailure = writeSet.getOnFailure(); + if (onFailure != null) { + onFailure.accept(t); + } + } + } + + private void closeAllHandles() throws Throwable { + Throwable lastThrowable = null; + for (DocumentWriteOperation doc : writeSet.getWriteSet()) { + try { + if (doc.getContent() instanceof Closeable) { + ((Closeable) doc.getContent()).close(); + } + if (doc.getMetadata() instanceof Closeable) { + ((Closeable) doc.getMetadata()).close(); + } + } catch (Throwable t) { + logger.error("error calling close()", t); + lastThrowable = t; + } + } + if (lastThrowable != null) throw lastThrowable; + } + + public BatchWriteSet getWriteSet() { + return writeSet; + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java index a87775285..424eaff13 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java @@ -3,7 +3,6 @@ */ package com.marklogic.client.datamovement.impl; -import java.io.Closeable; import java.util.*; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -12,7 +11,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.stream.Stream; import org.slf4j.Logger; @@ -22,10 +20,8 @@ import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.ServerTransform; -import com.marklogic.client.document.XMLDocumentManager; import com.marklogic.client.document.DocumentWriteOperation.OperationType; import com.marklogic.client.io.DocumentMetadataHandle; -import com.marklogic.client.io.Format; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.impl.Utilities; import com.marklogic.client.io.marker.AbstractWriteHandle; @@ -281,10 +277,7 @@ private BatchWriteSet newBatchWriteSet() { private BatchWriteSet newBatchWriteSet(long batchNum) { int hostToUse = (int) (batchNum % hostInfos.length); HostInfo host = hostInfos[hostToUse]; - DatabaseClient hostClient = host.client; - BatchWriteSet batchWriteSet = new BatchWriteSet(this, hostClient.newDocumentManager().newWriteSet(), - hostClient, getTransform(), getTemporalCollection()); - batchWriteSet.setBatchNumber(batchNum); + BatchWriteSet batchWriteSet = new BatchWriteSet(this, host.client, getTransform(), getTemporalCollection(), batchNum); batchWriteSet.onSuccess( () -> { sendSuccessToListeners(batchWriteSet); }); @@ -613,15 +606,15 @@ public synchronized WriteBatcher withForestConfig(ForestConfiguration forestConf for ( Runnable task : tasks ) { if ( task instanceof BatchWriter ) { BatchWriter writerTask = (BatchWriter) task; - if ( removedHostInfos.containsKey(writerTask.writeSet.getClient().getHost()) ) { + if ( removedHostInfos.containsKey(writerTask.getWriteSet().getClient().getHost()) ) { // this batch was targeting a host that's no longer on the list // if we re-add these docs they'll now be in batches that target acceptable hosts - BatchWriteSet writeSet = newBatchWriteSet(writerTask.writeSet.getBatchNumber()); + BatchWriteSet writeSet = newBatchWriteSet(writerTask.getWriteSet().getBatchNumber()); writeSet.onFailure(throwable -> { if ( throwable instanceof RuntimeException ) throw (RuntimeException) throwable; else throw new DataMovementException("Failed to retry batch after failover", throwable); }); - for ( WriteEvent doc : writerTask.writeSet.getBatchOfWriteEvents().getItems() ) { + for ( WriteEvent doc : writerTask.getWriteSet().getBatchOfWriteEvents().getItems() ) { writeSet.getWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } BatchWriter retryWriterTask = new BatchWriter(writeSet); @@ -649,71 +642,6 @@ public static class HostInfo { public DatabaseClient client; } - public static class BatchWriter implements Runnable { - private BatchWriteSet writeSet; - - public BatchWriter(BatchWriteSet writeSet) { - if ( writeSet.getWriteSet().size() == 0 ) { - throw new IllegalStateException("Attempt to write an empty batch"); - } - this.writeSet = writeSet; - } - - @Override - public void run() { - try { - Runnable onBeforeWrite = writeSet.getOnBeforeWrite(); - if ( onBeforeWrite != null ) { - onBeforeWrite.run(); - } - logger.trace("begin write batch {} to forest on host \"{}\"", writeSet.getBatchNumber(), writeSet.getClient().getHost()); - if ( writeSet.getTemporalCollection() == null ) { - writeSet.getClient().newDocumentManager().write( - writeSet.getWriteSet(), writeSet.getTransform(), null - ); - } else { - // to get access to the TemporalDocumentManager write overload we need to instantiate - // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content - // format, so we'll set the default content format to unknown - XMLDocumentManager docMgr = writeSet.getClient().newXMLDocumentManager(); - docMgr.setContentFormat(Format.UNKNOWN); - docMgr.write( - writeSet.getWriteSet(), writeSet.getTransform(), null, writeSet.getTemporalCollection() - ); - } - closeAllHandles(); - Runnable onSuccess = writeSet.getOnSuccess(); - if ( onSuccess != null ) { - onSuccess.run(); - } - } catch (Throwable t) { - logger.trace("failed batch sent to forest on host \"{}\"", writeSet.getClient().getHost()); - Consumer onFailure = writeSet.getOnFailure(); - if ( onFailure != null ) { - onFailure.accept(t); - } - } - } - - private void closeAllHandles() throws Throwable { - Throwable lastThrowable = null; - for ( DocumentWriteOperation doc : writeSet.getWriteSet() ) { - try { - if ( doc.getContent() instanceof Closeable ) { - ((Closeable) doc.getContent()).close(); - } - if ( doc.getMetadata() instanceof Closeable ) { - ((Closeable) doc.getMetadata()).close(); - } - } catch (Throwable t) { - logger.error("error calling close()", t); - lastThrowable = t; - } - } - if ( lastThrowable != null ) throw lastThrowable; - } - } - /** * The following classes and CompletableThreadPoolExecutor * CompletableRejectedExecutionHandler exist exclusively to enable the From 1e28465b1ec456fe6f9bf23067ae94f0fba87d89 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 10 Dec 2025 14:32:06 -0500 Subject: [PATCH 12/70] MLE-25782 Refactor: Explicit naming of writeSet objects Was going crazy trying to figure out if a "writeSet" was a BatchWriteSet or DocumentWriteSet. Now using more explicit names. --- .../datamovement/impl/BatchWriteSet.java | 14 +++++--- .../client/datamovement/impl/BatchWriter.java | 32 +++++++++---------- .../datamovement/impl/WriteBatcherImpl.java | 22 ++++++------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java index 8cd7593a6..0c08fdd7b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java @@ -12,10 +12,14 @@ import java.util.function.Consumer; +/** + * Mutable class that captures the documents to be written. Documents are added via calls to "getDocumentWriteSet()", where the + * DocumentWriteSet is empty when this class is constructed. + */ class BatchWriteSet { private final WriteBatcher batcher; - private final DocumentWriteSet writeSet; + private final DocumentWriteSet documentWriteSet; private final long batchNumber; private final DatabaseClient client; private final ServerTransform transform; @@ -27,15 +31,15 @@ class BatchWriteSet { BatchWriteSet(WriteBatcher batcher, DatabaseClient hostClient, ServerTransform transform, String temporalCollection, long batchNumber) { this.batcher = batcher; - this.writeSet = hostClient.newDocumentManager().newWriteSet(); + this.documentWriteSet = hostClient.newDocumentManager().newWriteSet(); this.client = hostClient; this.transform = transform; this.temporalCollection = temporalCollection; this.batchNumber = batchNumber; } - public DocumentWriteSet getWriteSet() { - return writeSet; + public DocumentWriteSet getDocumentWriteSet() { + return documentWriteSet; } public long getBatchNumber() { @@ -82,7 +86,7 @@ public WriteBatch getBatchOfWriteEvents() { .withJobWritesSoFar(itemsSoFar) .withJobTicket(batcher.getJobTicket()); - WriteEvent[] writeEvents = getWriteSet().stream() + WriteEvent[] writeEvents = getDocumentWriteSet().stream() .map(writeOperation -> new WriteEventImpl() .withTargetUri(writeOperation.getUri()) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java index c3d1e6ffe..037a781f3 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java @@ -16,41 +16,41 @@ class BatchWriter implements Runnable { private static Logger logger = LoggerFactory.getLogger(WriteBatcherImpl.class); - private final BatchWriteSet writeSet; + private final BatchWriteSet batchWriteSet; - BatchWriter(BatchWriteSet writeSet) { - if (writeSet.getWriteSet().size() == 0) { + BatchWriter(BatchWriteSet batchWriteSet) { + if (batchWriteSet.getDocumentWriteSet().size() == 0) { throw new IllegalStateException("Attempt to write an empty batch"); } - this.writeSet = writeSet; + this.batchWriteSet = batchWriteSet; } @Override public void run() { try { - logger.trace("begin write batch {} to forest on host \"{}\"", writeSet.getBatchNumber(), writeSet.getClient().getHost()); - if (writeSet.getTemporalCollection() == null) { - writeSet.getClient().newDocumentManager().write( - writeSet.getWriteSet(), writeSet.getTransform(), null + logger.trace("begin write batch {} to forest on host \"{}\"", batchWriteSet.getBatchNumber(), batchWriteSet.getClient().getHost()); + if (batchWriteSet.getTemporalCollection() == null) { + batchWriteSet.getClient().newDocumentManager().write( + batchWriteSet.getDocumentWriteSet(), batchWriteSet.getTransform(), null ); } else { // to get access to the TemporalDocumentManager write overload we need to instantiate // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content // format, so we'll set the default content format to unknown - XMLDocumentManager docMgr = writeSet.getClient().newXMLDocumentManager(); + XMLDocumentManager docMgr = batchWriteSet.getClient().newXMLDocumentManager(); docMgr.setContentFormat(Format.UNKNOWN); docMgr.write( - writeSet.getWriteSet(), writeSet.getTransform(), null, writeSet.getTemporalCollection() + batchWriteSet.getDocumentWriteSet(), batchWriteSet.getTransform(), null, batchWriteSet.getTemporalCollection() ); } closeAllHandles(); - Runnable onSuccess = writeSet.getOnSuccess(); + Runnable onSuccess = batchWriteSet.getOnSuccess(); if (onSuccess != null) { onSuccess.run(); } } catch (Throwable t) { - logger.trace("failed batch sent to forest on host \"{}\"", writeSet.getClient().getHost()); - Consumer onFailure = writeSet.getOnFailure(); + logger.trace("failed batch sent to forest on host \"{}\"", batchWriteSet.getClient().getHost()); + Consumer onFailure = batchWriteSet.getOnFailure(); if (onFailure != null) { onFailure.accept(t); } @@ -59,7 +59,7 @@ public void run() { private void closeAllHandles() throws Throwable { Throwable lastThrowable = null; - for (DocumentWriteOperation doc : writeSet.getWriteSet()) { + for (DocumentWriteOperation doc : batchWriteSet.getDocumentWriteSet()) { try { if (doc.getContent() instanceof Closeable) { ((Closeable) doc.getContent()).close(); @@ -75,7 +75,7 @@ private void closeAllHandles() throws Throwable { if (lastThrowable != null) throw lastThrowable; } - public BatchWriteSet getWriteSet() { - return writeSet; + public BatchWriteSet getBatchWriteSet() { + return batchWriteSet; } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java index 424eaff13..e7ae80d9c 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java @@ -204,7 +204,7 @@ public WriteBatcher add(DocumentWriteOperation writeOperation) { BatchWriteSet writeSet = newBatchWriteSet(); int minBatchSize = 0; if(defaultMetadata != null) { - writeSet.getWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); + writeSet.getDocumentWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); minBatchSize = 1; } for (int i=0; i < getBatchSize(); i++ ) { @@ -213,9 +213,9 @@ public WriteBatcher add(DocumentWriteOperation writeOperation) { // strange, there should have been a full batch of docs in the queue... break; } - writeSet.getWriteSet().add(doc); + writeSet.getDocumentWriteSet().add(doc); } - if ( writeSet.getWriteSet().size() > minBatchSize ) { + if ( writeSet.getDocumentWriteSet().size() > minBatchSize ) { threadPool.submit( new BatchWriter(writeSet) ); } } @@ -326,7 +326,7 @@ private void retry(WriteBatch batch, boolean callFailListeners) { }); } for (WriteEvent doc : batch.getItems()) { - writeSet.getWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); + writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } BatchWriter runnable = new BatchWriter(writeSet); runnable.run(); @@ -392,12 +392,12 @@ private void flush(boolean waitForCompletion) { } BatchWriteSet writeSet = newBatchWriteSet(); if(defaultMetadata != null) { - writeSet.getWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); + writeSet.getDocumentWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); } int j=0; for ( ; j < getBatchSize() && iter.hasNext(); j++ ) { DocumentWriteOperation doc = iter.next(); - writeSet.getWriteSet().add(doc); + writeSet.getDocumentWriteSet().add(doc); } threadPool.submit( new BatchWriter(writeSet) ); } @@ -406,7 +406,7 @@ private void flush(boolean waitForCompletion) { } private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { - batchWriteSet.setItemsSoFar(itemsSoFar.addAndGet(batchWriteSet.getWriteSet().size())); + batchWriteSet.setItemsSoFar(itemsSoFar.addAndGet(batchWriteSet.getDocumentWriteSet().size())); WriteBatch batch = batchWriteSet.getBatchOfWriteEvents(); for ( WriteBatchListener successListener : successListeners ) { try { @@ -606,16 +606,16 @@ public synchronized WriteBatcher withForestConfig(ForestConfiguration forestConf for ( Runnable task : tasks ) { if ( task instanceof BatchWriter ) { BatchWriter writerTask = (BatchWriter) task; - if ( removedHostInfos.containsKey(writerTask.getWriteSet().getClient().getHost()) ) { + if ( removedHostInfos.containsKey(writerTask.getBatchWriteSet().getClient().getHost()) ) { // this batch was targeting a host that's no longer on the list // if we re-add these docs they'll now be in batches that target acceptable hosts - BatchWriteSet writeSet = newBatchWriteSet(writerTask.getWriteSet().getBatchNumber()); + BatchWriteSet writeSet = newBatchWriteSet(writerTask.getBatchWriteSet().getBatchNumber()); writeSet.onFailure(throwable -> { if ( throwable instanceof RuntimeException ) throw (RuntimeException) throwable; else throw new DataMovementException("Failed to retry batch after failover", throwable); }); - for ( WriteEvent doc : writerTask.getWriteSet().getBatchOfWriteEvents().getItems() ) { - writeSet.getWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); + for ( WriteEvent doc : writerTask.getBatchWriteSet().getBatchOfWriteEvents().getItems() ) { + writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } BatchWriter retryWriterTask = new BatchWriter(writeSet); Runnable fretryWriterTask = (Runnable) threadPool.submit(retryWriterTask); From 8c2f2ebd245d05752cd4fe1dde6e67fbe5033488 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 11 Dec 2025 13:29:19 -0500 Subject: [PATCH 13/70] MLE-25959 Deprecating AUTH_TYPE_MARKLOGIC_CLOUD Should have renamed this in a prior release. --- .../marklogic/client/DatabaseClientBuilder.java | 14 +++++++++++--- .../client/impl/DatabaseClientPropertySource.java | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java index 976f870e6..a39d00802 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java @@ -29,7 +29,15 @@ public class DatabaseClientBuilder { public final static String PREFIX = "marklogic.client."; public final static String AUTH_TYPE_BASIC = "basic"; public final static String AUTH_TYPE_DIGEST = "digest"; - public final static String AUTH_TYPE_MARKLOGIC_CLOUD = "cloud"; + + public final static String AUTH_TYPE_CLOUD = "cloud"; + + /** + * @deprecated as of 8.1.0, use AUTH_TYPE_CLOUD instead + */ + @Deprecated + public final static String AUTH_TYPE_MARKLOGIC_CLOUD = AUTH_TYPE_CLOUD; + public final static String AUTH_TYPE_KERBEROS = "kerberos"; public final static String AUTH_TYPE_CERTIFICATE = "certificate"; public final static String AUTH_TYPE_SAML = "saml"; @@ -150,7 +158,7 @@ public DatabaseClientBuilder withDigestAuth(String username, String password) { } public DatabaseClientBuilder withCloudAuth(String apiKey, String basePath) { - return withAuthType(AUTH_TYPE_MARKLOGIC_CLOUD) + return withAuthType(AUTH_TYPE_CLOUD) .withCloudApiKey(apiKey) .withBasePath(basePath); } @@ -163,7 +171,7 @@ public DatabaseClientBuilder withCloudAuth(String apiKey, String basePath) { * @since 6.3.0 */ public DatabaseClientBuilder withCloudAuth(String apiKey, String basePath, Integer tokenDuration) { - return withAuthType(AUTH_TYPE_MARKLOGIC_CLOUD) + return withAuthType(AUTH_TYPE_CLOUD) .withCloudApiKey(apiKey) .withBasePath(basePath) .withCloudTokenDuration(tokenDuration != null ? tokenDuration.toString() : null); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java index 806815e05..ceffa379a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java @@ -185,7 +185,7 @@ private DatabaseClientFactory.SecurityContext newSecurityContext(String type, Co return newBasicAuthContext(connectionString); case DatabaseClientBuilder.AUTH_TYPE_DIGEST: return newDigestAuthContext(connectionString); - case DatabaseClientBuilder.AUTH_TYPE_MARKLOGIC_CLOUD: + case DatabaseClientBuilder.AUTH_TYPE_CLOUD: return newCloudAuthContext(); case DatabaseClientBuilder.AUTH_TYPE_KERBEROS: return newKerberosAuthContext(); @@ -400,7 +400,7 @@ private String getSSLProtocol(String authType) { } // For convenience for Progress Data Cloud users, assume the JVM's default SSLContext should trust the certificate // used by Progress Data Cloud. A user can always override this default behavior by providing their own SSLContext. - if ((sslProtocol == null || sslProtocol.length() == 0) && DatabaseClientBuilder.AUTH_TYPE_MARKLOGIC_CLOUD.equalsIgnoreCase(authType)) { + if ((sslProtocol == null || sslProtocol.length() == 0) && DatabaseClientBuilder.AUTH_TYPE_CLOUD.equalsIgnoreCase(authType)) { sslProtocol = "default"; } return sslProtocol; From 823409d6dacc109cdc6379c360b130c7bf147ff2 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 16 Dec 2025 10:20:04 -0500 Subject: [PATCH 14/70] MLE-25959 Quieted down PDC logging Info-level was too verbose for token generation. Also fixed a little bug in an error message built for an exception. --- ...ressDataCloudAuthenticationConfigurer.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java index afd35b527..3730cc6a2 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java @@ -24,7 +24,7 @@ class ProgressDataCloudAuthenticationConfigurer implements AuthenticationConfigu @Override public void configureAuthentication(OkHttpClient.Builder clientBuilder, DatabaseClientFactory.ProgressDataCloudAuthContext securityContext) { final String apiKey = securityContext.getApiKey(); - if (apiKey == null || apiKey.trim().length() < 1) { + if (apiKey == null || apiKey.trim().isEmpty()) { throw new IllegalArgumentException("No API key provided"); } TokenGenerator tokenGenerator = new DefaultTokenGenerator(this.host, securityContext); @@ -56,8 +56,8 @@ public DefaultTokenGenerator(String host, DatabaseClientFactory.ProgressDataClou public String generateToken() { final Response tokenResponse = callTokenEndpoint(); String token = getAccessTokenFromResponse(tokenResponse); - if (logger.isInfoEnabled()) { - logger.info("Successfully obtained authentication token"); + if (logger.isDebugEnabled()) { + logger.debug("Successfully obtained authentication token"); } return token; } @@ -70,8 +70,8 @@ private Response callTokenEndpoint() { OkHttpUtil.configureSocketFactory(clientBuilder, securityContext.getSSLContext(), securityContext.getTrustManager()); OkHttpUtil.configureHostnameVerifier(clientBuilder, securityContext.getSSLHostnameVerifier()); - if (logger.isInfoEnabled()) { - logger.info("Calling token endpoint at: " + tokenUrl); + if (logger.isDebugEnabled()) { + logger.debug("Calling token endpoint at: {}", tokenUrl); } final Call call = clientBuilder.build().newCall( @@ -85,7 +85,7 @@ private Response callTokenEndpoint() { return call.execute(); } catch (IOException e) { throw new ProgressDataCloudException(String.format("Unable to call token endpoint at %s; cause: %s", - tokenUrl, e.getMessage(), e)); + tokenUrl, e.getMessage()), e); } } @@ -97,7 +97,8 @@ protected HttpUrl buildTokenUrl() { .host(host) .port(443) .build() - .resolve(securityContext.getTokenEndpoint()).newBuilder(); + .resolve(securityContext.getTokenEndpoint()) + .newBuilder(); Integer duration = securityContext.getTokenDuration(); return duration != null ? @@ -146,7 +147,7 @@ public TokenAuthenticationInterceptor(TokenGenerator tokenGenerator) { @Override public Response intercept(Chain chain) throws IOException { Request.Builder builder = chain.request().newBuilder(); - addTokenToRequest(builder); + builder = addTokenToRequest(builder); Response response = chain.proceed(builder.build()); if (response.code() == 401) { logger.info("Received 401; will generate new token if necessary and retry request"); @@ -155,7 +156,7 @@ public Response intercept(Chain chain) throws IOException { generateNewTokenIfNecessary(currentToken); builder = chain.request().newBuilder(); - addTokenToRequest(builder); + builder = addTokenToRequest(builder); response = chain.proceed(builder.build()); } return response; From dcb8064d33c4b3ee75b032d4698f780ff9a766d8 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 29 Dec 2025 17:37:16 -0500 Subject: [PATCH 15/70] MLE-26420 Bumping dependencies and adding marklogic-junit5 In the next PR, will use marklogic-junit5 to make "real" tests that clear out all data except "test-data" documents. --- .copyrightconfig | 2 +- build.gradle | 2 ++ gradle.properties | 6 ++++-- .../build.gradle | 10 +++++----- marklogic-client-api/build.gradle | 20 ++++++++++++------- .../marklogic/client/test/ssl/SSLTest.java | 2 +- test-app/build.gradle | 5 ++++- .../ml-data/optic/test/collections.properties | 2 +- .../optic/vectors/collections.properties | 1 + .../optic/zipcodes/collections.properties | 2 +- .../ml-data/sample/collections.properties | 11 +++++----- .../ml-data/sample2/collections.properties | 2 +- 12 files changed, 40 insertions(+), 25 deletions(-) create mode 100644 test-app/src/main/ml-data/optic/vectors/collections.properties diff --git a/.copyrightconfig b/.copyrightconfig index ba242e11f..25e785b7c 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties diff --git a/build.gradle b/build.gradle index b71510d5f..d83bfdc8c 100644 --- a/build.gradle +++ b/build.gradle @@ -25,6 +25,8 @@ subprojects { repositories { mavenCentral() + mavenLocal() + // Needed so that ml-development-tools can resolve snapshots of marklogic-client-api. maven { url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" diff --git a/gradle.properties b/gradle.properties index ce21d2cfa..7e8c4ac9b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,10 +2,12 @@ group=com.marklogic version=8.0-SNAPSHOT publishUrl=file:../marklogic-java/releases -okhttpVersion=5.2.0 +okhttpVersion=5.3.2 # See https://github.com/FasterXML/jackson for more information on the Jackson libraries. -jacksonVersion=2.20.0 +jacksonVersion=2.20.1 + +junitVersion=6.0.1 # Defined at this level so that they can be set as system properties and used by the marklogic-client-api and test-app # project diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index 0bfccada1..2dcda2303 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -7,23 +7,23 @@ dependencies { testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" testImplementation 'org.skyscreamer:jsonassert:1.5.3' testImplementation 'org.slf4j:slf4j-api:2.0.17' - testImplementation 'commons-io:commons-io:2.20.0' + testImplementation 'commons-io:commons-io:2.21.0' testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" testImplementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" testImplementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" testImplementation "org.jdom:jdom2:2.0.6.1" - testImplementation 'org.apache.commons:commons-lang3:3.19.0' + testImplementation 'org.apache.commons:commons-lang3:3.20.0' testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" testImplementation 'ch.qos.logback:logback-classic:1.5.19' - testImplementation 'org.junit.jupiter:junit-jupiter:5.14.0' - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" + testImplementation 'org.xmlunit:xmlunit-legacy:2.11.0' // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0" + testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junitVersion}" } tasks.withType(Test).configureEach { diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 11855b87b..23b984883 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -6,6 +6,12 @@ plugins { id 'maven-publish' } +configurations.all { + // Ensure that no test dependencies below, namely marklogic-junit5, bring in their own version of marklogic-client-api, + // as we only want to use the one built by this project. + exclude group: 'com.marklogic', module: 'marklogic-client-api' +} + dependencies { // Using the latest version now that the 8.0.0 release requires Java 17. // This is now an implementation dependency as opposed to an api dependency in 7.x and earlier. @@ -36,23 +42,23 @@ dependencies { compileOnly 'org.dom4j:dom4j:2.2.0' compileOnly 'com.google.code.gson:gson:2.13.2' - testImplementation 'org.junit.jupiter:junit-jupiter:5.14.0' + testImplementation "com.marklogic:marklogic-junit5:2.0-SNAPSHOT" - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' + testImplementation 'org.xmlunit:xmlunit-legacy:2.11.0' testImplementation project(':examples') - testImplementation 'org.apache.commons:commons-lang3:3.19.0' + testImplementation 'org.apache.commons:commons-lang3:3.20.0' // Allows talking to the Manage API. testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" - testImplementation "org.mockito:mockito-core:5.20.0" + testImplementation "org.mockito:mockito-core:5.21.0" testImplementation "org.mockito:mockito-inline:5.2.0" - testImplementation "com.squareup.okhttp3:mockwebserver3:5.2.0" + testImplementation "com.squareup.okhttp3:mockwebserver3:${okhttpVersion}" testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" - testImplementation 'ch.qos.logback:logback-classic:1.5.19' + testImplementation 'ch.qos.logback:logback-classic:1.5.23' // Using this to avoid a schema validation issue with the regular xercesImpl testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' @@ -69,7 +75,7 @@ dependencies { // https://docs.gradle.org/current/userguide/upgrading_version_8.html#test_framework_implementation_dependencies // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0" + testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junitVersion}" } // Ensure that mlHost and mlPassword can override the defaults of localhost/admin if they've been modified diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java index 366bc6074..933ec2e77 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java @@ -81,7 +81,7 @@ public void testSSLAuth() throws NoSuchAlgorithmException, KeyManagementExceptio @Test // Not able to mock the X509Certificate class on Java 21. - @EnabledOnJre({JRE.JAVA_8, JRE.JAVA_11, JRE.JAVA_17}) + @EnabledOnJre({JRE.JAVA_17}) public void testHostnameVerifier() throws SSLException, CertificateParsingException { // three things our SSLHostnameVerifier will capture AtomicReference capturedHost = new AtomicReference<>(); diff --git a/test-app/build.gradle b/test-app/build.gradle index 80a907488..f8d21f472 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -5,6 +5,9 @@ buildscript { repositories { mavenCentral() + + mavenLocal() + // Needed for ml-gradle 6.2-SNAPSHOT maven { url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" @@ -26,7 +29,7 @@ dependencies { implementation "io.undertow:undertow-core:2.3.20.Final" implementation "io.undertow:undertow-servlet:2.3.20.Final" implementation 'org.slf4j:slf4j-api:2.0.17' - implementation 'ch.qos.logback:logback-classic:1.5.19' + implementation 'ch.qos.logback:logback-classic:1.5.23' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } diff --git a/test-app/src/main/ml-data/optic/test/collections.properties b/test-app/src/main/ml-data/optic/test/collections.properties index ee57b7b93..6ba0c7efd 100644 --- a/test-app/src/main/ml-data/optic/test/collections.properties +++ b/test-app/src/main/ml-data/optic/test/collections.properties @@ -1 +1 @@ -*=/optic/test,/optic/music +*=/optic/test,/optic/music,test-data diff --git a/test-app/src/main/ml-data/optic/vectors/collections.properties b/test-app/src/main/ml-data/optic/vectors/collections.properties new file mode 100644 index 000000000..a777f445c --- /dev/null +++ b/test-app/src/main/ml-data/optic/vectors/collections.properties @@ -0,0 +1 @@ +*=test-data diff --git a/test-app/src/main/ml-data/optic/zipcodes/collections.properties b/test-app/src/main/ml-data/optic/zipcodes/collections.properties index 0b92404e2..175de43d4 100644 --- a/test-app/src/main/ml-data/optic/zipcodes/collections.properties +++ b/test-app/src/main/ml-data/optic/zipcodes/collections.properties @@ -1 +1 @@ -*=zipcode +*=zipcode,test-data diff --git a/test-app/src/main/ml-data/sample/collections.properties b/test-app/src/main/ml-data/sample/collections.properties index e41971f1e..ffb43069a 100644 --- a/test-app/src/main/ml-data/sample/collections.properties +++ b/test-app/src/main/ml-data/sample/collections.properties @@ -1,5 +1,6 @@ -suggestion.xml=http://some.org/suggestions -first.xml=http://some.org/collection1,http://some.org/collection2 -lexicon-test1.xml=http://some.org/collection1,http://some.org/collection2 -lexicon-test2.xml=http://some.org/collection1,http://some.org/collection2 -second.txt=document-format-query-test +*=test-data +suggestion.xml=http://some.org/suggestions,test-data +first.xml=http://some.org/collection1,http://some.org/collection2,test-data +lexicon-test1.xml=http://some.org/collection1,http://some.org/collection2,test-data +lexicon-test2.xml=http://some.org/collection1,http://some.org/collection2,test-data +second.txt=document-format-query-test,test-data diff --git a/test-app/src/main/ml-data/sample2/collections.properties b/test-app/src/main/ml-data/sample2/collections.properties index e7d83da23..ebdc32646 100644 --- a/test-app/src/main/ml-data/sample2/collections.properties +++ b/test-app/src/main/ml-data/sample2/collections.properties @@ -1 +1 @@ -*=http://some.org/suggestions +*=http://some.org/suggestions,test-data From 3e84b5c43708b09811bbae8020d184c34a97a232 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 29 Dec 2025 18:52:55 -0500 Subject: [PATCH 16/70] MLE-26420 First test using marklogic-junit5 This doesn't test incremental writes yet, but it will soon. Just want to get a "real" test in place that wipes data from the database before it starts. Also toying with a template approach for WriteBatcher, so made DataMovementManager Closeable which is a non-breaking change. --- .../datamovement/DataMovementManager.java | 11 ++- .../client/test/AbstractClientTest.java | 28 ++++++++ .../datamovement/IncrementalWriteTest.java | 68 +++++++++++++++++++ .../test/rows/AbstractOpticUpdateTest.java | 7 +- 4 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java index 31152d6ef..f024b55bf 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java @@ -7,6 +7,7 @@ import com.marklogic.client.io.marker.ContentHandle; import com.marklogic.client.query.*; +import java.io.Closeable; import java.util.Iterator; /** @@ -33,7 +34,15 @@ * dataMovementManager.release(); *} */ -public interface DataMovementManager { +public interface DataMovementManager extends Closeable { + + /** + * @since 8.1.0 + */ + default void close() { + release(); + } + /** Calls release() on all host-specific DatabaseClient instances (but not on * the DatabaseClient instance used to create this DataMovementManager * instance). diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java new file mode 100644 index 000000000..020aa21d7 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.test; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.junit5.AbstractMarkLogicTest; + +/** + * Intended to be the base class for all future client API tests, as it properly prepares the database by deleting + * documents from previous test runs that were not created as part of deploying the test app. + */ +public abstract class AbstractClientTest extends AbstractMarkLogicTest { + + @Override + protected final DatabaseClient getDatabaseClient() { + return Common.newServerAdminClient(); + } + + @Override + protected final String getJavascriptForDeletingDocumentsBeforeTestRuns() { + return """ + declareUpdate(); + cts.uris('', [], cts.notQuery(cts.collectionQuery(['test-data', 'temporal-collection']))) + .toArray().forEach(item => xdmp.documentDelete(item)) + """; + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java new file mode 100644 index 000000000..83a81e1c5 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.test.datamovement; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class IncrementalWriteTest extends AbstractClientTest { + + private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + @Test + void test() { + AtomicInteger writtenCount = new AtomicInteger(); + + try (DatabaseClient client = Common.newClient()) { + WriteBatcherTemplate template = new WriteBatcherTemplate(client); + + template.runWriteJob(writeBatcher -> writeBatcher + .withThreadCount(1) + .withBatchSize(10) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), + + writeBatcher -> { + for (int i = 1; i <= 20; i++) { + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "" + i + "This is document number " + i + ""; + writeBatcher.add(uri, METADATA, new StringHandle(content)); + } + } + ); + } + + assertEquals(20, writtenCount.get()); + } + + // Experimenting with a template that gets rid of some annoying DMSDK boilerplate. + private record WriteBatcherTemplate(DatabaseClient databaseClient) { + + public void runWriteJob(Consumer writeBatcherConfigurer, Consumer writeBatcherUser) { + try (DataMovementManager dmm = databaseClient.newDataMovementManager()) { + WriteBatcher writeBatcher = dmm.newWriteBatcher(); + writeBatcherConfigurer.accept(writeBatcher); + + dmm.startJob(writeBatcher); + + writeBatcherUser.accept(writeBatcher); + writeBatcher.awaitCompletion(); + + dmm.stopJob(writeBatcher); + } + } + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java index 0afa19e8c..df9682e37 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java @@ -18,6 +18,7 @@ import com.marklogic.client.row.RawPlanDefinition; import com.marklogic.client.row.RowManager; import com.marklogic.client.row.RowRecord; +import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -32,7 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; -public abstract class AbstractOpticUpdateTest { +public abstract class AbstractOpticUpdateTest extends AbstractClientTest { private final static String XML_PREAMBLE = "\n"; @@ -42,10 +43,6 @@ public abstract class AbstractOpticUpdateTest { @BeforeEach public void setup() { - // Subclasses of this test are expected to only write URIs starting with /acme/ (which is used so that test - // URIs show up near the top when exploring the database in qconsole), so delete all of them before running the - // test to ensure a document doesn't already exist. - Common.deleteUrisWithPattern("/acme/*"); Common.client = Common.newClientBuilder().withUsername("writer-no-default-permissions").build(); rowManager = Common.client.newRowManager().withUpdate(true); op = rowManager.newPlanBuilder(); From b452860225e0c908b8ae3028a9aca0883d43e3e3 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 30 Dec 2025 13:30:41 -0500 Subject: [PATCH 17/70] MLE-26420 Some refactoring of WriteBatcher before incremental write PR Just some cleanup here before new functionality is added. Fixing some warnings in WriteBatcherImpl, and changed BatchWriter into a record. --- gradle.properties | 2 +- .../client/datamovement/WriteBatcher.java | 15 +-- .../client/datamovement/impl/BatchWriter.java | 93 ++++++++++--------- .../datamovement/impl/WriteBatcherImpl.java | 66 +++++-------- .../client/test/AbstractClientTest.java | 8 ++ 5 files changed, 87 insertions(+), 97 deletions(-) diff --git a/gradle.properties b/gradle.properties index 7e8c4ac9b..d109496b4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=8.0-SNAPSHOT +version=8.1-SNAPSHOT publishUrl=file:../marklogic-java/releases okhttpVersion=5.3.2 diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java index 2757ad58b..656b029cb 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java @@ -3,15 +3,15 @@ */ package com.marklogic.client.datamovement; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.ServerTransform; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.marker.AbstractWriteHandle; import com.marklogic.client.io.marker.DocumentMetadataWriteHandle; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + /** *

To facilitate long-running write jobs, batches documents added by many * external threads and coordinates internal threads to send the batches @@ -182,12 +182,7 @@ WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle, * * @param queryEvent the information about the batch that failed */ - public void retry(WriteBatch queryEvent); - - /* - public WriteBatcher withTransactionSize(int transactionSize); - public int getTransactionSize(); - */ + void retry(WriteBatch queryEvent); /** * Get the array of WriteBatchListener instances registered via @@ -361,5 +356,5 @@ WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle, * * @param writeBatch the information about the batch that failed */ - public void retryWithFailureListeners(WriteBatch writeBatch); + void retryWithFailureListeners(WriteBatch writeBatch); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java index 037a781f3..2173034dd 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java @@ -4,6 +4,7 @@ package com.marklogic.client.datamovement.impl; import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.XMLDocumentManager; import com.marklogic.client.io.Format; import org.slf4j.Logger; @@ -12,48 +13,58 @@ import java.io.Closeable; import java.util.function.Consumer; -class BatchWriter implements Runnable { +record BatchWriter(BatchWriteSet batchWriteSet) implements Runnable { private static Logger logger = LoggerFactory.getLogger(WriteBatcherImpl.class); - private final BatchWriteSet batchWriteSet; - - BatchWriter(BatchWriteSet batchWriteSet) { - if (batchWriteSet.getDocumentWriteSet().size() == 0) { - throw new IllegalStateException("Attempt to write an empty batch"); - } - this.batchWriteSet = batchWriteSet; - } - @Override public void run() { + if (batchWriteSet.getDocumentWriteSet() == null || batchWriteSet.getDocumentWriteSet().isEmpty()) { + logger.debug("Unexpected empty batch {}, skipping", batchWriteSet.getBatchNumber()); + return; + } + try { - logger.trace("begin write batch {} to forest on host \"{}\"", batchWriteSet.getBatchNumber(), batchWriteSet.getClient().getHost()); - if (batchWriteSet.getTemporalCollection() == null) { - batchWriteSet.getClient().newDocumentManager().write( - batchWriteSet.getDocumentWriteSet(), batchWriteSet.getTransform(), null - ); - } else { - // to get access to the TemporalDocumentManager write overload we need to instantiate - // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content - // format, so we'll set the default content format to unknown - XMLDocumentManager docMgr = batchWriteSet.getClient().newXMLDocumentManager(); - docMgr.setContentFormat(Format.UNKNOWN); - docMgr.write( - batchWriteSet.getDocumentWriteSet(), batchWriteSet.getTransform(), null, batchWriteSet.getTemporalCollection() - ); - } + logger.trace("Begin write batch {} to forest on host '{}'", batchWriteSet.getBatchNumber(), batchWriteSet.getClient().getHost()); + + DocumentWriteSet documentWriteSet = batchWriteSet.getDocumentWriteSet(); + writeDocuments(documentWriteSet); + + // This seems like it should be part of a finally block - but it's able to throw an exception. Which implies + // that onFailure() should occur when this fails, which seems odd??? closeAllHandles(); - Runnable onSuccess = batchWriteSet.getOnSuccess(); - if (onSuccess != null) { - onSuccess.run(); - } + + onSuccess(); } catch (Throwable t) { - logger.trace("failed batch sent to forest on host \"{}\"", batchWriteSet.getClient().getHost()); - Consumer onFailure = batchWriteSet.getOnFailure(); - if (onFailure != null) { - onFailure.accept(t); - } + onFailure(t); + } + } + + private void writeDocuments(DocumentWriteSet documentWriteSet) { + if (batchWriteSet.getTemporalCollection() == null) { + batchWriteSet.getClient().newDocumentManager().write(documentWriteSet, batchWriteSet.getTransform(), null); + } else { + // to get access to the TemporalDocumentManager write overload we need to instantiate + // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content + // format, so we'll set the default content format to unknown + XMLDocumentManager docMgr = batchWriteSet.getClient().newXMLDocumentManager(); + docMgr.setContentFormat(Format.UNKNOWN); + docMgr.write(documentWriteSet, batchWriteSet.getTransform(), null, batchWriteSet.getTemporalCollection()); + } + } + + private void onSuccess() { + Runnable onSuccess = batchWriteSet.getOnSuccess(); + if (onSuccess != null) { + onSuccess.run(); + } + } + + private void onFailure(Throwable t) { + logger.trace("Failed batch sent to forest on host \"{}\"", batchWriteSet.getClient().getHost()); + Consumer onFailure = batchWriteSet.getOnFailure(); + if (onFailure != null) { + onFailure.accept(t); } } @@ -61,21 +72,17 @@ private void closeAllHandles() throws Throwable { Throwable lastThrowable = null; for (DocumentWriteOperation doc : batchWriteSet.getDocumentWriteSet()) { try { - if (doc.getContent() instanceof Closeable) { - ((Closeable) doc.getContent()).close(); + if (doc.getContent() instanceof Closeable closeable) { + closeable.close(); } - if (doc.getMetadata() instanceof Closeable) { - ((Closeable) doc.getMetadata()).close(); + if (doc.getMetadata() instanceof Closeable closeable) { + closeable.close(); } } catch (Throwable t) { - logger.error("error calling close()", t); + logger.error("Error closing all handles in BatchWriter", t); lastThrowable = t; } } if (lastThrowable != null) throw lastThrowable; } - - public BatchWriteSet getBatchWriteSet() { - return batchWriteSet; - } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java index e7ae80d9c..154068522 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java @@ -3,41 +3,25 @@ */ package com.marklogic.client.datamovement.impl; -import java.util.*; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.datamovement.*; import com.marklogic.client.document.DocumentWriteOperation; -import com.marklogic.client.document.ServerTransform; import com.marklogic.client.document.DocumentWriteOperation.OperationType; -import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.document.ServerTransform; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.impl.Utilities; +import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.marker.AbstractWriteHandle; import com.marklogic.client.io.marker.ContentHandle; import com.marklogic.client.io.marker.DocumentMetadataWriteHandle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import com.marklogic.client.datamovement.DataMovementException; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.Forest; -import com.marklogic.client.datamovement.ForestConfiguration; -import com.marklogic.client.datamovement.JobTicket; -import com.marklogic.client.datamovement.WriteBatch; -import com.marklogic.client.datamovement.WriteBatchListener; -import com.marklogic.client.datamovement.WriteEvent; -import com.marklogic.client.datamovement.WriteFailureListener; -import com.marklogic.client.datamovement.WriteBatcher; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; /** * The implementation of WriteBatcher. @@ -254,19 +238,19 @@ public WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle } private void requireInitialized() { - if ( initialized == false ) { + if (!initialized) { throw new IllegalStateException("This operation must be called after starting this job"); } } private void requireNotInitialized() { - if ( initialized == true ) { + if (initialized) { throw new IllegalStateException("Configuration cannot be changed after starting this job or calling add or addAs"); } } private void requireNotStopped() { - if ( isStopped() == true ) throw new IllegalStateException("This instance has been stopped"); + if (isStopped()) throw new IllegalStateException("This instance has been stopped"); } private BatchWriteSet newBatchWriteSet() { @@ -278,12 +262,8 @@ private BatchWriteSet newBatchWriteSet(long batchNum) { int hostToUse = (int) (batchNum % hostInfos.length); HostInfo host = hostInfos[hostToUse]; BatchWriteSet batchWriteSet = new BatchWriteSet(this, host.client, getTransform(), getTemporalCollection(), batchNum); - batchWriteSet.onSuccess( () -> { - sendSuccessToListeners(batchWriteSet); - }); - batchWriteSet.onFailure( (throwable) -> { - sendThrowableToListeners(throwable, "Error writing batch: {}", batchWriteSet); - }); + batchWriteSet.onSuccess( () -> sendSuccessToListeners(batchWriteSet)); + batchWriteSet.onFailure(throwable -> sendThrowableToListeners(throwable, batchWriteSet)); return batchWriteSet; } @@ -311,7 +291,7 @@ public void retry(WriteBatch batch) { } private void retry(WriteBatch batch, boolean callFailListeners) { - if ( isStopped() == true ) { + if (isStopped()) { logger.warn("Job is now stopped, aborting the retry"); return; } @@ -385,9 +365,9 @@ private void flush(boolean waitForCompletion) { } Iterator iter = docs.iterator(); for ( int i=0; iter.hasNext(); i++ ) { - if ( isStopped() == true ) { + if (isStopped()) { logger.warn("Job is now stopped, preventing the flush of {} queued docs", docs.size() - i); - if ( waitForCompletion == true ) awaitCompletion(); + if (waitForCompletion) awaitCompletion(); return; } BatchWriteSet writeSet = newBatchWriteSet(); @@ -402,7 +382,7 @@ private void flush(boolean waitForCompletion) { threadPool.submit( new BatchWriter(writeSet) ); } - if ( waitForCompletion == true ) awaitCompletion(); + if (waitForCompletion) awaitCompletion(); } private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { @@ -417,7 +397,7 @@ private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { } } - private void sendThrowableToListeners(Throwable t, String message, BatchWriteSet batchWriteSet) { + private void sendThrowableToListeners(Throwable t, BatchWriteSet batchWriteSet) { batchWriteSet.setItemsSoFar(itemsSoFar.get()); WriteBatch batch = batchWriteSet.getBatchOfWriteEvents(); for ( WriteFailureListener failureListener : failureListeners ) { @@ -427,7 +407,7 @@ private void sendThrowableToListeners(Throwable t, String message, BatchWriteSet logger.error("Exception thrown by an onBatchFailure listener", t2); } } - if ( message != null ) logger.warn(message, t.toString()); + logger.warn("Error writing batch: {}", t.toString()); } @Override @@ -606,15 +586,15 @@ public synchronized WriteBatcher withForestConfig(ForestConfiguration forestConf for ( Runnable task : tasks ) { if ( task instanceof BatchWriter ) { BatchWriter writerTask = (BatchWriter) task; - if ( removedHostInfos.containsKey(writerTask.getBatchWriteSet().getClient().getHost()) ) { + if ( removedHostInfos.containsKey(writerTask.batchWriteSet().getClient().getHost()) ) { // this batch was targeting a host that's no longer on the list // if we re-add these docs they'll now be in batches that target acceptable hosts - BatchWriteSet writeSet = newBatchWriteSet(writerTask.getBatchWriteSet().getBatchNumber()); + BatchWriteSet writeSet = newBatchWriteSet(writerTask.batchWriteSet().getBatchNumber()); writeSet.onFailure(throwable -> { if ( throwable instanceof RuntimeException ) throw (RuntimeException) throwable; else throw new DataMovementException("Failed to retry batch after failover", throwable); }); - for ( WriteEvent doc : writerTask.getBatchWriteSet().getBatchOfWriteEvents().getItems() ) { + for ( WriteEvent doc : writerTask.batchWriteSet().getBatchOfWriteEvents().getItems() ) { writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } BatchWriter retryWriterTask = new BatchWriter(writeSet); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java index 020aa21d7..f1e8f726d 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java @@ -5,6 +5,7 @@ import com.marklogic.client.DatabaseClient; import com.marklogic.junit5.AbstractMarkLogicTest; +import org.junit.jupiter.api.AfterEach; /** * Intended to be the base class for all future client API tests, as it properly prepares the database by deleting @@ -25,4 +26,11 @@ protected final String getJavascriptForDeletingDocumentsBeforeTestRuns() { .toArray().forEach(item => xdmp.documentDelete(item)) """; } + + @AfterEach + void releaseClient() { + if (Common.client != null) { + Common.client.release(); + } + } } From 37954b5fc9997f7cae1a8ca211b4073adf8eccda Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 30 Dec 2025 16:21:20 -0500 Subject: [PATCH 18/70] MLE-26420 Refactor: formatted some JSON test files This is so Copilot won't complain in the other open PR. --- .copyrightconfig | 2 +- .../ml-config/databases/content-database.json | 604 +++++----- .../databases/func-content-database.json | 1010 ++++++++--------- .../databases/func-schemas-database.json | 4 +- .../ml-config/databases/modules-database.json | 4 +- .../ml-config/databases/schemas-database.json | 4 +- 6 files changed, 814 insertions(+), 814 deletions(-) diff --git a/.copyrightconfig b/.copyrightconfig index 25e785b7c..1782ce3dc 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml diff --git a/test-app/src/main/ml-config/databases/content-database.json b/test-app/src/main/ml-config/databases/content-database.json index ec7a36f01..99b1c48c3 100644 --- a/test-app/src/main/ml-config/databases/content-database.json +++ b/test-app/src/main/ml-config/databases/content-database.json @@ -1,303 +1,303 @@ { - "database-name": "java-unittest", - "schema-database": "java-unittest-schemas", - "triple-index": true, - "uri-lexicon": true, - "collection-lexicon": true, - "maintain-last-modified": true, - "range-element-index": [ - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "lastModified", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "http://marklogic.com/xdmp/property", - "localname": "last-modified", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "http://nwalsh.com/ns/photolib", - "localname": "tag", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "http://nwalsh.com/ns/photolib", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "double", - "namespace-uri": "", - "localname": "double", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "int", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "grandchild", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "string", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "system-start", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "system-end", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "valid-start", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "valid-end", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "range-element-attribute-index": [ - { - "scalar-type": "date", - "parent-namespace-uri": "http://nwalsh.com/ns/photolib", - "parent-localname": "view", - "namespace-uri": "", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "element-word-lexicon": [ - { - "namespace-uri": "", - "localname": "suggest", - "collation": "http://marklogic.com/collation/" - } - ], - "path-namespace": [ - { - "prefix": "rootOrg", - "namespace-uri": "root.org" - }, - { - "prefix": "targetOrg", - "namespace-uri": "target.org" - } - ], - "range-path-index": [ - { - "scalar-type": "long", - "path-expression": "com.marklogic.client.test.City/population", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.test.City/alternateNames", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.test.Country/continent", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.test.TimeTest/calendarTest", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.test.TimeTest/calendarTestCet", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.test.TimeTest/dateTest", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - } - ], - "field": [ - { - "field-name": "", - "include-root": true - }, - { - "field-name": "int1", - "include-root": false - }, - { - "field-name": "int2", - "include-root": false - } - ], - "range-field-index": [ - { - "scalar-type": "int", - "field-name": "int1", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "field-name": "int2", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-index": [ - { - "namespace-uri": "", - "localname": "latLong", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-child-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "com.marklogic.client.test.City", - "namespace-uri": "", - "localname": "latLong", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-pair-index": [ - { - "parent-namespace-uri": "http://marklogic.com/ns/test/places", - "parent-localname": "place", - "latitude-namespace-uri": "http://marklogic.com/ns/test/places", - "latitude-localname": "lat", - "longitude-namespace-uri": "http://marklogic.com/ns/test/places", - "longitude-localname": "long", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "parent-namespace-uri": "", - "parent-localname": "com.marklogic.client.test.City", - "latitude-namespace-uri": "", - "latitude-localname": "latitude", - "longitude-namespace-uri": "", - "longitude-localname": "longitude", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "parent-namespace-uri": "", - "parent-localname": "point", - "latitude-namespace-uri": "", - "latitude-localname": "lat", - "longitude-namespace-uri": "", - "longitude-localname": "lon", - "coordinate-system": "wgs84/double", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-path-index": [ - { - "path-expression": "com.marklogic.client.test.City/latLong", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "path-expression": "/rootOrg:geo/targetOrg:path", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-region-path-index": [ - { - "path-expression": "/country/region", - "coordinate-system": "wgs84", - "units": "miles", - "geohash-precision": 3, - "invalid-values": "ignore" - }, - { - "path-expression": "/country/region", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 3, - "invalid-values": "ignore" - } - ], - "default-ruleset": [ - { - "location": "rdfs.rules" - } - ] -} \ No newline at end of file + "database-name": "java-unittest", + "schema-database": "java-unittest-schemas", + "triple-index": true, + "uri-lexicon": true, + "collection-lexicon": true, + "maintain-last-modified": true, + "range-element-index": [ + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "lastModified", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "http://marklogic.com/xdmp/property", + "localname": "last-modified", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "http://nwalsh.com/ns/photolib", + "localname": "tag", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "http://nwalsh.com/ns/photolib", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "double", + "namespace-uri": "", + "localname": "double", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "int", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "grandchild", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "string", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "system-start", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "system-end", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "valid-start", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "valid-end", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "range-element-attribute-index": [ + { + "scalar-type": "date", + "parent-namespace-uri": "http://nwalsh.com/ns/photolib", + "parent-localname": "view", + "namespace-uri": "", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "element-word-lexicon": [ + { + "namespace-uri": "", + "localname": "suggest", + "collation": "http://marklogic.com/collation/" + } + ], + "path-namespace": [ + { + "prefix": "rootOrg", + "namespace-uri": "root.org" + }, + { + "prefix": "targetOrg", + "namespace-uri": "target.org" + } + ], + "range-path-index": [ + { + "scalar-type": "long", + "path-expression": "com.marklogic.client.test.City/population", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.test.City/alternateNames", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.test.Country/continent", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.test.TimeTest/calendarTest", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.test.TimeTest/calendarTestCet", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.test.TimeTest/dateTest", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + } + ], + "field": [ + { + "field-name": "", + "include-root": true + }, + { + "field-name": "int1", + "include-root": false + }, + { + "field-name": "int2", + "include-root": false + } + ], + "range-field-index": [ + { + "scalar-type": "int", + "field-name": "int1", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "field-name": "int2", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-index": [ + { + "namespace-uri": "", + "localname": "latLong", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-child-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "com.marklogic.client.test.City", + "namespace-uri": "", + "localname": "latLong", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-pair-index": [ + { + "parent-namespace-uri": "http://marklogic.com/ns/test/places", + "parent-localname": "place", + "latitude-namespace-uri": "http://marklogic.com/ns/test/places", + "latitude-localname": "lat", + "longitude-namespace-uri": "http://marklogic.com/ns/test/places", + "longitude-localname": "long", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "parent-namespace-uri": "", + "parent-localname": "com.marklogic.client.test.City", + "latitude-namespace-uri": "", + "latitude-localname": "latitude", + "longitude-namespace-uri": "", + "longitude-localname": "longitude", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "parent-namespace-uri": "", + "parent-localname": "point", + "latitude-namespace-uri": "", + "latitude-localname": "lat", + "longitude-namespace-uri": "", + "longitude-localname": "lon", + "coordinate-system": "wgs84/double", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-path-index": [ + { + "path-expression": "com.marklogic.client.test.City/latLong", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "path-expression": "/rootOrg:geo/targetOrg:path", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-region-path-index": [ + { + "path-expression": "/country/region", + "coordinate-system": "wgs84", + "units": "miles", + "geohash-precision": 3, + "invalid-values": "ignore" + }, + { + "path-expression": "/country/region", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 3, + "invalid-values": "ignore" + } + ], + "default-ruleset": [ + { + "location": "rdfs.rules" + } + ] +} diff --git a/test-app/src/main/ml-config/databases/func-content-database.json b/test-app/src/main/ml-config/databases/func-content-database.json index 28b4ef38a..20417a0aa 100644 --- a/test-app/src/main/ml-config/databases/func-content-database.json +++ b/test-app/src/main/ml-config/databases/func-content-database.json @@ -1,213 +1,213 @@ { - "database-name": "java-functest", - "schema-database": "java-functest-schemas", - "triple-index": true, - "uri-lexicon": true, - "collection-lexicon": true, - "maintain-last-modified": false, - "trailing-wildcard-searches": true, - "stemmed-searches": "basic", - "range-element-index": [ - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "srchNumber", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "", - "localname": "srchDate", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "srchLevel", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "srchCity", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "http://action/", - "localname": "title", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "http://noun/", - "localname": "title", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "decimal", - "namespace-uri": "http://example.com", - "localname": "rating", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "http://example.com", - "localname": "scoville", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "http://purl.org/dc/elements/1.1/", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "city", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "decimal", - "namespace-uri": "http://test.aggr.com", - "localname": "score", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "http://test.tups.com", - "localname": "rate", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "title", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "popularity", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "double", - "namespace-uri": "", - "localname": "distance", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "bday", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "height1", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "", - "localname": "height2", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "cityName", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "cityTeam", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "long", - "namespace-uri": "", - "localname": "cityPopulation", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "long", - "namespace-uri": "", - "localname": "inventory", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "id", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "animal", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, + "database-name": "java-functest", + "schema-database": "java-functest-schemas", + "triple-index": true, + "uri-lexicon": true, + "collection-lexicon": true, + "maintain-last-modified": false, + "trailing-wildcard-searches": true, + "stemmed-searches": "basic", + "range-element-index": [ + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "srchNumber", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "", + "localname": "srchDate", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "srchLevel", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "srchCity", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "http://action/", + "localname": "title", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "http://noun/", + "localname": "title", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "decimal", + "namespace-uri": "http://example.com", + "localname": "rating", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "http://example.com", + "localname": "scoville", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "http://purl.org/dc/elements/1.1/", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "city", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "decimal", + "namespace-uri": "http://test.aggr.com", + "localname": "score", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "http://test.tups.com", + "localname": "rate", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "title", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "popularity", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "double", + "namespace-uri": "", + "localname": "distance", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "bday", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "height1", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "", + "localname": "height2", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "cityName", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "cityTeam", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "long", + "namespace-uri": "", + "localname": "cityPopulation", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "long", + "namespace-uri": "", + "localname": "inventory", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "id", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "animal", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, { "scalar-type": "int", "namespace-uri": "", @@ -241,300 +241,300 @@ "invalid-values": "reject" } ], - "element-word-lexicon": [ - { - "namespace-uri": "", - "localname": "city", - "collation": "http://marklogic.com/collation/" - } - ], - "geospatial-element-index": [ - { - "namespace-uri": "", - "localname": "latLonPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "range-path-index": [ - { - "scalar-type": "string", - "path-expression": "/Employee/fn", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "int", - "path-expression": "/root/popularity", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "decimal", - "path-expression": "//@amt", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "/doc/name", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.functionaltest.Artifact/name", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.functionaltest.Artifact/manufacturer/com.marklogic.client.functionaltest.Company/name", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "long", - "path-expression": "com.marklogic.client.functionaltest.Artifact/inventory", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.functionaltest.ArtifactIndexedOnCalendar/expiryDate", - "collation": "", - "range-value-positions": true, - "invalid-values": "reject" - } - ], - "range-element-attribute-index": [ - { - "scalar-type": "decimal", - "parent-namespace-uri": "http://cloudbank.com", - "parent-localname": "price", - "namespace-uri": "", - "localname": "amt", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "parent-namespace-uri": "http://example.com", - "parent-localname": "entry", - "namespace-uri": "", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-path-index": [ - { - "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": "false", - "invalid-values": "ignore" - } - ], - "field": [ - { - "field-name": "", - "include-root": true - }, - { - "field-name": "pop", - "include-root": false, - "included-element": [ - { - "namespace-uri": "", - "localname": "popularity", - "weight": 2, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - }, - { - "field-name": "para", - "include-root": false, - "included-element": [ - { - "namespace-uri": "", - "localname": "p", - "weight": 5, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - }, - { - "field-name": "description", - "include-root": true, - "included-element": [ - { - "namespace-uri": "", - "localname": "description", - "weight": 1, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - }, - { - "field-name": "bbqtext", - "include-root": true, - "included-element": [ - { - "namespace-uri": "http://example.com", - "localname": "title", - "weight": 1, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - }, - { - "namespace-uri": "http://example.com", - "localname": "abstract", - "weight": 1, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - } - ], - "geospatial-element-index": [ - { - "namespace-uri": "", - "localname": "latLonPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "namespace-uri": "", - "localname": "g-elem-point", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-child-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "g-elem-child-parent", - "namespace-uri": "", - "localname": "g-elem-child-point", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-pair-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "com.marklogic.client.functionaltest.GeoCompany", - "latitude-namespace-uri": "", - "latitude-localname": "latitude", - "longitude-namespace-uri": "", - "longitude-localname": "longitude", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "parent-namespace-uri": "", - "parent-localname": "g-elem-pair", - "latitude-namespace-uri": "", - "latitude-localname": "lat", - "longitude-namespace-uri": "", - "longitude-localname": "long", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-attribute-pair-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "g-attr-pair", - "latitude-namespace-uri": "", - "latitude-localname": "lat", - "longitude-namespace-uri": "", - "longitude-localname": "long", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-path-index": [ - { - "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "path-expression": "/doc/g-elem-point", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "ignore" - } - ], - "geospatial-region-path-index": [ - { - "path-expression": "/root/item/linestring", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/point", - "coordinate-system": "wgs84", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/circle", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/box", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/polygon", - "coordinate-system": "wgs84", - "units": "miles", - "geohash-precision": 3, - "invalid-values": "reject" - } - ] + "element-word-lexicon": [ + { + "namespace-uri": "", + "localname": "city", + "collation": "http://marklogic.com/collation/" + } + ], + "geospatial-element-index": [ + { + "namespace-uri": "", + "localname": "latLonPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "range-path-index": [ + { + "scalar-type": "string", + "path-expression": "/Employee/fn", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "int", + "path-expression": "/root/popularity", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "decimal", + "path-expression": "//@amt", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "/doc/name", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.functionaltest.Artifact/name", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.functionaltest.Artifact/manufacturer/com.marklogic.client.functionaltest.Company/name", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "long", + "path-expression": "com.marklogic.client.functionaltest.Artifact/inventory", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.functionaltest.ArtifactIndexedOnCalendar/expiryDate", + "collation": "", + "range-value-positions": true, + "invalid-values": "reject" + } + ], + "range-element-attribute-index": [ + { + "scalar-type": "decimal", + "parent-namespace-uri": "http://cloudbank.com", + "parent-localname": "price", + "namespace-uri": "", + "localname": "amt", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "parent-namespace-uri": "http://example.com", + "parent-localname": "entry", + "namespace-uri": "", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-path-index": [ + { + "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": "false", + "invalid-values": "ignore" + } + ], + "field": [ + { + "field-name": "", + "include-root": true + }, + { + "field-name": "pop", + "include-root": false, + "included-element": [ + { + "namespace-uri": "", + "localname": "popularity", + "weight": 2, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + }, + { + "field-name": "para", + "include-root": false, + "included-element": [ + { + "namespace-uri": "", + "localname": "p", + "weight": 5, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + }, + { + "field-name": "description", + "include-root": true, + "included-element": [ + { + "namespace-uri": "", + "localname": "description", + "weight": 1, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + }, + { + "field-name": "bbqtext", + "include-root": true, + "included-element": [ + { + "namespace-uri": "http://example.com", + "localname": "title", + "weight": 1, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + }, + { + "namespace-uri": "http://example.com", + "localname": "abstract", + "weight": 1, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + } + ], + "geospatial-element-index": [ + { + "namespace-uri": "", + "localname": "latLonPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "namespace-uri": "", + "localname": "g-elem-point", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-child-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "g-elem-child-parent", + "namespace-uri": "", + "localname": "g-elem-child-point", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-pair-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "com.marklogic.client.functionaltest.GeoCompany", + "latitude-namespace-uri": "", + "latitude-localname": "latitude", + "longitude-namespace-uri": "", + "longitude-localname": "longitude", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "parent-namespace-uri": "", + "parent-localname": "g-elem-pair", + "latitude-namespace-uri": "", + "latitude-localname": "lat", + "longitude-namespace-uri": "", + "longitude-localname": "long", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-attribute-pair-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "g-attr-pair", + "latitude-namespace-uri": "", + "latitude-localname": "lat", + "longitude-namespace-uri": "", + "longitude-localname": "long", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-path-index": [ + { + "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "path-expression": "/doc/g-elem-point", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "ignore" + } + ], + "geospatial-region-path-index": [ + { + "path-expression": "/root/item/linestring", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/point", + "coordinate-system": "wgs84", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/circle", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/box", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/polygon", + "coordinate-system": "wgs84", + "units": "miles", + "geohash-precision": 3, + "invalid-values": "reject" + } + ] } diff --git a/test-app/src/main/ml-config/databases/func-schemas-database.json b/test-app/src/main/ml-config/databases/func-schemas-database.json index 50c471e93..5a694f552 100644 --- a/test-app/src/main/ml-config/databases/func-schemas-database.json +++ b/test-app/src/main/ml-config/databases/func-schemas-database.json @@ -1,3 +1,3 @@ { - "database-name": "java-functest-schemas" -} \ No newline at end of file + "database-name": "java-functest-schemas" +} diff --git a/test-app/src/main/ml-config/databases/modules-database.json b/test-app/src/main/ml-config/databases/modules-database.json index 35657f4da..3d747b864 100644 --- a/test-app/src/main/ml-config/databases/modules-database.json +++ b/test-app/src/main/ml-config/databases/modules-database.json @@ -1,3 +1,3 @@ { - "database-name": "%%MODULES_DATABASE%%" -} \ No newline at end of file + "database-name": "%%MODULES_DATABASE%%" +} diff --git a/test-app/src/main/ml-config/databases/schemas-database.json b/test-app/src/main/ml-config/databases/schemas-database.json index f0f14c748..582808531 100644 --- a/test-app/src/main/ml-config/databases/schemas-database.json +++ b/test-app/src/main/ml-config/databases/schemas-database.json @@ -1,3 +1,3 @@ { - "database-name": "%%SCHEMAS_DATABASE%%" -} \ No newline at end of file + "database-name": "%%SCHEMAS_DATABASE%%" +} From 1c8dd857ce24d2c777c0adde82478ec5e8d07f92 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 30 Dec 2025 17:01:51 -0500 Subject: [PATCH 19/70] MLE-26420 Added DocumentWriteSetFilter Started making tests under "com.marklogic.client.datamovement" as well so that protected methods can be unit-tested. --- .copyrightconfig | 2 +- CODEOWNERS | 2 +- .../datamovement/DocumentWriteSetFilter.java | 39 +++++++++++ .../client/datamovement/WriteBatcher.java | 13 ++++ .../datamovement/impl/BatchWriteSet.java | 24 ++++++- .../client/datamovement/impl/BatchWriter.java | 13 +++- .../datamovement/impl/WriteBatcherImpl.java | 15 ++-- .../okhttp/RetryIOExceptionInterceptor.java | 5 +- .../WriteNakedPropertiesTest.java | 13 +--- .../filter/RemoveAllDocumentsFilterTest.java | 42 ++++++++++++ .../filter/WriteBatcherTemplate.java | 27 ++++++++ .../client/test/AbstractClientTest.java | 1 + .../datamovement/IncrementalWriteTest.java | 68 ------------------- 13 files changed, 175 insertions(+), 89 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java rename marklogic-client-api/src/test/java/com/marklogic/client/{test => }/datamovement/WriteNakedPropertiesTest.java (80%) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java delete mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java diff --git a/.copyrightconfig b/.copyrightconfig index 1782ce3dc..c87b8a91b 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS index 94edd30a2..4eff9ce7f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,4 +2,4 @@ # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. -* @anu3990 @billfarber @rjrudin @stevebio +* @billfarber @rjrudin @stevebio diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java new file mode 100644 index 000000000..7f00d8161 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.document.DocumentWriteSet; + +import java.util.function.Function; + +/** + * A filter that can modify a DocumentWriteSet before it is written to the database. + * + * @since 8.1.0 + */ +public interface DocumentWriteSetFilter extends Function { + + interface Context { + /** + * @return the DocumentWriteSet to be written + */ + DocumentWriteSet getDocumentWriteSet(); + + /** + * @return the batch number + */ + long getBatchNumber(); + + /** + * @return the DatabaseClient being used for this batch + */ + DatabaseClient getDatabaseClient(); + + /** + * @return the temporal collection name, or null if not writing to a temporal collection + */ + String getTemporalCollection(); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java index 656b029cb..0facc145f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java @@ -357,4 +357,17 @@ WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle, * @param writeBatch the information about the batch that failed */ void retryWithFailureListeners(WriteBatch writeBatch); + + /** + * Sets a filter to modify or replace the DocumentWriteSet before it is written. + * The filter can return either the modified DocumentWriteSet or a new one. + * If the filter returns null or an empty DocumentWriteSet, no write will occur. + * + * @param filter the function to apply before writing + * @return this instance for method chaining + * @since 8.1.0 + */ + default WriteBatcher withDocumentWriteSetFilter(DocumentWriteSetFilter filter) { + return this; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java index 0c08fdd7b..f6e91c91a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java @@ -4,6 +4,7 @@ package com.marklogic.client.datamovement.impl; import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DocumentWriteSetFilter; import com.marklogic.client.datamovement.WriteBatch; import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.datamovement.WriteEvent; @@ -16,15 +17,17 @@ * Mutable class that captures the documents to be written. Documents are added via calls to "getDocumentWriteSet()", where the * DocumentWriteSet is empty when this class is constructed. */ -class BatchWriteSet { +class BatchWriteSet implements DocumentWriteSetFilter.Context { private final WriteBatcher batcher; - private final DocumentWriteSet documentWriteSet; private final long batchNumber; private final DatabaseClient client; private final ServerTransform transform; private final String temporalCollection; + // Can be overridden after creation + private DocumentWriteSet documentWriteSet; + private long itemsSoFar; private Runnable onSuccess; private Consumer onFailure; @@ -38,10 +41,21 @@ class BatchWriteSet { this.batchNumber = batchNumber; } + /** + * Must be called if a DocumentWriteSetFilter modified the DocumentWriteSet owned by this class. + * + * @since 8.1.0 + */ + void updateWithFilteredDocumentWriteSet(DocumentWriteSet filteredDocumentWriteSet) { + this.documentWriteSet = filteredDocumentWriteSet; + } + + @Override public DocumentWriteSet getDocumentWriteSet() { return documentWriteSet; } + @Override public long getBatchNumber() { return batchNumber; } @@ -50,6 +64,11 @@ public void setItemsSoFar(long itemsSoFar) { this.itemsSoFar = itemsSoFar; } + @Override + public DatabaseClient getDatabaseClient() { + return client; + } + public DatabaseClient getClient() { return client; } @@ -58,6 +77,7 @@ public ServerTransform getTransform() { return transform; } + @Override public String getTemporalCollection() { return temporalCollection; } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java index 2173034dd..a2ebe835d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java @@ -3,6 +3,7 @@ */ package com.marklogic.client.datamovement.impl; +import com.marklogic.client.datamovement.DocumentWriteSetFilter; import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.XMLDocumentManager; @@ -13,7 +14,7 @@ import java.io.Closeable; import java.util.function.Consumer; -record BatchWriter(BatchWriteSet batchWriteSet) implements Runnable { +record BatchWriter(BatchWriteSet batchWriteSet, DocumentWriteSetFilter filter) implements Runnable { private static Logger logger = LoggerFactory.getLogger(WriteBatcherImpl.class); @@ -28,6 +29,16 @@ public void run() { logger.trace("Begin write batch {} to forest on host '{}'", batchWriteSet.getBatchNumber(), batchWriteSet.getClient().getHost()); DocumentWriteSet documentWriteSet = batchWriteSet.getDocumentWriteSet(); + if (filter != null) { + documentWriteSet = filter.apply(batchWriteSet); + if (documentWriteSet == null || documentWriteSet.isEmpty()) { + logger.debug("Filter returned empty write set for batch {}, skipping write", batchWriteSet.getBatchNumber()); + closeAllHandles(); + return; + } + batchWriteSet.updateWithFilteredDocumentWriteSet(documentWriteSet); + } + writeDocuments(documentWriteSet); // This seems like it should be part of a finally block - but it's able to throw an exception. Which implies diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java index 154068522..1b376fb85 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java @@ -126,6 +126,7 @@ public class WriteBatcherImpl private boolean initialized = false; private CompletableThreadPoolExecutor threadPool = null; private DocumentMetadataHandle defaultMetadata; + private DocumentWriteSetFilter documentWriteSetFilter; public WriteBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forestConfig) { super(moveMgr); @@ -200,7 +201,7 @@ public WriteBatcher add(DocumentWriteOperation writeOperation) { writeSet.getDocumentWriteSet().add(doc); } if ( writeSet.getDocumentWriteSet().size() > minBatchSize ) { - threadPool.submit( new BatchWriter(writeSet) ); + threadPool.submit( new BatchWriter(writeSet, documentWriteSetFilter) ); } } return this; @@ -308,7 +309,7 @@ private void retry(WriteBatch batch, boolean callFailListeners) { for (WriteEvent doc : batch.getItems()) { writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } - BatchWriter runnable = new BatchWriter(writeSet); + BatchWriter runnable = new BatchWriter(writeSet, documentWriteSetFilter); runnable.run(); } @Override @@ -379,7 +380,7 @@ private void flush(boolean waitForCompletion) { DocumentWriteOperation doc = iter.next(); writeSet.getDocumentWriteSet().add(doc); } - threadPool.submit( new BatchWriter(writeSet) ); + threadPool.submit( new BatchWriter(writeSet, documentWriteSetFilter) ); } if (waitForCompletion) awaitCompletion(); @@ -597,7 +598,7 @@ public synchronized WriteBatcher withForestConfig(ForestConfiguration forestConf for ( WriteEvent doc : writerTask.batchWriteSet().getBatchOfWriteEvents().getItems() ) { writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } - BatchWriter retryWriterTask = new BatchWriter(writeSet); + BatchWriter retryWriterTask = new BatchWriter(writeSet, documentWriteSetFilter); Runnable fretryWriterTask = (Runnable) threadPool.submit(retryWriterTask); threadPool.replaceTask(writerTask, fretryWriterTask); // jump to the next task @@ -846,4 +847,10 @@ public void addAll(Stream operations) { public DocumentMetadataHandle getDocumentMetadata() { return defaultMetadata; } + + @Override + public WriteBatcher withDocumentWriteSetFilter(DocumentWriteSetFilter filter) { + this.documentWriteSetFilter = filter; + return this; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java index 656e399c5..b2f57e0c3 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java @@ -3,6 +3,7 @@ */ package com.marklogic.client.impl.okhttp; +import com.marklogic.client.MarkLogicIOException; import okhttp3.Interceptor; import okhttp3.Request; import okhttp3.Response; @@ -47,7 +48,7 @@ public Response intercept(Chain chain) throws IOException { for (int attempt = 0; attempt <= maxRetries; attempt++) { try { return chain.proceed(request); - } catch (IOException e) { + } catch (MarkLogicIOException | IOException e) { if (attempt == maxRetries || !isRetryableIOException(e)) { logger.warn("Not retryable: {}; {}", e.getClass(), e.getMessage()); throw e; @@ -65,7 +66,7 @@ public Response intercept(Chain chain) throws IOException { throw new IllegalStateException("Unexpected end of retry loop"); } - private boolean isRetryableIOException(IOException e) { + private boolean isRetryableIOException(Exception e) { return e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof UnknownHostException || diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteNakedPropertiesTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/WriteNakedPropertiesTest.java similarity index 80% rename from marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteNakedPropertiesTest.java rename to marklogic-client-api/src/test/java/com/marklogic/client/datamovement/WriteNakedPropertiesTest.java index e97f87158..fb7b58d89 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteNakedPropertiesTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/WriteNakedPropertiesTest.java @@ -1,14 +1,12 @@ /* * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -package com.marklogic.client.test.datamovement; +package com.marklogic.client.datamovement; import com.marklogic.client.DatabaseClient; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import javax.xml.namespace.QName; @@ -16,12 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class WriteNakedPropertiesTest { - - @BeforeEach - void setup() { - Common.newRestAdminClient().newXMLDocumentManager().delete("/naked.xml"); - } +class WriteNakedPropertiesTest extends AbstractClientTest { @Test void test() { diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java new file mode 100644 index 000000000..40a78b817 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RemoveAllDocumentsFilterTest extends AbstractClientTest { + + private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + AtomicInteger writtenCount = new AtomicInteger(); + + @Test + void filterRemovesAllDocuments() { + new WriteBatcherTemplate(Common.newClient()).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(context -> context.getDatabaseClient().newDocumentManager().newWriteSet()) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), + + writeBatcher -> { + for (int i = 1; i <= 10; i++) { + writeBatcher.add("/incremental/test/doc-" + i + ".xml", METADATA, new StringHandle("")); + } + } + ); + + assertEquals(0, writtenCount.get(), "No documents should have been written since the filter removed them all. " + + "This test is verifying that no error will occur either when the filter doesn't return any documents."); + assertCollectionSize("incremental-test", 0); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java new file mode 100644 index 000000000..62e066949 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.WriteBatcher; + +import java.util.function.Consumer; + +// Experimenting with a template that gets rid of some annoying DMSDK boilerplate. +record WriteBatcherTemplate(DatabaseClient databaseClient) { + + public void runWriteJob(Consumer writeBatcherConfigurer, Consumer writeBatcherUser) { + try (DataMovementManager dmm = databaseClient.newDataMovementManager()) { + WriteBatcher writeBatcher = dmm.newWriteBatcher(); + writeBatcherConfigurer.accept(writeBatcher); + + dmm.startJob(writeBatcher); + writeBatcherUser.accept(writeBatcher); + writeBatcher.flushAndWait(); + writeBatcher.awaitCompletion(); + dmm.stopJob(writeBatcher); + } + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java index f1e8f726d..0f9c5af8a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java @@ -31,6 +31,7 @@ protected final String getJavascriptForDeletingDocumentsBeforeTestRuns() { void releaseClient() { if (Common.client != null) { Common.client.release(); + Common.client = null; } } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java deleted file mode 100644 index 83a81e1c5..000000000 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/IncrementalWriteTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. - */ -package com.marklogic.client.test.datamovement; - -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.WriteBatcher; -import com.marklogic.client.io.DocumentMetadataHandle; -import com.marklogic.client.io.StringHandle; -import com.marklogic.client.test.AbstractClientTest; -import com.marklogic.client.test.Common; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class IncrementalWriteTest extends AbstractClientTest { - - private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() - .withCollections("incremental-test") - .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); - - @Test - void test() { - AtomicInteger writtenCount = new AtomicInteger(); - - try (DatabaseClient client = Common.newClient()) { - WriteBatcherTemplate template = new WriteBatcherTemplate(client); - - template.runWriteJob(writeBatcher -> writeBatcher - .withThreadCount(1) - .withBatchSize(10) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), - - writeBatcher -> { - for (int i = 1; i <= 20; i++) { - String uri = "/incremental/test/doc-" + i + ".xml"; - String content = "" + i + "This is document number " + i + ""; - writeBatcher.add(uri, METADATA, new StringHandle(content)); - } - } - ); - } - - assertEquals(20, writtenCount.get()); - } - - // Experimenting with a template that gets rid of some annoying DMSDK boilerplate. - private record WriteBatcherTemplate(DatabaseClient databaseClient) { - - public void runWriteJob(Consumer writeBatcherConfigurer, Consumer writeBatcherUser) { - try (DataMovementManager dmm = databaseClient.newDataMovementManager()) { - WriteBatcher writeBatcher = dmm.newWriteBatcher(); - writeBatcherConfigurer.accept(writeBatcher); - - dmm.startJob(writeBatcher); - - writeBatcherUser.accept(writeBatcher); - writeBatcher.awaitCompletion(); - - dmm.stopJob(writeBatcher); - } - } - } -} From f6cfa96d8170ddf0a1d249f6f422fcc5147af97a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 10 Dec 2025 15:19:38 -0500 Subject: [PATCH 20/70] MLE-26420 Can now perform incremental writes IncrementalWriteFilter is the entry point, with a Builder for customizing its behavior. --- marklogic-client-api/build.gradle | 5 + .../filter/IncrementalWriteEvalFilter.java | 57 +++++ .../filter/IncrementalWriteFilter.java | 187 +++++++++++++++ .../filter/IncrementalWriteOpticFilter.java | 55 +++++ .../filter/IncrementalWriteFilterTest.java | 48 ++++ .../filter/IncrementalWriteTest.java | 226 ++++++++++++++++++ .../ml-config/databases/content-database.json | 16 ++ 7 files changed, 594 insertions(+) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 23b984883..b292e3c79 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -37,6 +37,11 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:${jacksonVersion}" + // Dependencies for hash generation. Can be safely omitted if not using the incremental write feature. But neither + // has any transitive dependencies, and thus their impact on the dependency tree is minimal. + implementation "io.github.erdtman:java-json-canonicalization:1.1" + implementation "net.openhft:zero-allocation-hashing:0.27ea1" + // Only used by extras (which some examples then depend on) compileOnly 'org.jdom:jdom2:2.0.6.1' compileOnly 'org.dom4j:dom4j:2.2.0' diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java new file mode 100644 index 000000000..50257bb14 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.marklogic.client.datamovement.DocumentWriteSetFilter; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.io.JacksonHandle; + +import java.util.function.Consumer; + +/** + * Uses server-side JavaScript code to get the existing hash values for a set of URIs. + * + * @since 8.1.0 + */ +class IncrementalWriteEvalFilter extends IncrementalWriteFilter { + + private static final String EVAL_SCRIPT = """ + const tuples = cts.valueTuples([cts.uriReference(), cts.fieldReference(fieldName)], null, cts.documentQuery(uris)); + const response = {}; + for (var tuple of tuples) { + response[tuple[0]] = tuple[1]; + } + response + """; + + IncrementalWriteEvalFilter(String fieldName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { + super(fieldName, canonicalizeJson, skippedDocumentsConsumer); + } + + @Override + public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { + ArrayNode uris = new ObjectMapper().createArrayNode(); + for (DocumentWriteOperation doc : context.getDocumentWriteSet()) { + if (DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(doc.getOperationType())) { + uris.add(doc.getUri()); + } + } + + JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) + .addVariable("fieldName", fieldName) + .addVariable("uris", new JacksonHandle(uris)) + .evalAs(JsonNode.class); + + return filterDocuments(context, uri -> { + if (response.has(uri)) { + return response.get(uri).asText(); + } + return null; + }); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java new file mode 100644 index 000000000..03c1c465a --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.datamovement.DocumentWriteSetFilter; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.impl.HandleAccessor; +import com.marklogic.client.io.BaseHandle; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; +import net.openhft.hashing.LongHashFunction; +import org.erdtman.jcs.JsonCanonicalizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A DocumentWriteSetFilter that skips writing documents whose content has not changed since the last write + * based on a hash value stored in a MarkLogic field. + * + * @since 8.1.0 + */ +public abstract class IncrementalWriteFilter implements DocumentWriteSetFilter { + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + + private String fieldName = "incrementalWriteHash"; + private boolean canonicalizeJson = true; + private boolean useEvalQuery = false; + private Consumer skippedDocumentsConsumer; + + /** + * @param fieldName the name of the MarkLogic field that will hold the hash value; defaults to "incrementalWriteHash". + */ + public Builder fieldName(String fieldName) { + this.fieldName = fieldName; + return this; + } + + /** + * @param canonicalizeJson whether to canonicalize JSON content before hashing; defaults to true. + * Delegates to https://github.com/erdtman/java-json-canonicalization for canonicalization. + */ + public Builder canonicalizeJson(boolean canonicalizeJson) { + this.canonicalizeJson = canonicalizeJson; + return this; + } + + /** + * @param useEvalQuery if true, evaluate server-side JavaScript instead of an Optic query for retrieving hash values; defaults to false. + */ + public Builder useEvalQuery(boolean useEvalQuery) { + this.useEvalQuery = useEvalQuery; + return this; + } + + /** + * @param skippedDocumentsConsumer a consumer that will be called with any documents in a batch that were skipped because their content had not changed. + */ + public Builder onDocumentsSkipped(Consumer skippedDocumentsConsumer) { + this.skippedDocumentsConsumer = skippedDocumentsConsumer; + return this; + } + + public IncrementalWriteFilter build() { + if (useEvalQuery) { + return new IncrementalWriteEvalFilter(fieldName, canonicalizeJson, skippedDocumentsConsumer); + } + return new IncrementalWriteOpticFilter(fieldName, canonicalizeJson, skippedDocumentsConsumer); + } + } + + protected final String fieldName; + private final boolean canonicalizeJson; + private final Consumer skippedDocumentsConsumer; + + // Hardcoding this for now, with a good general purpose hashing function. + // See https://xxhash.com for benchmarks. + private final LongHashFunction hashFunction = LongHashFunction.xx3(); + + public IncrementalWriteFilter(String fieldName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { + this.fieldName = fieldName; + this.canonicalizeJson = canonicalizeJson; + this.skippedDocumentsConsumer = skippedDocumentsConsumer; + } + + protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { + final DocumentWriteSet newWriteSet = context.getDatabaseClient().newDocumentManager().newWriteSet(); + final List skippedDocuments = new ArrayList<>(); + + for (DocumentWriteOperation doc : context.getDocumentWriteSet()) { + if (!DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(doc.getOperationType())) { + newWriteSet.add(doc); + continue; + } + + final String contentHash = serializeContent(doc); + final String existingHash = hashRetriever.apply(doc.getUri()); + if (logger.isTraceEnabled()) { + logger.trace("URI: {}, existing Hash: {}, new Hash: {}", doc.getUri(), existingHash, contentHash); + } + + if (existingHash != null) { + if (!existingHash.equals(contentHash)) { + newWriteSet.add(addHashToMetadata(doc, fieldName, contentHash)); + } else if (skippedDocumentsConsumer != null) { + skippedDocuments.add(doc); + } else { + // No consumer, so skip the document silently. + } + } else { + newWriteSet.add(addHashToMetadata(doc, fieldName, contentHash)); + } + } + + if (!skippedDocuments.isEmpty()) { + skippedDocumentsConsumer.accept(skippedDocuments.toArray(new DocumentWriteOperation[0])); + } + + return newWriteSet; + } + + private String serializeContent(DocumentWriteOperation doc) { + String content = HandleAccessor.contentAsString(doc.getContent()); + + Format format = null; + if (doc.getContent() instanceof BaseHandle baseHandle) { + format = baseHandle.getFormat(); + } + + if (canonicalizeJson && (Format.JSON.equals(format) || isPossiblyJsonContent(content))) { + JsonCanonicalizer jc; + try { + jc = new JsonCanonicalizer(content); + return jc.getEncodedString(); + } catch (IOException e) { + // Going to improve this in the next PR, as I think we can throw an exception if Format = JSON. + logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", + doc.getUri(), e.getMessage()); + } + } + + return content; + } + + private boolean isPossiblyJsonContent(String content) { + // This isn't 100% reliable, as the content could be text that just happens to start with { or [, and so + // we'll still need to catch an exception if we try to canonicalize non-JSON content. + String trimmed = content.trim(); + return trimmed.startsWith("{") || trimmed.startsWith("["); + } + + private String computeHash(String content) { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + long hash = hashFunction.hashBytes(bytes); + return Long.toHexString(hash); + } + + protected static DocumentWriteOperation addHashToMetadata(DocumentWriteOperation op, String fieldName, String hash) { + DocumentMetadataHandle newMetadata = new DocumentMetadataHandle(); + if (op.getMetadata() != null) { + DocumentMetadataHandle originalMetadata = (DocumentMetadataHandle) op.getMetadata(); + newMetadata.setPermissions(originalMetadata.getPermissions()); + newMetadata.setCollections(originalMetadata.getCollections()); + newMetadata.setQuality(originalMetadata.getQuality()); + newMetadata.setProperties(originalMetadata.getProperties()); + newMetadata.getMetadataValues().putAll(originalMetadata.getMetadataValues()); + } + newMetadata.getMetadataValues().put(fieldName, hash); + return new DocumentWriteOperationImpl(op.getUri(), newMetadata, op.getContent(), op.getTemporalDocumentURI()); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java new file mode 100644 index 000000000..592186eb1 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.row.RowTemplate; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Uses an Optic query to get the existing hash values for a set of URIs. + * + * @since 8.1.0 + */ +class IncrementalWriteOpticFilter extends IncrementalWriteFilter { + + IncrementalWriteOpticFilter(String fieldName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { + super(fieldName, canonicalizeJson, skippedDocumentsConsumer); + } + + @Override + public DocumentWriteSet apply(Context context) { + final String[] uris = context.getDocumentWriteSet().stream() + .filter(op -> DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(op.getOperationType())) + .map(DocumentWriteOperation::getUri) + .toArray(String[]::new); + + // It doesn't seem possible yet to use a DSL query and bind an array of strings to a "uris" param, so using + // a serialized query instead. That doesn't allow a user to override the query though. + Map existingHashes = new RowTemplate(context.getDatabaseClient()).query(op -> + op.fromLexicons(Map.of( + "uri", op.cts.uriReference(), + "hash", op.cts.fieldReference(super.fieldName) + )).where( + op.cts.documentQuery(op.xs.stringSeq(uris)) + ), + + rows -> { + Map map = new HashMap<>(); + rows.forEach(row -> { + String uri = row.getString("uri"); + String existingHash = row.getString("hash"); + map.put(uri, existingHash); + }); + return map; + } + ); + + return filterDocuments(context, uri -> existingHashes.get(uri)); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java new file mode 100644 index 000000000..f1fe81518 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Unit tests that make no connection to MarkLogic. + */ +class IncrementalWriteFilterTest { + + /** + * Verifies that when a hash is added, a new metadata object is created so that a doc-specific hash field can be + * added without affecting any other document that might be sharing the same metadata object. + */ + @Test + void addHashToMetadata() { + DocumentMetadataHandle metadata = new DocumentMetadataHandle() + .withCollections("c1") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ) + .withQuality(2) + .withProperty("prop1", "value1") + .withMetadataValue("meta1", "value1"); + + DocumentWriteOperation doc1 = new DocumentWriteOperationImpl("/1.xml", metadata, new StringHandle("")); + DocumentWriteOperation doc2 = new DocumentWriteOperationImpl("/2.xml", metadata, new StringHandle("")); + + doc2 = IncrementalWriteFilter.addHashToMetadata(doc2, "theField", "abc123"); + + assertEquals(metadata, doc1.getMetadata(), "doc1 should still have the original metadata object"); + + DocumentMetadataHandle metadata2 = (DocumentMetadataHandle) doc2.getMetadata(); + assertEquals("c1", metadata2.getCollections().iterator().next(), "collection should be preserved"); + assertEquals(DocumentMetadataHandle.Capability.READ, metadata2.getPermissions().get("rest-reader").iterator().next(), "permission should be preserved"); + assertEquals(2, metadata2.getQuality(), "quality should be preserved"); + assertEquals("value1", metadata2.getProperties().get("prop1"), "property should be preserved"); + + assertEquals("value1", metadata2.getMetadataValues().get("meta1"), "metadata value should be preserved"); + assertEquals("abc123", metadata2.getMetadataValues().get("theField"), "hash field should be added"); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java new file mode 100644 index 000000000..89c1417ee --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.WriteBatcher; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class IncrementalWriteTest extends AbstractClientTest { + + private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + AtomicInteger writtenCount = new AtomicInteger(); + AtomicInteger skippedCount = new AtomicInteger(); + ObjectMapper objectMapper = new ObjectMapper(); + + IncrementalWriteFilter filter; + + @BeforeEach + void setup() { + // Need a user with eval privileges so that the eval filter can be tested. + Common.client = Common.newEvalClient(); + } + + @Test + void opticFilter() { + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + runTest(); + } + + @Test + void evalFilter() { + filter = IncrementalWriteFilter.newBuilder() + .useEvalQuery(true) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + runTest(); + } + + @Test + void filterRemovesAllDocuments() { + new WriteBatcherTemplate(Common.client).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(context -> context.getDatabaseClient().newDocumentManager().newWriteSet()) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), + + writeBatcher -> { + for (int i = 1; i <= 10; i++) { + writeBatcher.add("/incremental/test/doc-" + i + ".xml", METADATA, new StringHandle("")); + } + } + ); + + assertEquals(0, writtenCount.get(), "No documents should have been written since the filter removed them all. " + + "This test is verifying that no error will occur either when the filter doesn't return any documents."); + assertCollectionSize("incremental-test", 0); + } + + @Test + void jsonKeysOutOfOrder() { + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + List docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("number", i); + doc.put("text", "hello"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("text", "hello"); + doc.put("number", i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get()); + assertEquals(10, skippedCount.get(), "Since JSON canonicalization is enabled by default, the documents " + + "should be recognized as unchanged even though their keys are in a different order."); + } + + @Test + void jsonKeysOutOfOrderWithNoCanonicalization() { + filter = IncrementalWriteFilter.newBuilder() + .canonicalizeJson(false) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + List docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("number", i); + doc.put("text", "hello"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("text", "hello"); + doc.put("number", i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(20, writtenCount.get(), "Since JSON canonicalization is disabled, all documents should be " + + "written again since their keys are in a different order."); + assertEquals(0, skippedCount.get()); + } + + private void runTest() { + writeTenDocuments(); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get(), "No docs should have been skipped on the first write."); + + writeTenDocuments(); + assertEquals(10, skippedCount.get(), "All docs should have been skipped since their content hasn't changed."); + assertEquals(10, writtenCount.get(), "The count of written should still be 10 since all docs should have been skipped on the second write."); + + modifyFiveDocuments(); + assertEquals(10, skippedCount.get()); + assertEquals(15, writtenCount.get(), "5 documents should have been modified, with their hashes being updated."); + + writeTenDocuments(); + assertEquals(15, skippedCount.get(), "The 5 unmodified documents should have been skipped."); + assertEquals(20, writtenCount.get(), "The 5 modified documents should have been overwritten since their content changed."); + } + + private void writeTenDocuments() { + new WriteBatcherTemplate(Common.client).runWriteJob(writeBatcher -> writeBatcher + .withThreadCount(1).withBatchSize(5) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) + .withDocumentWriteSetFilter(filter), + + writeBatcher -> { + for (int i = 1; i <= 10; i++) { + // Consistent URIs are required for incremental writes to work. + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "This is document number " + i + ""; + writeBatcher.add(uri, METADATA, new StringHandle(content)); + } + } + ); + } + + private void modifyFiveDocuments() { + new WriteBatcherTemplate(Common.client).runWriteJob(writeBatcher -> writeBatcher + .withThreadCount(1).withBatchSize(5) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) + .withDocumentWriteSetFilter(filter), + + writeBatcher -> { + for (int i = 6; i <= 10; i++) { + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "This is modified content"; + writeBatcher.add(uri, METADATA, new StringHandle(content)); + } + } + ); + } + + private void writeDocs(List docs) { + new WriteBatcherTemplate(Common.client).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(filter) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), + + writeBatcher -> docs.forEach(writeBatcher::add) + ); + } + + // Experimenting with a template that gets rid of some annoying DMSDK boilerplate. + private record WriteBatcherTemplate(DatabaseClient databaseClient) { + + public void runWriteJob(Consumer writeBatcherConfigurer, Consumer writeBatcherUser) { + try (DataMovementManager dmm = databaseClient.newDataMovementManager()) { + WriteBatcher writeBatcher = dmm.newWriteBatcher(); + writeBatcherConfigurer.accept(writeBatcher); + + dmm.startJob(writeBatcher); + writeBatcherUser.accept(writeBatcher); + writeBatcher.flushAndWait(); + writeBatcher.awaitCompletion(); + dmm.stopJob(writeBatcher); + } + } + } +} diff --git a/test-app/src/main/ml-config/databases/content-database.json b/test-app/src/main/ml-config/databases/content-database.json index 99b1c48c3..4e869464f 100644 --- a/test-app/src/main/ml-config/databases/content-database.json +++ b/test-app/src/main/ml-config/databases/content-database.json @@ -188,6 +188,15 @@ { "field-name": "int2", "include-root": false + }, + { + "field-name": "incrementalWriteHash", + "metadata": "", + "stemmed-searches": "off", + "word-searches": false, + "fast-phrase-searches": false, + "fast-case-sensitive-searches": false, + "fast-diacritic-sensitive-searches": false } ], "range-field-index": [ @@ -204,6 +213,13 @@ "collation": "", "range-value-positions": false, "invalid-values": "reject" + }, + { + "scalar-type": "string", + "collation": "http://marklogic.com/collation/", + "field-name": "incrementalWriteHash", + "range-value-positions": false, + "invalid-values": "reject" } ], "geospatial-element-index": [ From 0e6f0aec7a06dc81981d6d7f1b8c4f3a2526963d Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 31 Dec 2025 11:30:15 -0500 Subject: [PATCH 21/70] MLE-26420 Improved error handling for incremental writes --- .../filter/IncrementalWriteFilter.java | 4 +- .../filter/IncrementalWriteTest.java | 94 ++++++++----------- 2 files changed, 44 insertions(+), 54 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 03c1c465a..18dc50387 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -149,7 +149,9 @@ private String serializeContent(DocumentWriteOperation doc) { jc = new JsonCanonicalizer(content); return jc.getEncodedString(); } catch (IOException e) { - // Going to improve this in the next PR, as I think we can throw an exception if Format = JSON. + // If the Format is actually JSON, then the write to MarkLogic should ultimately fail, which is the + // error message the user would want to see via a batch failure listener. So in all cases, if we cannot + // canonicalize something that appears to be JSON, we log a warning and return the original content for hashing. logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 89c1417ee..438784f8e 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -5,12 +5,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; import com.marklogic.client.io.JacksonHandle; import com.marklogic.client.io.StringHandle; import com.marklogic.client.test.AbstractClientTest; @@ -21,9 +19,9 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; +import java.util.concurrent.atomic.AtomicReference; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; class IncrementalWriteTest extends AbstractClientTest { @@ -33,23 +31,26 @@ class IncrementalWriteTest extends AbstractClientTest { AtomicInteger writtenCount = new AtomicInteger(); AtomicInteger skippedCount = new AtomicInteger(); + AtomicReference batchFailure = new AtomicReference<>(); ObjectMapper objectMapper = new ObjectMapper(); + List docs = new ArrayList<>(); IncrementalWriteFilter filter; @BeforeEach void setup() { // Need a user with eval privileges so that the eval filter can be tested. Common.client = Common.newEvalClient(); - } - @Test - void opticFilter() { + // Default filter implementation, should be suitable for most tests. filter = IncrementalWriteFilter.newBuilder() .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) .build(); + } - runTest(); + @Test + void opticFilter() { + verifyIncrementalWriteWorks(); } @Test @@ -59,35 +60,11 @@ void evalFilter() { .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) .build(); - runTest(); - } - - @Test - void filterRemovesAllDocuments() { - new WriteBatcherTemplate(Common.client).runWriteJob( - writeBatcher -> writeBatcher - .withDocumentWriteSetFilter(context -> context.getDatabaseClient().newDocumentManager().newWriteSet()) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), - - writeBatcher -> { - for (int i = 1; i <= 10; i++) { - writeBatcher.add("/incremental/test/doc-" + i + ".xml", METADATA, new StringHandle("")); - } - } - ); - - assertEquals(0, writtenCount.get(), "No documents should have been written since the filter removed them all. " + - "This test is verifying that no error will occur either when the filter doesn't return any documents."); - assertCollectionSize("incremental-test", 0); + verifyIncrementalWriteWorks(); } @Test void jsonKeysOutOfOrder() { - filter = IncrementalWriteFilter.newBuilder() - .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) - .build(); - - List docs = new ArrayList<>(); for (int i = 1; i <= 10; i++) { ObjectNode doc = objectMapper.createObjectNode(); doc.put("number", i); @@ -146,7 +123,34 @@ void jsonKeysOutOfOrderWithNoCanonicalization() { assertEquals(0, skippedCount.get()); } - private void runTest() { + @Test + void invalidJsonWithNoFormat() { + docs.add(new DocumentWriteOperationImpl("/not-json.txt", METADATA, new StringHandle("{\"not actually json"))); + writeDocs(docs); + + assertEquals(1, writtenCount.get(), "When the format is not specified and the content looks like JSON " + + "because it starts with a '{', the JSON canonicalization should fail and log a warning. The " + + "document should still be written with a hash generated based on the text in the document."); + + assertNull(batchFailure.get(), "No failure should have been thrown since the format on the content is " + + "not JSON, and thus the content should be hashed as text."); + } + + @Test + void invalidJsonWithFormat() { + docs.add(new DocumentWriteOperationImpl("/not.json", METADATA, new StringHandle("not actually json").withFormat(Format.JSON))); + writeDocs(docs); + + assertNotNull(batchFailure.get(), "A failure should have been thrown by the server since the content is not " + + "JSON. But the failure to canonicalize should still be logged, as the user will be far more interested " + + "in the error from the server."); + + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("failed to apply resource at documents"), + "Expecting the server to throw an error. Actual message: " + message); + } + + private void verifyIncrementalWriteWorks() { writeTenDocuments(); assertEquals(10, writtenCount.get()); assertEquals(0, skippedCount.get(), "No docs should have been skipped on the first write."); @@ -201,26 +205,10 @@ private void writeDocs(List docs) { new WriteBatcherTemplate(Common.client).runWriteJob( writeBatcher -> writeBatcher .withDocumentWriteSetFilter(filter) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) + .onBatchFailure((batch, failure) -> batchFailure.set(failure)), writeBatcher -> docs.forEach(writeBatcher::add) ); } - - // Experimenting with a template that gets rid of some annoying DMSDK boilerplate. - private record WriteBatcherTemplate(DatabaseClient databaseClient) { - - public void runWriteJob(Consumer writeBatcherConfigurer, Consumer writeBatcherUser) { - try (DataMovementManager dmm = databaseClient.newDataMovementManager()) { - WriteBatcher writeBatcher = dmm.newWriteBatcher(); - writeBatcherConfigurer.accept(writeBatcher); - - dmm.startJob(writeBatcher); - writeBatcherUser.accept(writeBatcher); - writeBatcher.flushAndWait(); - writeBatcher.awaitCompletion(); - dmm.stopJob(writeBatcher); - } - } - } } From 57a023c6a2e53106b74d78013ea843f2e76c17d3 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 31 Dec 2025 14:27:05 -0500 Subject: [PATCH 22/70] MLE-26420 Nicer error when missing index for incremental write --- .../filter/IncrementalWriteEvalFilter.java | 28 +++--- .../filter/IncrementalWriteFilter.java | 2 +- .../filter/IncrementalWriteOpticFilter.java | 48 ++++++---- .../filter/IncrementalWriteTest.java | 94 +++++++++++++------ 4 files changed, 113 insertions(+), 59 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index 50257bb14..46aba85c1 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.marklogic.client.FailedRequestException; import com.marklogic.client.datamovement.DocumentWriteSetFilter; import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.DocumentWriteSet; @@ -42,16 +43,21 @@ public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { } } - JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) - .addVariable("fieldName", fieldName) - .addVariable("uris", new JacksonHandle(uris)) - .evalAs(JsonNode.class); - - return filterDocuments(context, uri -> { - if (response.has(uri)) { - return response.get(uri).asText(); - } - return null; - }); + try { + JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) + .addVariable("fieldName", fieldName) + .addVariable("uris", new JacksonHandle(uris)) + .evalAs(JsonNode.class); + + return filterDocuments(context, uri -> { + if (response.has(uri)) { + return response.get(uri).asText(); + } + return null; + }); + } catch (FailedRequestException e) { + String message = "Unable to query for existing incremental write hashes; cause: " + e.getMessage(); + throw new FailedRequestException(message, e.getFailedRequest()); + } } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 18dc50387..8fe27d05d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -109,7 +109,7 @@ protected final DocumentWriteSet filterDocuments(Context context, Function existingHashes = new RowTemplate(context.getDatabaseClient()).query(op -> - op.fromLexicons(Map.of( - "uri", op.cts.uriReference(), - "hash", op.cts.fieldReference(super.fieldName) - )).where( - op.cts.documentQuery(op.xs.stringSeq(uris)) - ), - - rows -> { - Map map = new HashMap<>(); - rows.forEach(row -> { - String uri = row.getString("uri"); - String existingHash = row.getString("hash"); - map.put(uri, existingHash); - }); - return map; - } - ); - - return filterDocuments(context, uri -> existingHashes.get(uri)); + RowTemplate rowTemplate = new RowTemplate(context.getDatabaseClient()); + + try { + Map existingHashes = rowTemplate.query(op -> + op.fromLexicons(Map.of( + "uri", op.cts.uriReference(), + "hash", op.cts.fieldReference(super.fieldName) + )).where( + op.cts.documentQuery(op.xs.stringSeq(uris)) + ), + + rows -> { + Map map = new HashMap<>(); + rows.forEach(row -> { + String uri = row.getString("uri"); + String existingHash = row.getString("hash"); + map.put(uri, existingHash); + }); + return map; + } + ); + + return filterDocuments(context, uri -> existingHashes.get(uri)); + } catch (FailedRequestException e) { + String message = "Unable to query for existing incremental write hashes; cause: " + e.getMessage(); + throw new FailedRequestException(message, e.getFailedRequest()); + } } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 438784f8e..388473553 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.*; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; @@ -150,8 +150,40 @@ void invalidJsonWithFormat() { "Expecting the server to throw an error. Actual message: " + message); } + @Test + void noRangeIndexForField() { + filter = IncrementalWriteFilter.newBuilder() + .fieldName("non-existent-field") + .build(); + + writeTenDocuments(); + + assertNotNull(batchFailure.get()); + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("XDMP-FIELDRIDXNOTFOUND"), + "When the user tries to use the incremental write feature without the required range index, we should " + + "fail with a helpful error message. Actual message: " + message); + } + + @Test + void noRangeIndexForFieldWithEval() { + filter = IncrementalWriteFilter.newBuilder() + .fieldName("non-existent-field") + .useEvalQuery(true) + .build(); + + writeTenDocuments(); + + assertNotNull(batchFailure.get()); + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("XDMP-FIELDRIDXNOTFOUND"), + "When the user tries to use the incremental write feature without the required range index, we should " + + "fail with a helpful error message. Actual message: " + message); + } + private void verifyIncrementalWriteWorks() { writeTenDocuments(); + verifyDocumentsHasHashInMetadataKey(); assertEquals(10, writtenCount.get()); assertEquals(0, skippedCount.get(), "No docs should have been skipped on the first write."); @@ -169,36 +201,44 @@ private void verifyIncrementalWriteWorks() { } private void writeTenDocuments() { - new WriteBatcherTemplate(Common.client).runWriteJob(writeBatcher -> writeBatcher - .withThreadCount(1).withBatchSize(5) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) - .withDocumentWriteSetFilter(filter), - - writeBatcher -> { - for (int i = 1; i <= 10; i++) { - // Consistent URIs are required for incremental writes to work. - String uri = "/incremental/test/doc-" + i + ".xml"; - String content = "This is document number " + i + ""; - writeBatcher.add(uri, METADATA, new StringHandle(content)); - } + docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + // Consistent URIs are required for incremental writes to work. + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "This is document number " + i + ""; + docs.add(new DocumentWriteOperationImpl(uri, METADATA, new StringHandle(content))); + } + writeDocs(docs); + } + + private void verifyDocumentsHasHashInMetadataKey() { + GenericDocumentManager mgr = Common.client.newDocumentManager(); + mgr.setMetadataCategories(DocumentManager.Metadata.METADATAVALUES); + DocumentPage page = mgr.search(Common.client.newQueryManager().newStructuredQueryBuilder().collection("incremental-test"), 1); + while (page.hasNext()) { + DocumentRecord doc = page.next(); + DocumentMetadataHandle metadata = doc.getMetadata(new DocumentMetadataHandle()); + assertTrue(metadata.getMetadataValues().containsKey("incrementalWriteHash"), + "Document " + doc.getUri() + " should have an incrementalWriteHash in its metadata values."); + + String hash = metadata.getMetadataValues().get("incrementalWriteHash"); + try { + // Can use Java's support for parsing unsigned longs in base 16 to verify the hash is valid. + Long.parseUnsignedLong(hash, 16); + } catch (NumberFormatException e) { + fail("Document " + doc.getUri() + " has an invalid incrementalWriteHash value: " + hash); } - ); + } } private void modifyFiveDocuments() { - new WriteBatcherTemplate(Common.client).runWriteJob(writeBatcher -> writeBatcher - .withThreadCount(1).withBatchSize(5) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) - .withDocumentWriteSetFilter(filter), - - writeBatcher -> { - for (int i = 6; i <= 10; i++) { - String uri = "/incremental/test/doc-" + i + ".xml"; - String content = "This is modified content"; - writeBatcher.add(uri, METADATA, new StringHandle(content)); - } - } - ); + docs = new ArrayList<>(); + for (int i = 6; i <= 10; i++) { + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "This is modified content"; + docs.add(new DocumentWriteOperationImpl(uri, METADATA, new StringHandle(content))); + } + writeDocs(docs); } private void writeDocs(List docs) { From d7f4b7a3036403b5b448122067318848770e8a55 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 31 Dec 2025 15:07:22 -0500 Subject: [PATCH 23/70] MLE-26428 Added timestamp to metadata for incremental write --- .../filter/IncrementalWriteEvalFilter.java | 9 ++-- .../filter/IncrementalWriteFilter.java | 43 +++++++++++++------ .../filter/IncrementalWriteOpticFilter.java | 7 +-- .../filter/IncrementalWriteFilterTest.java | 6 ++- .../filter/IncrementalWriteTest.java | 9 ++-- 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index 46aba85c1..aa725eee8 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -22,7 +22,7 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { private static final String EVAL_SCRIPT = """ - const tuples = cts.valueTuples([cts.uriReference(), cts.fieldReference(fieldName)], null, cts.documentQuery(uris)); + const tuples = cts.valueTuples([cts.uriReference(), cts.fieldReference(hashKeyName)], null, cts.documentQuery(uris)); const response = {}; for (var tuple of tuples) { response[tuple[0]] = tuple[1]; @@ -30,8 +30,9 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { response """; - IncrementalWriteEvalFilter(String fieldName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { - super(fieldName, canonicalizeJson, skippedDocumentsConsumer); + IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, + Consumer skippedDocumentsConsumer) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); } @Override @@ -45,7 +46,7 @@ public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { try { JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) - .addVariable("fieldName", fieldName) + .addVariable("hashKeyName", hashKeyName) .addVariable("uris", new JacksonHandle(uris)) .evalAs(JsonNode.class); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 8fe27d05d..cb14f469b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; @@ -39,16 +40,25 @@ public static Builder newBuilder() { public static class Builder { - private String fieldName = "incrementalWriteHash"; + private String hashKeyName = "incrementalWriteHash"; + private String timestampKeyName = "incrementalWriteTimestamp"; private boolean canonicalizeJson = true; private boolean useEvalQuery = false; private Consumer skippedDocumentsConsumer; /** - * @param fieldName the name of the MarkLogic field that will hold the hash value; defaults to "incrementalWriteHash". + * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". */ - public Builder fieldName(String fieldName) { - this.fieldName = fieldName; + public Builder hashKeyName(String keyName) { + this.hashKeyName = keyName; + return this; + } + + /** + * @param keyName the name of the MarkLogic metadata key that will hold the timestamp value; defaults to "incrementalWriteTimestamp". + */ + public Builder timestampKeyName(String keyName) { + this.timestampKeyName = keyName; return this; } @@ -79,13 +89,14 @@ public Builder onDocumentsSkipped(Consumer skippedDocu public IncrementalWriteFilter build() { if (useEvalQuery) { - return new IncrementalWriteEvalFilter(fieldName, canonicalizeJson, skippedDocumentsConsumer); + return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); } - return new IncrementalWriteOpticFilter(fieldName, canonicalizeJson, skippedDocumentsConsumer); + return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); } } - protected final String fieldName; + protected final String hashKeyName; + private final String timestampKeyName; private final boolean canonicalizeJson; private final Consumer skippedDocumentsConsumer; @@ -93,8 +104,9 @@ public IncrementalWriteFilter build() { // See https://xxhash.com for benchmarks. private final LongHashFunction hashFunction = LongHashFunction.xx3(); - public IncrementalWriteFilter(String fieldName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { - this.fieldName = fieldName; + public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { + this.hashKeyName = hashKeyName; + this.timestampKeyName = timestampKeyName; this.canonicalizeJson = canonicalizeJson; this.skippedDocumentsConsumer = skippedDocumentsConsumer; } @@ -102,6 +114,7 @@ public IncrementalWriteFilter(String fieldName, boolean canonicalizeJson, Consum protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { final DocumentWriteSet newWriteSet = context.getDatabaseClient().newDocumentManager().newWriteSet(); final List skippedDocuments = new ArrayList<>(); + final String timestamp = Instant.now().toString(); for (DocumentWriteOperation doc : context.getDocumentWriteSet()) { if (!DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(doc.getOperationType())) { @@ -117,14 +130,14 @@ protected final DocumentWriteSet filterDocuments(Context context, Function skippedDocumentsConsumer) { - super(fieldName, canonicalizeJson, skippedDocumentsConsumer); + IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, + Consumer skippedDocumentsConsumer) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); } @Override @@ -38,7 +39,7 @@ public DocumentWriteSet apply(Context context) { Map existingHashes = rowTemplate.query(op -> op.fromLexicons(Map.of( "uri", op.cts.uriReference(), - "hash", op.cts.fieldReference(super.fieldName) + "hash", op.cts.fieldReference(super.hashKeyName) )).where( op.cts.documentQuery(op.xs.stringSeq(uris)) ), diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java index f1fe81518..8c0cded96 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java @@ -9,6 +9,8 @@ import com.marklogic.client.io.StringHandle; import org.junit.jupiter.api.Test; +import java.time.Instant; + import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -32,7 +34,8 @@ void addHashToMetadata() { DocumentWriteOperation doc1 = new DocumentWriteOperationImpl("/1.xml", metadata, new StringHandle("")); DocumentWriteOperation doc2 = new DocumentWriteOperationImpl("/2.xml", metadata, new StringHandle("")); - doc2 = IncrementalWriteFilter.addHashToMetadata(doc2, "theField", "abc123"); + final String timestamp = Instant.now().toString(); + doc2 = IncrementalWriteFilter.addHashToMetadata(doc2, "theField", "abc123", "theTimestamp", timestamp); assertEquals(metadata, doc1.getMetadata(), "doc1 should still have the original metadata object"); @@ -44,5 +47,6 @@ void addHashToMetadata() { assertEquals("value1", metadata2.getMetadataValues().get("meta1"), "metadata value should be preserved"); assertEquals("abc123", metadata2.getMetadataValues().get("theField"), "hash field should be added"); + assertEquals(timestamp, metadata2.getMetadataValues().get("theTimestamp"), "timestamp should be added"); } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 388473553..47ea0d28d 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -153,7 +153,7 @@ void invalidJsonWithFormat() { @Test void noRangeIndexForField() { filter = IncrementalWriteFilter.newBuilder() - .fieldName("non-existent-field") + .hashKeyName("non-existent-field") .build(); writeTenDocuments(); @@ -168,7 +168,7 @@ void noRangeIndexForField() { @Test void noRangeIndexForFieldWithEval() { filter = IncrementalWriteFilter.newBuilder() - .fieldName("non-existent-field") + .hashKeyName("non-existent-field") .useEvalQuery(true) .build(); @@ -218,8 +218,6 @@ private void verifyDocumentsHasHashInMetadataKey() { while (page.hasNext()) { DocumentRecord doc = page.next(); DocumentMetadataHandle metadata = doc.getMetadata(new DocumentMetadataHandle()); - assertTrue(metadata.getMetadataValues().containsKey("incrementalWriteHash"), - "Document " + doc.getUri() + " should have an incrementalWriteHash in its metadata values."); String hash = metadata.getMetadataValues().get("incrementalWriteHash"); try { @@ -228,6 +226,9 @@ private void verifyDocumentsHasHashInMetadataKey() { } catch (NumberFormatException e) { fail("Document " + doc.getUri() + " has an invalid incrementalWriteHash value: " + hash); } + + String timestamp = metadata.getMetadataValues().get("incrementalWriteTimestamp"); + assertNotNull(timestamp, "Document " + doc.getUri() + " should have an incrementalWriteTimestamp value."); } } From 971d42f9edd7ee6ee0cc80f2318cf0580fa52887 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 2 Jan 2026 11:29:40 -0500 Subject: [PATCH 24/70] MLE-26428 More sensible null handling for incremental write --- .../filter/IncrementalWriteFilter.java | 12 ++++-- .../filter/IncrementalWriteTest.java | 39 ++++++++++++++++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index cb14f469b..b87f29ccb 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.filter; @@ -50,7 +50,10 @@ public static class Builder { * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". */ public Builder hashKeyName(String keyName) { - this.hashKeyName = keyName; + // Don't let user shoot themselves in the foot with an empty key name. + if (keyName != null && !keyName.trim().isEmpty()) { + this.hashKeyName = keyName; + } return this; } @@ -58,7 +61,10 @@ public Builder hashKeyName(String keyName) { * @param keyName the name of the MarkLogic metadata key that will hold the timestamp value; defaults to "incrementalWriteTimestamp". */ public Builder timestampKeyName(String keyName) { - this.timestampKeyName = keyName; + // Don't let user shoot themselves in the foot with an empty key name. + if (keyName != null && !keyName.trim().isEmpty()) { + this.timestampKeyName = keyName; + } return this; } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 47ea0d28d..95bda1326 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.filter; @@ -181,6 +181,43 @@ void noRangeIndexForFieldWithEval() { "fail with a helpful error message. Actual message: " + message); } + @Test + void customTimestampKeyName() { + filter = IncrementalWriteFilter.newBuilder() + .hashKeyName("incrementalWriteHash") + .timestampKeyName("myTimestamp") + .build(); + + writeTenDocuments(); + + DocumentMetadataHandle metadata = Common.client.newDocumentManager().readMetadata("/incremental/test/doc-1.xml", + new DocumentMetadataHandle()); + + assertNotNull(metadata.getMetadataValues().get("myTimestamp")); + assertNotNull(metadata.getMetadataValues().get("incrementalWriteHash")); + assertFalse(metadata.getMetadataValues().containsKey("incrementalWriteTimestamp")); + } + + /** + * The thought for this test is that if the user passes null in (which could happen via our Spark connector), + * they're breaking the feature. So don't let them do that - ignore null and use the default values. + */ + @Test + void nullIsIgnoredForKeyNames() { + filter = IncrementalWriteFilter.newBuilder() + .hashKeyName(null) + .timestampKeyName(null) + .build(); + + writeTenDocuments(); + + DocumentMetadataHandle metadata = Common.client.newDocumentManager().readMetadata("/incremental/test/doc-1.xml", + new DocumentMetadataHandle()); + + assertNotNull(metadata.getMetadataValues().get("incrementalWriteHash")); + assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); + } + private void verifyIncrementalWriteWorks() { writeTenDocuments(); verifyDocumentsHasHashInMetadataKey(); From 408406cb9ed0da52482b68a3fc07441144d4e1bf Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 2 Jan 2026 14:53:33 -0500 Subject: [PATCH 25/70] MLE-26427 Initial exclusion support for JSON This isn't quite done - I want to do a PR for excluding XML next, and then refactor the code, likely moving the tests into a new test class. But this pushes things forward a bit with exclusions. --- .../filter/ContentExclusionUtil.java | 77 +++++++++++++++++++ .../filter/IncrementalWriteEvalFilter.java | 6 +- .../filter/IncrementalWriteFilter.java | 22 +++++- .../filter/IncrementalWriteOpticFilter.java | 6 +- .../filter/IncrementalWriteTest.java | 59 ++++++++++++++ 5 files changed, 161 insertions(+), 9 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java new file mode 100644 index 000000000..0510cc0b0 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.core.JsonPointer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for applying content exclusions to documents before hash calculation. + * Supports removing specific paths from JSON and XML documents using JSON Pointer and XPath expressions. + * + * @since 8.1.0 + */ +public class ContentExclusionUtil { + + private static final Logger logger = LoggerFactory.getLogger(ContentExclusionUtil.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + /** + * Applies JSON Pointer exclusions to JSON content by removing the specified paths. + * + * @param uri the document URI (used for logging purposes) + * @param jsonContent the JSON content as a string + * @param jsonPointers array of RFC 6901 JSON Pointer expressions identifying properties to exclude + * @return the modified JSON content with specified paths removed + * @throws JsonProcessingException if the JSON content cannot be parsed or serialized + */ + public static String applyJsonExclusions(String uri, String jsonContent, String[] jsonPointers) throws JsonProcessingException { + if (jsonPointers == null || jsonPointers.length == 0) { + return jsonContent; + } + + JsonNode rootNode = OBJECT_MAPPER.readTree(jsonContent); + for (String jsonPointer : jsonPointers) { + removeNodeAtPointer(uri, rootNode, jsonPointer); + } + return OBJECT_MAPPER.writeValueAsString(rootNode); + } + + /** + * Removes a node at the specified JSON Pointer path from the given root node. + * + * @param uri the document URI (used for logging purposes) + * @param rootNode the root JSON node + * @param jsonPointer the JSON Pointer expression identifying the node to remove + */ + private static void removeNodeAtPointer(String uri, JsonNode rootNode, String jsonPointer) { + JsonPointer pointer = JsonPointer.compile(jsonPointer); + JsonNode targetNode = rootNode.at(pointer); + + if (targetNode.isMissingNode()) { + logger.debug("JSONPointer '{}' does not exist in document {}, skipping", jsonPointer, uri); + return; + } + + // Use Jackson's JsonPointer API to get parent and field name + JsonPointer parentPointer = pointer.head(); + JsonNode parentNode = rootNode.at(parentPointer); + + if (parentNode.isObject()) { + String fieldName = pointer.last().getMatchingProperty(); + ((ObjectNode) parentNode).remove(fieldName); + } else if (parentNode.isArray()) { + logger.warn("Array element exclusion not supported for JSONPointer '{}'. " + + "Consider excluding the entire array property instead.", jsonPointer); + } + } + + // Future method for XML exclusions + // public static String applyXmlExclusions(String xmlContent, String[] xpaths) { ... } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index aa725eee8..54343d80e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.filter; @@ -31,8 +31,8 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { """; IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); + Consumer skippedDocumentsConsumer, String[] jsonExclusions) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index b87f29ccb..86cc14e62 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -45,6 +45,7 @@ public static class Builder { private boolean canonicalizeJson = true; private boolean useEvalQuery = false; private Consumer skippedDocumentsConsumer; + private String[] jsonExclusions; /** * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". @@ -93,11 +94,20 @@ public Builder onDocumentsSkipped(Consumer skippedDocu return this; } + /** + * @param jsonPointers JSON Pointer expressions (RFC 6901) identifying JSON properties to exclude from hash calculation. + * For example, "/metadata/timestamp" or "/user/lastModified". + */ + public Builder jsonExclusions(String... jsonPointers) { + this.jsonExclusions = jsonPointers; + return this; + } + public IncrementalWriteFilter build() { if (useEvalQuery) { - return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); + return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); } - return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); + return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); } } @@ -105,16 +115,18 @@ public IncrementalWriteFilter build() { private final String timestampKeyName; private final boolean canonicalizeJson; private final Consumer skippedDocumentsConsumer; + private final String[] jsonExclusions; // Hardcoding this for now, with a good general purpose hashing function. // See https://xxhash.com for benchmarks. private final LongHashFunction hashFunction = LongHashFunction.xx3(); - public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer) { + public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions) { this.hashKeyName = hashKeyName; this.timestampKeyName = timestampKeyName; this.canonicalizeJson = canonicalizeJson; this.skippedDocumentsConsumer = skippedDocumentsConsumer; + this.jsonExclusions = jsonExclusions; } protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { @@ -165,6 +177,10 @@ private String serializeContent(DocumentWriteOperation doc) { if (canonicalizeJson && (Format.JSON.equals(format) || isPossiblyJsonContent(content))) { JsonCanonicalizer jc; try { + if (jsonExclusions != null && jsonExclusions.length > 0) { + // TBD on error handling here, want to get XML supported first. + content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, jsonExclusions); + } jc = new JsonCanonicalizer(content); return jc.getEncodedString(); } catch (IOException e) { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java index 5d6d57642..3cb8f44e0 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.filter; @@ -20,8 +20,8 @@ class IncrementalWriteOpticFilter extends IncrementalWriteFilter { IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer); + Consumer skippedDocumentsConsumer, String[] jsonExclusions) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); } @Override diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 95bda1326..9929fdc98 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -218,6 +218,65 @@ void nullIsIgnoredForKeyNames() { assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); } + @Test + void jsonExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .jsonExclusions("/timestamp", "/metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents with three keys + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2025-01-01T10:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2025-01-01T10:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed + doc.putObject("metadata") + .put("lastModified", "2026-01-02T15:30:00Z") // Changed + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Modified Document " + i); // Changed + doc.put("timestamp", "2026-01-02T16:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2026-01-02T16:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } + private void verifyIncrementalWriteWorks() { writeTenDocuments(); verifyDocumentsHasHashInMetadataKey(); From 1cf578995afe290c7a8d91717f68eaa7c559ae15 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 5 Jan 2026 10:40:43 -0500 Subject: [PATCH 26/70] MLE-26460 Refactor: Cleaning up usage of QueryConfig Made QueryConfig into a record so it's immutable, making it easier to understand. And passing it to the constructor of QueryBatcherImpl instead of having 6 separate args. --- .../impl/DataMovementManagerImpl.java | 9 +- .../impl/DataMovementServices.java | 118 ++++++++---------- .../datamovement/impl/QueryBatcherImpl.java | 77 ++++++------ .../client/datamovement/impl/QueryConfig.java | 16 +++ 4 files changed, 113 insertions(+), 107 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java index d31a46fbe..7b2362c8a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java @@ -1,11 +1,10 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientBuilder; -import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.datamovement.*; import com.marklogic.client.impl.DatabaseClientImpl; import com.marklogic.client.io.marker.ContentHandle; @@ -123,10 +122,8 @@ private QueryBatcher newQueryBatcherImpl(SearchQueryDefinition query) { QueryBatcherImpl queryBatcher = null; // preprocess the query if the effective version is at least 10.0-5 if (Long.compareUnsigned(getServerVersion(), Long.parseUnsignedLong("10000500")) >= 0) { - DataMovementServices.QueryConfig queryConfig = service.initConfig("POST", query); - queryBatcher = new QueryBatcherImpl(query, this, queryConfig.forestConfig, - queryConfig.serializedCtsQuery, queryConfig.filtered, - queryConfig.maxDocToUriBatchRatio, queryConfig.defaultDocBatchSize, queryConfig.maxUriBatchSize); + QueryConfig queryConfig = service.initConfig("POST", query); + queryBatcher = new QueryBatcherImpl(query, this, queryConfig); } else { queryBatcher = new QueryBatcherImpl(query, this, getForestConfig()); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java index 8699c81e8..eb58c15e2 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; @@ -23,7 +23,8 @@ import java.util.List; public class DataMovementServices { - private static Logger logger = LoggerFactory.getLogger(DataMovementServices.class); + + private static final Logger logger = LoggerFactory.getLogger(DataMovementServices.class); private DatabaseClient client; @@ -36,59 +37,58 @@ public DataMovementServices setClient(DatabaseClient client) { return this; } - QueryConfig initConfig(String method, SearchQueryDefinition qdef) { - logger.debug("initializing forest configuration with query"); - if (qdef == null) throw new IllegalArgumentException("null query definition"); - - JsonNode result = ((DatabaseClientImpl) this.client).getServices() - .forestInfo(null, method, new RequestParameters(), qdef, new JacksonHandle()) - .get(); - // System.out.println(result.toPrettyString()); - - QueryConfig queryConfig = new QueryConfig(); - - try { - ObjectMapper mapper = new ObjectMapper(); - JsonNode queryResult = result.get("query"); - if (queryResult != null && queryResult.isObject() && queryResult.has("ctsquery")) { - queryConfig.serializedCtsQuery = mapper.writeValueAsString(queryResult); - logger.debug("initialized query to: {}", queryConfig.serializedCtsQuery); - } - JsonNode filteredResult = result.get("filtered"); - if (filteredResult != null && filteredResult.isBoolean()) { - queryConfig.filtered = filteredResult.asBoolean(); - logger.debug("initialized filtering to: {}", queryConfig.filtered.toString()); - } - JsonNode maxDocToUriBatchRatio = result.get("maxDocToUriBatchRatio"); - if (maxDocToUriBatchRatio != null && maxDocToUriBatchRatio.isInt()) { - queryConfig.maxDocToUriBatchRatio = maxDocToUriBatchRatio.asInt(); - logger.debug("initialized maxDocToUriBatchRatio to : {}", queryConfig.maxDocToUriBatchRatio); - } else { - queryConfig.maxDocToUriBatchRatio = -1; - } - JsonNode defaultDocBatchSize = result.get("defaultDocBatchSize"); - if (defaultDocBatchSize != null && defaultDocBatchSize.isInt()) { - queryConfig.defaultDocBatchSize = defaultDocBatchSize.asInt(); - logger.debug("initialized defaultDocBatchSize to : {}", queryConfig.defaultDocBatchSize); - } else { - queryConfig.defaultDocBatchSize = -1; - } - JsonNode maxUriBatchSize = result.get("maxUriBatchSize"); - if (maxUriBatchSize != null && maxUriBatchSize.isInt()) { - queryConfig.maxUriBatchSize = maxUriBatchSize.asInt(); - logger.debug("initialized maxUriBatchSize to : {}", queryConfig.maxUriBatchSize); - } else { - queryConfig.maxUriBatchSize = -1; - } - - } catch (JsonProcessingException e) { - logger.error("failed to initialize query", e); - } - - queryConfig.forestConfig = makeForestConfig(result.has("forests") ? result.get("forests") : result); - - return queryConfig; - } + QueryConfig initConfig(String method, SearchQueryDefinition qdef) { + logger.debug("initializing forest configuration with query"); + if (qdef == null) throw new IllegalArgumentException("null query definition"); + + JsonNode result = ((DatabaseClientImpl) this.client).getServices() + .forestInfo(null, method, new RequestParameters(), qdef, new JacksonHandle()) + .get(); + + JsonNode queryResult = result.get("query"); + + String serializedCtsQuery = null; + if (queryResult != null && queryResult.isObject() && queryResult.has("ctsquery")) { + try { + serializedCtsQuery = new ObjectMapper().writeValueAsString(queryResult); + logger.debug("initialized query to: {}", serializedCtsQuery); + } catch (JsonProcessingException e) { + logger.warn("Unable to serialize query result while initializing QueryBatcher; cause: {}", e.getMessage()); + } + } + + JsonNode filteredResult = result.get("filtered"); + Boolean filtered = null; + if (filteredResult != null && filteredResult.isBoolean()) { + filtered = filteredResult.asBoolean(); + logger.debug("initialized filtering to: {}", filtered); + } + + JsonNode maxDocToUriBatchRatioNode = result.get("maxDocToUriBatchRatio"); + int maxDocToUriBatchRatio = -1; + if (maxDocToUriBatchRatioNode != null && maxDocToUriBatchRatioNode.isInt()) { + maxDocToUriBatchRatio = maxDocToUriBatchRatioNode.asInt(); + logger.debug("initialized maxDocToUriBatchRatio to : {}", maxDocToUriBatchRatio); + } + + JsonNode defaultDocBatchSizeNode = result.get("defaultDocBatchSize"); + int defaultDocBatchSize = -1; + if (defaultDocBatchSizeNode != null && defaultDocBatchSizeNode.isInt()) { + defaultDocBatchSize = defaultDocBatchSizeNode.asInt(); + logger.debug("initialized defaultDocBatchSize to : {}", defaultDocBatchSize); + } + + JsonNode maxUriBatchSizeNode = result.get("maxUriBatchSize"); + int maxUriBatchSize = -1; + if (maxUriBatchSizeNode != null && maxUriBatchSizeNode.isInt()) { + maxUriBatchSize = maxUriBatchSizeNode.asInt(); + logger.debug("initialized maxUriBatchSize to : {}", maxUriBatchSize); + } + + ForestConfiguration forestConfig = makeForestConfig(result.has("forests") ? result.get("forests") : result); + return new QueryConfig(serializedCtsQuery, forestConfig, filtered, + maxDocToUriBatchRatio, defaultDocBatchSize, maxUriBatchSize); + } ForestConfigurationImpl readForestConfig() { logger.debug("initializing forest configuration"); @@ -183,12 +183,4 @@ private String generateJobId() { return UUID.randomUUID().toString(); } - static class QueryConfig { - String serializedCtsQuery; - ForestConfiguration forestConfig; - Boolean filtered; - int maxDocToUriBatchRatio; - int defaultDocBatchSize; - int maxUriBatchSize; - } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java index f9581867a..031e8b3df 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; @@ -42,16 +42,17 @@ * startIterating, withForestConfig, and retry. */ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { - private static Logger logger = LoggerFactory.getLogger(QueryBatcherImpl.class); + private static final Logger logger = LoggerFactory.getLogger(QueryBatcherImpl.class); private String queryMethod; private SearchQueryDefinition query; - private SearchQueryDefinition originalQuery; private Boolean filtered; private Iterator iterator; private boolean threadCountSet = false; - private List urisReadyListeners = new ArrayList<>(); - private List failureListeners = new ArrayList<>(); - private List jobCompletionListeners = new ArrayList<>(); + + private final List urisReadyListeners = new ArrayList<>(); + private final List failureListeners = new ArrayList<>(); + private final List jobCompletionListeners = new ArrayList<>(); + private QueryThreadPoolExecutor threadPool; private boolean consistentSnapshot = false; private final AtomicLong batchNumber = new AtomicLong(0); @@ -61,10 +62,13 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { private Map forestResults = new HashMap<>(); private Map forestIsDone = new HashMap<>(); private Map retryForestMap = new HashMap<>(); - private AtomicBoolean runJobCompletionListeners = new AtomicBoolean(false); + + private final AtomicBoolean runJobCompletionListeners = new AtomicBoolean(false); private final Object lock = new Object(); private final Map> blackListedTasks = new HashMap<>(); + private boolean isSingleThreaded = false; + private long maxUris = Long.MAX_VALUE; private long maxBatches = Long.MAX_VALUE; private int maxDocToUriBatchRatio; @@ -72,40 +76,37 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { private int defaultDocBatchSize; private int maxUriBatchSize; - QueryBatcherImpl( - SearchQueryDefinition originalQuery, DataMovementManager moveMgr, ForestConfiguration forestConfig, - String serializedCtsQuery, Boolean filtered, int maxDocToUriBatchRatio, int defaultDocBatchSize, int maxUriBatchSize - ) { - this(moveMgr, forestConfig, maxDocToUriBatchRatio, defaultDocBatchSize, maxUriBatchSize); - // TODO: skip conversion in DataMovementManagerImpl.newQueryBatcherImpl() unless canSerializeQueryAsJSON() - if (serializedCtsQuery != null && serializedCtsQuery.length() > 0 && - originalQuery instanceof AbstractSearchQueryDefinition && - ((AbstractSearchQueryDefinition) originalQuery).canSerializeQueryAsJSON()) { - QueryManagerImpl queryMgr = (QueryManagerImpl) getPrimaryClient().newQueryManager(); - this.queryMethod = "POST"; - this.query = queryMgr.newRawCtsQueryDefinition(new StringHandle(serializedCtsQuery).withFormat(Format.JSON)); - this.originalQuery = originalQuery; - if (filtered != null) { - this.filtered = filtered; - } - } else { - initQuery(originalQuery); - } - } + QueryBatcherImpl(SearchQueryDefinition originalQuery, DataMovementManager moveMgr, QueryConfig queryConfig) { + this(moveMgr, queryConfig); + + final String serializedCtsQuery = queryConfig.serializedCtsQuery(); + if (serializedCtsQuery != null && !serializedCtsQuery.isEmpty() && + originalQuery instanceof AbstractSearchQueryDefinition && + ((AbstractSearchQueryDefinition) originalQuery).canSerializeQueryAsJSON()) { + QueryManagerImpl queryMgr = (QueryManagerImpl) getPrimaryClient().newQueryManager(); + this.queryMethod = "POST"; + this.query = queryMgr.newRawCtsQueryDefinition(new StringHandle(serializedCtsQuery).withFormat(Format.JSON)); + this.filtered = queryConfig.filtered(); + } else { + initQuery(originalQuery); + } + } + public QueryBatcherImpl(SearchQueryDefinition query, DataMovementManager moveMgr, ForestConfiguration forestConfig) { this(moveMgr, forestConfig); initQuery(query); } + public QueryBatcherImpl(Iterator iterator, DataMovementManager moveMgr, ForestConfiguration forestConfig) { this(moveMgr, forestConfig); this.iterator = iterator; } - private QueryBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forestConfig, - int maxDocToUriBatchRatio, int defaultDocBatchSize, int maxUriBatchSize) { - this(moveMgr, forestConfig); - this.maxDocToUriBatchRatio = maxDocToUriBatchRatio; - this.defaultDocBatchSize = defaultDocBatchSize; - this.maxUriBatchSize = maxUriBatchSize; + + private QueryBatcherImpl(DataMovementManager moveMgr, QueryConfig queryConfig) { + this(moveMgr, queryConfig.forestConfig()); + this.maxDocToUriBatchRatio = queryConfig.maxDocToUriBatchRatio(); + this.defaultDocBatchSize = queryConfig.defaultDocBatchSize(); + this.maxUriBatchSize = queryConfig.maxUriBatchSize(); withBatchSize(defaultDocBatchSize); } private QueryBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forestConfig) { @@ -187,7 +188,7 @@ public void retryWithFailureListeners(QueryEvent queryEvent) { } private void retry(QueryEvent queryEvent, boolean callFailListeners) { - if ( isStopped() == true ) { + if ( isStopped()) { logger.warn("Job is now stopped, aborting the retry"); return; } @@ -449,7 +450,7 @@ public synchronized void start(JobTicket ticket) { private synchronized void initialize() { Forest[] forests = getForestConfig().listForests(); - if ( threadCountSet == false ) { + if ( !threadCountSet ) { if ( query != null ) { logger.warn("threadCount not set--defaulting to number of forests ({})", forests.length); withThreadCount(forests.length * docToUriBatchRatio); @@ -529,7 +530,7 @@ public synchronized QueryBatcher withForestConfig(ForestConfiguration forestConf List newClientList = clients(hostNames); clientList.set(newClientList); boolean started = (threadPool != null); - if ( started == true && oldForests.size() > 0 ) calculateDeltas(oldForests, forests); + if ( started && !oldForests.isEmpty() ) calculateDeltas(oldForests, forests); return this; } @@ -550,7 +551,7 @@ private synchronized void calculateDeltas(Set oldForests, Forest[] fores // this forest is not black-listed blackListedForests.remove(forest); } - if ( blackListedForests.size() > 0 ) { + if ( !blackListedForests.isEmpty() ) { DataMovementManagerImpl moveMgrImpl = getMoveMgr(); String primaryHost = moveMgrImpl.getPrimaryClient().getHost(); if ( getHostNames(blackListedForests).contains(primaryHost) ) { @@ -562,7 +563,7 @@ private synchronized void calculateDeltas(Set oldForests, Forest[] fores } private synchronized void cleanupExistingTasks(Set addedForests, Set restartedForests, Set blackListedForests) { - if ( blackListedForests.size() > 0 ) { + if ( !blackListedForests.isEmpty() ) { logger.warn("removing jobs related to hosts [{}] from the queue", getHostNames(blackListedForests)); // since some forests have been removed, let's remove from the queue any jobs that were targeting that forest List tasks = new ArrayList<>(); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java new file mode 100644 index 000000000..f46883622 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.impl; + +import com.marklogic.client.datamovement.ForestConfiguration; + +record QueryConfig( + String serializedCtsQuery, + ForestConfiguration forestConfig, + Boolean filtered, + int maxDocToUriBatchRatio, + int defaultDocBatchSize, + int maxUriBatchSize +) { +} From aa95e398c8dd2039a7200f9707cbb2f705974b88 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 5 Jan 2026 10:43:12 -0500 Subject: [PATCH 27/70] MLE-26460 Preventing maxDocToUriBatchRatio value of -1 --- .../impl/DataMovementServices.java | 7 +++ .../FewerServerThreadsThanForestsTest.java | 62 +++++++++++++++++++ .../com/marklogic/client/test/Common.java | 8 ++- 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java index eb58c15e2..de1754a50 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java @@ -71,6 +71,13 @@ QueryConfig initConfig(String method, SearchQueryDefinition qdef) { logger.debug("initialized maxDocToUriBatchRatio to : {}", maxDocToUriBatchRatio); } + // Per GitHub bug 1872 and MLE-26460, the server may return -1 when there are fewer server threads than forests. + // A value of -1 will cause later problems when constructing a LinkedBlockingQueue with a negative capacity. + // So defaulting this to 1 to avoid later errors. + if (maxDocToUriBatchRatio <= 0) { + maxDocToUriBatchRatio = 1; + } + JsonNode defaultDocBatchSizeNode = result.get("defaultDocBatchSize"); int defaultDocBatchSize = -1; if (defaultDocBatchSizeNode != null && defaultDocBatchSizeNode.isInt()) { diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java new file mode 100644 index 000000000..b7d7b1650 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import com.marklogic.mgmt.ManageClient; +import com.marklogic.mgmt.resource.appservers.ServerManager; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FewerServerThreadsThanForestsTest extends AbstractClientTest { + + @Test + void test() { + DatabaseClient client = Common.newClient(); + final int forestCount = client.newDataMovementManager().readForestConfig().listForests().length; + if (forestCount < 2) { + logger.info("This test requires multiple forests so that the server thread count can be set to the " + + "number of forests minus one; skipping test"); + return; + } + + adjustServerThreads(forestCount - 1); + try { + DataMovementManager dmm = client.newDataMovementManager(); + AtomicInteger uriCount = new AtomicInteger(); + QueryBatcher queryBatcher = dmm.newQueryBatcher(client.newQueryManager().newStructuredQueryBuilder().collection("/optic/test")) + .withThreadCount(1) + .onUrisReady(batch -> uriCount.addAndGet(batch.getItems().length)); + dmm.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dmm.stopJob(queryBatcher); + + assertEquals(4, uriCount.get(), "Verifies that the 4 test documents were found, and more importantly, " + + "that the new default maxDocToUriBatchRatio of 1 was applied correctly when the number of " + + "server threads is less than the number of forests. This is for bug 1872 in GitHub. Prior to this " + + "fix, the maxDocToUriBatchRatio of -1 returned by the server caused an error when the " + + "LinkedBlockingQueue was constructed with a negative capacity."); + } finally { + // We can safely use this number because we know the test-app doesn't change this. + final int defaultServerThreadCount = 32; + adjustServerThreads(defaultServerThreadCount); + } + } + + private void adjustServerThreads(final int threads) { + logger.info("Adjusting server threads to {}", threads); + Common.newAdminManager().invokeActionRequiringRestart(() -> { + ManageClient manageClient = Common.newManageClient(); + ObjectNode payload = Common.newServerPayload().put("threads", threads); + new ServerManager(manageClient).save(payload.toString()); + return true; + }); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java index 2437bf255..b5a45af95 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -13,6 +13,8 @@ import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.ManageConfig; +import com.marklogic.mgmt.admin.AdminConfig; +import com.marklogic.mgmt.admin.AdminManager; import org.springframework.util.FileCopyUtils; import org.w3c.dom.DOMException; import org.w3c.dom.Document; @@ -258,6 +260,10 @@ public static ManageClient newManageClient() { return new ManageClient(new ManageConfig(HOST, 8002, SERVER_ADMIN_USER, SERVER_ADMIN_PASS)); } + public static AdminManager newAdminManager() { + return new AdminManager(new AdminConfig(HOST, 8001, SERVER_ADMIN_USER, SERVER_ADMIN_PASS)); + } + public static ObjectNode newServerPayload() { ObjectNode payload = new ObjectMapper().createObjectNode(); payload.put("server-name", SERVER_NAME); From 263ea0ac26fa0fb6dd4373006055f7a481dcb2b6 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 5 Jan 2026 14:49:31 -0500 Subject: [PATCH 28/70] MLE-26427 Updating copyright config Hoping that this causes the checker to automatically update files. --- .github/workflows/pr-workflow.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index d11ced4a0..4562e18d6 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -18,6 +18,6 @@ jobs: name: © Validate Copyright Headers uses: marklogic/pr-workflows/.github/workflows/copyright-check.yml@main permissions: - contents: read + contents: write pull-requests: write - issues: write \ No newline at end of file + issues: write From acdeb0cf435b3c9628e761633c41f48633a85252 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 5 Jan 2026 14:05:31 -0500 Subject: [PATCH 29/70] MLE-26427 Refactoring JAXP usage Doing this before adding support for XML exclusions in the incremental write feature. Standardizes on a single way of creating a DocumentBuilderFactory. Also made a year-change-based fix in LegalHoldsTest. --- .../marklogic/client/impl/NodeConverter.java | 16 +-- .../marklogic/client/impl/XmlFactories.java | 103 ++++++++++++------ .../com/marklogic/client/io/DOMHandle.java | 65 ++--------- .../client/io/DocumentMetadataHandle.java | 7 +- .../client/test/BufferableHandleTest.java | 17 ++- .../com/marklogic/client/test/Common.java | 6 +- .../client/test/DeleteSearchTest.java | 15 +-- .../com/marklogic/client/test/EvalTest.java | 14 +-- .../test/GeospatialRegionQueriesTest.java | 9 +- .../marklogic/client/test/HandleAsTest.java | 8 +- .../client/test/PlanGeneratedBase.java | 10 +- .../client/test/QueryOptionsManagerTest.java | 15 +-- .../client/test/RawQueryDefinitionTest.java | 9 +- .../client/test/RequestLoggerTest.java | 14 +-- .../client/test/SearchFacetTest.java | 9 +- .../client/test/XMLDocumentTest.java | 13 +-- .../test/datamovement/LegalHoldsTest.java | 4 +- .../test/datamovement/ScenariosTest.java | 10 +- .../test/io/DocumentMetadataHandleTest.java | 9 +- 19 files changed, 144 insertions(+), 209 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java index a885dc6e8..1baf50ba5 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -35,7 +35,6 @@ public class NodeConverter { static private ObjectMapper mapper; - static private DocumentBuilderFactory documentBuilderFactory; static private XMLInputFactory xmlInputFactory; private NodeConverter() { @@ -49,16 +48,7 @@ static private ObjectMapper getMapper() { } return mapper; } - static private DocumentBuilderFactory getDocumentBuilderFactory() { - // okay if one thread overwrites another during lazy initialization - if (documentBuilderFactory == null) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - documentBuilderFactory = factory; - } - return documentBuilderFactory; - } + static private XMLInputFactory getXMLInputFactory() { // okay if one thread overwrites another during lazy initialization if (xmlInputFactory == null) { @@ -265,7 +255,7 @@ static public Stream ReaderToJsonParser(Stream val static public Document InputStreamToDocument(InputStream inputStream) { try { - return (inputStream == null) ? null : getDocumentBuilderFactory().newDocumentBuilder().parse(inputStream); + return (inputStream == null) ? null : XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().parse(inputStream); } catch(SAXException | IOException | ParserConfigurationException e) { throw new RuntimeException(e); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java index 6f1484159..6773ef19f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -7,24 +7,25 @@ import org.slf4j.LoggerFactory; import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.FactoryConfigurationError; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; import java.lang.ref.SoftReference; +import java.util.function.Supplier; public final class XmlFactories { private static final Logger logger = LoggerFactory.getLogger(XmlFactories.class); private static final CachedInstancePerThreadSupplier cachedOutputFactory = - new CachedInstancePerThreadSupplier(new Supplier() { - @Override - public XMLOutputFactory get() { - return makeNewOutputFactory(); - } - }); + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewOutputFactory); + + private static final CachedInstancePerThreadSupplier cachedDocumentBuilderFactory = + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewDocumentBuilderFactory); private XmlFactories() {} // preventing instances of utility class @@ -62,21 +63,78 @@ public static TransformerFactory makeNewTransformerFactory() { try { factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); } catch (TransformerConfigurationException e) { - logger.warn("Unable to set {} on TransformerFactory; cause: {}", XMLConstants.FEATURE_SECURE_PROCESSING, e.getMessage()); + logTransformerWarning(XMLConstants.FEATURE_SECURE_PROCESSING, e.getMessage()); } try { factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); } catch (IllegalArgumentException e) { - logger.warn("Unable to set {} on TransformerFactory; cause: {}", XMLConstants.ACCESS_EXTERNAL_DTD, e.getMessage()); + logTransformerWarning(XMLConstants.ACCESS_EXTERNAL_DTD, e.getMessage()); } try { factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); } catch (IllegalArgumentException e) { - logger.warn("Unable to set {} on TransformerFactory; cause: {}", XMLConstants.ACCESS_EXTERNAL_STYLESHEET, e.getMessage()); + logTransformerWarning(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, e.getMessage()); } return factory; } + private static void logTransformerWarning(String xmlConstant, String errorMessage) { + logger.warn("Unable to set {} on TransformerFactory; cause: {}", xmlConstant, errorMessage); + } + + private static DocumentBuilderFactory makeNewDocumentBuilderFactory() { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + // Default to best practices for conservative security including recommendations per + // https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.md + try { + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set FEATURE_SECURE_PROCESSING on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set disallow-doctype-decl on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set external-general-entities on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set external-parameter-entities on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set load-external-dtd on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + factory.setNamespaceAware(true); + factory.setValidating(false); + + return factory; + } + + /** + * Returns a shared {@link DocumentBuilderFactory} configured with secure defaults. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a securely configured {@link DocumentBuilderFactory} + * + * @since 8.1.0 + * + * @see #makeNewDocumentBuilderFactory() if you really (really?) need a non-shared instance + */ + public static DocumentBuilderFactory getDocumentBuilderFactory() { + return cachedDocumentBuilderFactory.get(); + } + /** * Returns a shared {@link XMLOutputFactory}. This factory will have its * {@link XMLOutputFactory#IS_REPAIRING_NAMESPACES} property set to {@code true}. @@ -88,31 +146,12 @@ public static TransformerFactory makeNewTransformerFactory() { * * @throws FactoryConfigurationError see {@link XMLOutputFactory#newInstance()} * - * @see #makeNewOutputFactory() if you really (really?) need an non-shared instance + * @see #makeNewOutputFactory() if you really (really?) need a non-shared instance */ public static XMLOutputFactory getOutputFactory() { return cachedOutputFactory.get(); } - /** - * Represents a supplier of results. - * - *

There is no requirement that a new or distinct result be returned each - * time the supplier is invoked. - * - * @param the type of results supplied by this supplier - */ - // TODO replace with java.util.function.Supplier after Java 8 migration - interface Supplier { - - /** - * Gets a result. - * - * @return a result - */ - T get(); - } - /** * A supplier that caches results per thread. *

@@ -129,7 +168,7 @@ interface Supplier { */ private static class CachedInstancePerThreadSupplier implements Supplier { - private final ThreadLocal> cachedInstances = new ThreadLocal>(); + private final ThreadLocal> cachedInstances = new ThreadLocal<>(); /** * The underlying supplier, invoked to originally retrieve the per-thread result @@ -167,7 +206,7 @@ public T get() { } // ... and retain it for later re-use - cachedInstances.set(new SoftReference(cachedInstance)); + cachedInstances.set(new SoftReference<>(cachedInstance)); } return cachedInstance; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java index f06846fd1..90d8c0fc4 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java @@ -1,25 +1,11 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.io; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; - -import javax.xml.XMLConstants; -import javax.xml.namespace.QName; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; - +import com.marklogic.client.MarkLogicIOException; +import com.marklogic.client.MarkLogicInternalException; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.marker.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,15 +13,14 @@ import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.w3c.dom.ls.DOMImplementationLS; -import org.w3c.dom.ls.LSException; -import org.w3c.dom.ls.LSInput; -import org.w3c.dom.ls.LSOutput; -import org.w3c.dom.ls.LSParser; -import org.w3c.dom.ls.LSResourceResolver; +import org.w3c.dom.ls.*; -import com.marklogic.client.MarkLogicIOException; -import com.marklogic.client.MarkLogicInternalException; +import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.*; +import java.io.*; +import java.nio.charset.StandardCharsets; /** * A DOM Handle represents XML content as a DOM document for reading or writing. @@ -199,7 +184,7 @@ public String toString() { */ public DocumentBuilderFactory getFactory() throws ParserConfigurationException { if (factory == null) - factory = makeDocumentBuilderFactory(); + factory = XmlFactories.getDocumentBuilderFactory(); return factory; } /** @@ -209,32 +194,6 @@ public DocumentBuilderFactory getFactory() throws ParserConfigurationException { public void setFactory(DocumentBuilderFactory factory) { this.factory = factory; } - protected DocumentBuilderFactory makeDocumentBuilderFactory() throws ParserConfigurationException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - // default to best practices for conservative security including recommendations per - // https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.md - try { - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - } catch (ParserConfigurationException e) {} - factory.setXIncludeAware(false); - factory.setExpandEntityReferences(false); - factory.setNamespaceAware(true); - factory.setValidating(false); - - return factory; - } /** * Get the processor used to evaluate XPath expressions. diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java index 643e76ca4..30582b82b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.io; @@ -577,10 +577,7 @@ protected void receiveContent(InputStream content) { Document document = null; if (content != null) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); document = builder.parse(new InputSource(new InputStreamReader(content, StandardCharsets.UTF_8))); content.close(); } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java index 73f9d6c19..5c2e7e756 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java @@ -1,11 +1,14 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; import com.marklogic.client.test.util.Referred; import com.marklogic.client.test.util.Refers; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; import org.custommonkey.xmlunit.SimpleNamespaceContext; import org.custommonkey.xmlunit.XMLUnit; import org.custommonkey.xmlunit.XpathEngine; @@ -17,16 +20,14 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.util.HashMap; import java.util.Map; import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class BufferableHandleTest { static private XpathEngine xpather; @@ -61,11 +62,7 @@ public void testReadWrite() throws JAXBException, ParserConfigurationException, SAXException, IOException, XpathException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - - Document domDocument = factory.newDocumentBuilder().newDocument(); + Document domDocument = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); root.setAttribute("foo", "bar"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java index b5a45af95..26c01ec61 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java @@ -9,6 +9,7 @@ import com.marklogic.client.DatabaseClientBuilder; import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.impl.okhttp.RetryIOExceptionInterceptor; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.mgmt.ManageClient; @@ -228,10 +229,7 @@ public static String testDocumentToString(Document document) { public static Document testStringToDocument(String document) { try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - return factory.newDocumentBuilder().parse( + return XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().parse( new InputSource(new StringReader(document))); } catch (SAXException e) { throw new RuntimeException(e); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java index 901655e81..3012f3800 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -7,6 +7,7 @@ import com.marklogic.client.document.DocumentDescriptor; import com.marklogic.client.document.GenericDocumentManager; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.query.DeleteQueryDefinition; import com.marklogic.client.query.QueryManager; @@ -15,8 +16,6 @@ import org.w3c.dom.Element; import org.w3c.dom.ls.DOMImplementationLS; -import javax.xml.parsers.DocumentBuilderFactory; - import static org.junit.jupiter.api.Assertions.*; @TestMethodOrder(MethodOrderer.MethodName.class) @@ -32,11 +31,7 @@ public static void beforeClass() throws Exception { } public static void writeDoc() throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - - Document domDocument = factory.newDocumentBuilder().newDocument(); + Document domDocument = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); root.setAttribute("foo", "bar"); @@ -45,7 +40,7 @@ public static void writeDoc() throws Exception { domDocument.appendChild(root); @SuppressWarnings("unused") - String domString = ((DOMImplementationLS) factory.newDocumentBuilder() + String domString = ((DOMImplementationLS) XmlFactories.getDocumentBuilderFactory().newDocumentBuilder() .getDOMImplementation()).createLSSerializer().writeToString(domDocument); XMLDocumentManager docMgr = client.newXMLDocumentManager(); @@ -61,7 +56,7 @@ public void test_A_Delete() { GenericDocumentManager docMgr = client.newDocumentManager(); DocumentDescriptor desc = docMgr.exists(docId); assertNotNull(desc); - assertEquals(desc.getUri(), docId); + assertEquals(docId, desc.getUri()); QueryManager queryMgr = client.newQueryManager(); DeleteQueryDefinition qdef = queryMgr.newDeleteDefinition(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java index fc9599425..17f76ee5c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -18,21 +18,20 @@ import com.marklogic.client.eval.EvalResultIterator; import com.marklogic.client.eval.ServerEvaluationCall; import com.marklogic.client.impl.HandleAccessor; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; import com.marklogic.client.query.DeleteQueryDefinition; import com.marklogic.client.query.QueryManager; -import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.resource.appservers.ServerManager; +import jakarta.xml.bind.DatatypeConverter; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; -import jakarta.xml.bind.DatatypeConverter; import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.BufferedReader; import java.io.IOException; @@ -257,12 +256,9 @@ public void getNullTests() throws DatatypeConfigurationException, JsonProcessing } private void runAndTestXQuery(ServerEvaluationCall call) - throws JsonProcessingException, IOException, SAXException, ParserConfigurationException, DatatypeConfigurationException + throws IOException, SAXException, ParserConfigurationException, DatatypeConfigurationException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document document = factory.newDocumentBuilder() + Document document = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder() .parse(this.getClass().getClassLoader().getResourceAsStream("1-empty-1.0.xml")); call = call.addNamespace("myPrefix", "http://marklogic.com/test") .addVariable("myPrefix:myString", "Mars") diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java index a9d60fdeb..2adb961d1 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java @@ -1,11 +1,12 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.io.StringHandle; @@ -19,7 +20,6 @@ import org.w3c.dom.Element; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -54,10 +54,7 @@ private static void buildEnvironment() throws ParserConfigurationException { XMLDocumentManager docMgr = Common.client.newXMLDocumentManager(); DocumentWriteSet writeset =docMgr.newWriteSet(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElement("country"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java index 4ca6f1604..178ebc9f8 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -31,7 +31,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; @@ -124,10 +123,7 @@ public void testBuiltinReadWrite() String xmlDocId = "/test/testAs1.xml"; XMLDocumentManager xmlMgr = Common.client.newXMLDocumentManager(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder xmlDocBldr = factory.newDocumentBuilder(); + DocumentBuilder xmlDocBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document beforeDocument = xmlDocBldr.newDocument(); Element root = beforeDocument.createElement("doc"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java index 151d760ca..1c45af995 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.marklogic.client.MarkLogicIOException; import com.marklogic.client.expression.PlanBuilder; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.Format; import com.marklogic.client.row.RowManager; import com.marklogic.client.row.RowRecord; @@ -19,7 +20,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -131,11 +131,7 @@ private void checkXML(String testName, String kind, String expectedRaw, RowRecor // TODO: assertions on kind if set assertEquals(Format.XML, row.getContentFormat("t")); assertEquals("application/xml", row.getContentMimetype("t")); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setCoalescing(true); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document expected = builder.parse(new ByteArrayInputStream(expectedRaw.getBytes())); Document actual = row.getContentAs("t", Document.class); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java index 80c6a1852..603e8919a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -9,7 +9,9 @@ import com.marklogic.client.ResourceNotFoundException; import com.marklogic.client.ResourceNotResendableException; import com.marklogic.client.admin.QueryOptionsManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; +import jakarta.xml.bind.JAXBException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -17,14 +19,10 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.ls.DOMImplementationLS; -import org.xml.sax.SAXException; -import jakarta.xml.bind.JAXBException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.File; -import java.io.IOException; import static org.junit.jupiter.api.Assertions.*; @@ -60,14 +58,11 @@ public void testQueryOptionsManager() @Test public void testXMLDocsAsSearchOptions() - throws ParserConfigurationException, SAXException, IOException, ResourceNotFoundException, ForbiddenUserException, FailedRequestException, ResourceNotResendableException + throws ParserConfigurationException, ResourceNotFoundException, ForbiddenUserException, FailedRequestException, ResourceNotResendableException { String optionsName = "invalid"; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElementNS("http://marklogic.com/appservices/search","options"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java index 90cc36741..f8c8500c5 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -11,6 +11,7 @@ import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.document.JSONDocumentManager; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; import com.marklogic.client.io.marker.StructureWriteHandle; import com.marklogic.client.query.*; @@ -25,7 +26,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -541,10 +541,7 @@ public void test_issue581_RawStructuredQueryFromFileHandle() throws Exception { } private static Document parseXml(String xml) throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document document = builder.parse(new InputSource(new StringReader(xml))); return document; } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java index 8277a25ce..fe9810e8e 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java @@ -1,10 +1,11 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; import com.marklogic.client.document.XMLDocumentManager; import com.marklogic.client.impl.OutputStreamTee; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.io.StringHandle; @@ -20,11 +21,11 @@ import org.w3c.dom.ls.DOMImplementationLS; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.*; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class RequestLoggerTest { @BeforeAll @@ -81,13 +82,10 @@ public void testCopyTee() throws IOException { } @Test - public void testWriteReadLog() throws IOException, ParserConfigurationException { + public void testWriteReadLog() throws ParserConfigurationException { String docId = "/test/testWrite1.xml"; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java index fb55b5274..89909c199 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -8,6 +8,7 @@ import com.marklogic.client.ResourceNotFoundException; import com.marklogic.client.ResourceNotResendableException; import com.marklogic.client.admin.QueryOptionsManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.query.FacetResult; @@ -21,7 +22,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.io.StringReader; @@ -124,10 +124,7 @@ public void testFacetSearch() throws IOException, ParserConfigurationException, SAXException, FailedRequestException, ForbiddenUserException, ResourceNotFoundException, ResourceNotResendableException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document document = builder.parse(new InputSource(new StringReader(options))); mgr = Common.restAdminClient.newServerConfigManager().newQueryOptionsManager(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java index 6bfde88fe..44e1b16d0 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -27,7 +27,6 @@ import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLStreamException; @@ -81,10 +80,7 @@ public void testReadWrite() { String docId = "/test/testWrite1.xml"; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document domDocument = factory.newDocumentBuilder().newDocument(); + Document domDocument = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); root.setAttribute("foo", "bar"); @@ -355,10 +351,7 @@ public void testPatch() throws Exception { DocumentPatchHandle patchHandle = patchBldr.build(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); for (int i=0; i < 2; i++) { Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElement("root"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java index 48c6bb7c7..b332f3395 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.datamovement; @@ -88,7 +88,7 @@ public void scenario10() throws Exception { // TODO This test failed when 2022 became 2023; increasing -7 to a higher number fixed it. The test could obviously // use some rework to ensure that it doesn't fail every time the year changes, but this comment is being left here // so that if/when this does fail in the future, it'll be easy to fix. - date.roll(Calendar.YEAR, -10); + date.roll(Calendar.YEAR, -11); StructuredQueryBuilder sqb = new StructuredQueryBuilder(); StructuredQueryDefinition query = sqb.and( diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java index c20a2181c..ab2001c9a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java @@ -1,10 +1,11 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.datamovement; import com.marklogic.client.DatabaseClient; import com.marklogic.client.datamovement.*; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.test.Common; import org.junit.jupiter.api.AfterAll; @@ -14,7 +15,6 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; -import javax.xml.parsers.DocumentBuilderFactory; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -64,10 +64,8 @@ private class Message { public Map getBody() throws Exception { Map map = new HashMap<>(); map.put("uri", "http://marklogic.com/my/test/uri"); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document document = factory.newDocumentBuilder().newDocument(); + + Document document = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element element = document.createElement("test"); document.appendChild(element); map.put("content", document); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java index ac427b445..a2d5087eb 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.io; @@ -7,6 +7,7 @@ import com.marklogic.client.document.BinaryDocumentManager; import com.marklogic.client.document.DocumentManager.Metadata; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.DocumentMetadataHandle.*; import com.marklogic.client.io.FileHandle; @@ -27,7 +28,6 @@ import org.xml.sax.SAXException; import javax.xml.namespace.QName; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.File; import java.io.FileInputStream; @@ -88,10 +88,7 @@ public void testReadWriteMetadata() throws SAXException, IOException, XpathExcep ""+ ""; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document document = factory.newDocumentBuilder().newDocument(); + Document document = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element third = document.createElement("third"); Element child = document.createElement("third.first"); child.setTextContent("value third one"); From 86f5aa7a4c22368164a4192bb3e3ab0e7f2e3409 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 5 Jan 2026 15:02:55 -0500 Subject: [PATCH 30/70] MLE-26427 Initial exclusion support for XML Going to tackle error handling and unhappy path stuff next. More refactoring to be done as well. --- .../filter/ContentExclusionUtil.java | 80 +++++++++- .../filter/IncrementalWriteEvalFilter.java | 4 +- .../filter/IncrementalWriteFilter.java | 26 +++- .../filter/IncrementalWriteOpticFilter.java | 4 +- .../marklogic/client/impl/XmlFactories.java | 49 ++++++ .../com/marklogic/client/io/DOMHandle.java | 10 +- .../filter/AbstractIncrementalWriteTest.java | 53 +++++++ ...ApplyExclusionsToIncrementalWriteTest.java | 142 ++++++++++++++++++ .../filter/IncrementalWriteTest.java | 100 +----------- 9 files changed, 352 insertions(+), 116 deletions(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java index 0510cc0b0..a2d5c4fb8 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java @@ -8,8 +8,27 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.XmlFactories; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import java.io.ByteArrayInputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; /** * Utility class for applying content exclusions to documents before hash calculation. @@ -17,7 +36,7 @@ * * @since 8.1.0 */ -public class ContentExclusionUtil { +class ContentExclusionUtil { private static final Logger logger = LoggerFactory.getLogger(ContentExclusionUtil.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -31,7 +50,7 @@ public class ContentExclusionUtil { * @return the modified JSON content with specified paths removed * @throws JsonProcessingException if the JSON content cannot be parsed or serialized */ - public static String applyJsonExclusions(String uri, String jsonContent, String[] jsonPointers) throws JsonProcessingException { + static String applyJsonExclusions(String uri, String jsonContent, String[] jsonPointers) throws JsonProcessingException { if (jsonPointers == null || jsonPointers.length == 0) { return jsonContent; } @@ -72,6 +91,59 @@ private static void removeNodeAtPointer(String uri, JsonNode rootNode, String js } } - // Future method for XML exclusions - // public static String applyXmlExclusions(String xmlContent, String[] xpaths) { ... } + /** + * Applies XPath exclusions to XML content by removing the specified elements. + * + * @param uri the document URI (used for logging purposes) + * @param xmlContent the XML content as a string + * @param xpathExpressions array of XPath expressions identifying elements to exclude + * @return the modified XML content with specified elements removed + * @throws Exception if the XML content cannot be parsed or serialized + */ + static String applyXmlExclusions(String uri, String xmlContent, String... xpathExpressions) throws Exception { + if (xpathExpressions == null || xpathExpressions.length == 0) { + return xmlContent; + } + + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); + Document document = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8))); + applyXmlExclusions(uri, document, xpathExpressions); + return serializeDocument(document); + } + + private static void applyXmlExclusions(String uri, Document document, String[] xpathExpressions) { + final XPath xpath = XmlFactories.getXPathFactory().newXPath(); + for (String xpathExpression : xpathExpressions) { + try { + XPathExpression expr = xpath.compile(xpathExpression); + QName returnType = XPathConstants.NODESET; + NodeList nodes = (NodeList) expr.evaluate(document, returnType); + + if (nodes.getLength() == 0) { + logger.debug("XPath '{}' does not match any nodes in document {}, skipping", xpathExpression, uri); + continue; + } + + // Remove nodes in reverse order to avoid index issues + for (int i = nodes.getLength() - 1; i >= 0; i--) { + Node node = nodes.item(i); + Node parent = node.getParentNode(); + if (parent != null) { + parent.removeChild(node); + } + } + } catch (XPathExpressionException e) { + logger.warn("Invalid XPath expression '{}' for document {}: {}", xpathExpression, uri, e.getMessage()); + } + } + } + + private static String serializeDocument(Document document) throws TransformerException { + Transformer transformer = XmlFactories.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.setOutputProperty(OutputKeys.INDENT, "no"); + StringWriter writer = new StringWriter(); + transformer.transform(new DOMSource(document), new StreamResult(writer)); + return writer.toString(); + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index 54343d80e..c48d95273 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -31,8 +31,8 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { """; IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 86cc14e62..46dd06cd8 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -46,6 +46,7 @@ public static class Builder { private boolean useEvalQuery = false; private Consumer skippedDocumentsConsumer; private String[] jsonExclusions; + private String[] xmlExclusions; /** * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". @@ -103,11 +104,20 @@ public Builder jsonExclusions(String... jsonPointers) { return this; } + /** + * @param xpathExpressions XPath expressions identifying XML elements to exclude from hash calculation. + * For example, "//timestamp" or "//metadata/lastModified". + */ + public Builder xmlExclusions(String... xpathExpressions) { + this.xmlExclusions = xpathExpressions; + return this; + } + public IncrementalWriteFilter build() { if (useEvalQuery) { - return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } - return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } } @@ -116,17 +126,19 @@ public IncrementalWriteFilter build() { private final boolean canonicalizeJson; private final Consumer skippedDocumentsConsumer; private final String[] jsonExclusions; + private final String[] xmlExclusions; // Hardcoding this for now, with a good general purpose hashing function. // See https://xxhash.com for benchmarks. private final LongHashFunction hashFunction = LongHashFunction.xx3(); - public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions) { + public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { this.hashKeyName = hashKeyName; this.timestampKeyName = timestampKeyName; this.canonicalizeJson = canonicalizeJson; this.skippedDocumentsConsumer = skippedDocumentsConsumer; this.jsonExclusions = jsonExclusions; + this.xmlExclusions = xmlExclusions; } protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { @@ -178,7 +190,6 @@ private String serializeContent(DocumentWriteOperation doc) { JsonCanonicalizer jc; try { if (jsonExclusions != null && jsonExclusions.length > 0) { - // TBD on error handling here, want to get XML supported first. content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, jsonExclusions); } jc = new JsonCanonicalizer(content); @@ -190,6 +201,13 @@ private String serializeContent(DocumentWriteOperation doc) { logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); } + } else if (xmlExclusions != null && xmlExclusions.length > 0) { + try { + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, xmlExclusions); + } catch (Exception e) { + logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", + doc.getUri(), e.getMessage()); + } } return content; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java index 3cb8f44e0..d760f3ab4 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -20,8 +20,8 @@ class IncrementalWriteOpticFilter extends IncrementalWriteFilter { IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions); + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java index 6773ef19f..5f931dfba 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java @@ -12,8 +12,10 @@ import javax.xml.stream.FactoryConfigurationError; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; +import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; +import javax.xml.xpath.XPathFactory; import java.lang.ref.SoftReference; import java.util.function.Supplier; @@ -27,6 +29,12 @@ public final class XmlFactories { private static final CachedInstancePerThreadSupplier cachedDocumentBuilderFactory = new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewDocumentBuilderFactory); + private static final CachedInstancePerThreadSupplier cachedXPathFactory = + new CachedInstancePerThreadSupplier<>(XPathFactory::newInstance); + + private static final CachedInstancePerThreadSupplier cachedTransformerFactory = + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewTransformerFactory); + private XmlFactories() {} // preventing instances of utility class /** @@ -152,6 +160,47 @@ public static XMLOutputFactory getOutputFactory() { return cachedOutputFactory.get(); } + /** + * Returns a shared {@link XPathFactory}. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a {@link XPathFactory} + * + * @since 8.1.0 + */ + public static XPathFactory getXPathFactory() { + return cachedXPathFactory.get(); + } + + /** + * Returns a shared {@link TransformerFactory} configured with secure defaults. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a securely configured {@link TransformerFactory} + * + * @since 8.1.0 + */ + public static TransformerFactory getTransformerFactory() { + return cachedTransformerFactory.get(); + } + + /** + * Creates a new {@link Transformer} from the shared {@link TransformerFactory}. + * + * @since 8.1.0 + */ + public static Transformer newTransformer() { + try { + return getTransformerFactory().newTransformer(); + } catch (TransformerConfigurationException e) { + throw new RuntimeException("Unable to create new Transformer from TransformerFactory", e); + } + } + /** * A supplier that caches results per thread. *

diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java index 90d8c0fc4..a19dd138e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java @@ -18,7 +18,10 @@ import javax.xml.namespace.QName; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.*; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; import java.io.*; import java.nio.charset.StandardCharsets; @@ -205,7 +208,7 @@ public void setFactory(DocumentBuilderFactory factory) { */ public XPath getXPathProcessor() { if (xpathProcessor == null) - xpathProcessor = makeXPathProcessorFactory().newXPath(); + xpathProcessor = XmlFactories.getXPathFactory().newXPath(); return xpathProcessor; } /** @@ -216,9 +219,6 @@ public XPath getXPathProcessor() { public void setXPathProcessor(XPath xpathProcessor) { this.xpathProcessor = xpathProcessor; } - protected XPathFactory makeXPathProcessorFactory() { - return XPathFactory.newInstance(); - } /** * Evaluate a string XPath expression against the retrieved document. diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java new file mode 100644 index 000000000..6c136f0e5 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.BeforeEach; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +abstract class AbstractIncrementalWriteTest extends AbstractClientTest { + + static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + AtomicInteger writtenCount = new AtomicInteger(); + AtomicInteger skippedCount = new AtomicInteger(); + AtomicReference batchFailure = new AtomicReference<>(); + ObjectMapper objectMapper = new ObjectMapper(); + + List docs = new ArrayList<>(); + IncrementalWriteFilter filter; + + @BeforeEach + void setup() { + // Need a user with eval privileges so that the eval filter can be tested. + Common.client = Common.newEvalClient(); + + // Default filter implementation, should be suitable for most tests. + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + } + + final void writeDocs(List docs) { + new WriteBatcherTemplate(Common.client).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(filter) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) + .onBatchFailure((batch, failure) -> batchFailure.set(failure)), + + writeBatcher -> docs.forEach(writeBatcher::add) + ); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java new file mode 100644 index 000000000..19f5dd339 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ApplyExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest { + + @Test + void jsonExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .jsonExclusions("/timestamp", "/metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents with three keys + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2025-01-01T10:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2025-01-01T10:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed + doc.putObject("metadata") + .put("lastModified", "2026-01-02T15:30:00Z") // Changed + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Modified Document " + i); // Changed + doc.put("timestamp", "2026-01-02T16:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2026-01-02T16:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } + + @Test + void xmlExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .xmlExclusions("//timestamp", "//metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = "" + + "" + i + "" + + "Document " + i + "" + + "2025-01-01T10:00:00Z" + + "" + + "Test User" + + "2025-01-01T10:00:00Z" + + "" + + ""; + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = "" + + "" + i + "" + + "Document " + i + "" + + "2026-01-02T15:30:00Z" + // Changed + "" + + "Test User" + + "2026-01-02T15:30:00Z" + // Changed + "" + + ""; + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = "" + + "" + i + "" + + "Modified Document " + i + "" + // Changed + "2026-01-02T16:00:00Z" + + "" + + "Test User" + + "2026-01-02T16:00:00Z" + + "" + + ""; + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 9929fdc98..723296b72 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -3,7 +3,6 @@ */ package com.marklogic.client.datamovement.filter; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.marklogic.client.document.*; import com.marklogic.client.impl.DocumentWriteOperationImpl; @@ -11,42 +10,15 @@ import com.marklogic.client.io.Format; import com.marklogic.client.io.JacksonHandle; import com.marklogic.client.io.StringHandle; -import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.*; -class IncrementalWriteTest extends AbstractClientTest { - - private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() - .withCollections("incremental-test") - .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); - - AtomicInteger writtenCount = new AtomicInteger(); - AtomicInteger skippedCount = new AtomicInteger(); - AtomicReference batchFailure = new AtomicReference<>(); - ObjectMapper objectMapper = new ObjectMapper(); - - List docs = new ArrayList<>(); - IncrementalWriteFilter filter; - - @BeforeEach - void setup() { - // Need a user with eval privileges so that the eval filter can be tested. - Common.client = Common.newEvalClient(); - - // Default filter implementation, should be suitable for most tests. - filter = IncrementalWriteFilter.newBuilder() - .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) - .build(); - } +class IncrementalWriteTest extends AbstractIncrementalWriteTest { @Test void opticFilter() { @@ -218,65 +190,6 @@ void nullIsIgnoredForKeyNames() { assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); } - @Test - void jsonExclusions() { - filter = IncrementalWriteFilter.newBuilder() - .jsonExclusions("/timestamp", "/metadata/lastModified") - .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) - .build(); - - // Write initial documents with three keys - docs = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - ObjectNode doc = objectMapper.createObjectNode(); - doc.put("id", i); - doc.put("name", "Document " + i); - doc.put("timestamp", "2025-01-01T10:00:00Z"); - doc.putObject("metadata") - .put("lastModified", "2025-01-01T10:00:00Z") - .put("author", "Test User"); - docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); - } - - writeDocs(docs); - assertEquals(5, writtenCount.get()); - assertEquals(0, skippedCount.get()); - - // Write again with different values for excluded fields - should be skipped - docs = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - ObjectNode doc = objectMapper.createObjectNode(); - doc.put("id", i); - doc.put("name", "Document " + i); - doc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed - doc.putObject("metadata") - .put("lastModified", "2026-01-02T15:30:00Z") // Changed - .put("author", "Test User"); - docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); - } - - writeDocs(docs); - assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); - assertEquals(5, skippedCount.get()); - - // Write again with actual content change - should NOT be skipped - docs = new ArrayList<>(); - for (int i = 1; i <= 5; i++) { - ObjectNode doc = objectMapper.createObjectNode(); - doc.put("id", i); - doc.put("name", "Modified Document " + i); // Changed - doc.put("timestamp", "2026-01-02T16:00:00Z"); - doc.putObject("metadata") - .put("lastModified", "2026-01-02T16:00:00Z") - .put("author", "Test User"); - docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); - } - - writeDocs(docs); - assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); - assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); - } - private void verifyIncrementalWriteWorks() { writeTenDocuments(); verifyDocumentsHasHashInMetadataKey(); @@ -337,15 +250,4 @@ private void modifyFiveDocuments() { } writeDocs(docs); } - - private void writeDocs(List docs) { - new WriteBatcherTemplate(Common.client).runWriteJob( - writeBatcher -> writeBatcher - .withDocumentWriteSetFilter(filter) - .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) - .onBatchFailure((batch, failure) -> batchFailure.set(failure)), - - writeBatcher -> docs.forEach(writeBatcher::add) - ); - } } From 95b274ca93311be23a6171e42741f1a136402f52 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 15 Jan 2026 14:31:39 -0500 Subject: [PATCH 31/70] MLE-26460 Fixing test failures Some test failures on Jenkins due to a test plumbing issue. --- .../com/marklogic/client/test/AbstractClientTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java index 0f9c5af8a..2b454103c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -20,9 +20,14 @@ protected final DatabaseClient getDatabaseClient() { @Override protected final String getJavascriptForDeletingDocumentsBeforeTestRuns() { + // The "/acme/" directory was previously deleted by AbstractOpticUpdateTest. It still needs to be deleted + // since some tests end up copying URIs to that directory but retain the 'test-data' collection. return """ declareUpdate(); - cts.uris('', [], cts.notQuery(cts.collectionQuery(['test-data', 'temporal-collection']))) + cts.uris('', [], cts.orQuery([ + cts.notQuery(cts.collectionQuery(['test-data', 'temporal-collection'])), + cts.directoryQuery('/acme/', 'infinity') + ])) .toArray().forEach(item => xdmp.documentDelete(item)) """; } From 79887f082f3dfbbe245ea5a19b760b85133d794f Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 16 Jan 2026 10:55:00 -0500 Subject: [PATCH 32/70] MLE-26427 Fixing Polaris issues Most of these already existed in OkHttpServices, but Polaris decided to report them recently --- .../filter/ContentExclusionUtil.java | 7 +++-- .../filter/IncrementalWriteFilter.java | 12 ++++++-- .../marklogic/client/impl/OkHttpServices.java | 28 ++++++++++++++----- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java index a2d5c4fb8..cd2ba9618 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java @@ -83,8 +83,11 @@ private static void removeNodeAtPointer(String uri, JsonNode rootNode, String js JsonNode parentNode = rootNode.at(parentPointer); if (parentNode.isObject()) { - String fieldName = pointer.last().getMatchingProperty(); - ((ObjectNode) parentNode).remove(fieldName); + JsonPointer lastSegment = pointer.last(); + if (lastSegment != null) { + String fieldName = lastSegment.getMatchingProperty(); + ((ObjectNode) parentNode).remove(fieldName); + } } else if (parentNode.isArray()) { logger.warn("Array element exclusion not supported for JSONPointer '{}'. " + "Consider excluding the entire array property instead.", jsonPointer); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 46dd06cd8..73f4fd5f9 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -152,7 +152,15 @@ protected final DocumentWriteSet filterDocuments(Context context, Function cookies = new ArrayList<>(); + final String location = response.headers().get("Location"); + + final List cookies = new ArrayList<>(); for (String setCookie : response.headers(HEADER_SET_COOKIE)) { ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); cookies.add(cookie); } + closeResponse(response); if (location == null) throw new MarkLogicInternalException("transaction open failed to provide location"); if (!location.contains("/")) { @@ -2562,6 +2567,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to read " @@ -2749,6 +2755,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to delete " @@ -3181,6 +3188,7 @@ public Response apply(Request.Builder funcBuilder) { }; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + Objects.requireNonNull(response); int status = response.code(); checkStatus(response, status, operation, "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); @@ -3712,6 +3720,7 @@ public RESTServiceResultIterator postMultipartForm( Consumer resendableConsumer = null; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + Objects.requireNonNull(response); int status = response.code(); checkStatus(response, status, "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); return makeResults(OkHttpServiceResultIterator::new, reqlog, "apply", "resource", response); @@ -3782,6 +3791,7 @@ private U postIteratedResourceImpl( ); Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + Objects.requireNonNull(response); checkStatus(response, response.code(), "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); boolean shouldStreamResults = "eval".equalsIgnoreCase(path) || "invoke".equalsIgnoreCase(path); @@ -4820,6 +4830,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException( @@ -5040,7 +5051,8 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); - int status = response.code(); + Objects.requireNonNull(response); + final int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to match", extractErrorFields(response)); @@ -5618,9 +5630,11 @@ private void executeRequest(CallResponseImpl responseImpl) { if (session != null) { List cookies = new ArrayList<>(); - for (String setCookie : response.headers(HEADER_SET_COOKIE)) { - ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); - cookies.add(cookie); + if (response != null) { + for (String setCookie : response.headers(HEADER_SET_COOKIE)) { + ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); + cookies.add(cookie); + } } ((SessionStateImpl) session).setCookies(cookies); } From af01d9a8542c603ad57a184395f014505e36a3ce Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 16 Jan 2026 12:24:04 -0500 Subject: [PATCH 33/70] MLE-26427 More Polaris fixes --- .../datamovement/filter/IncrementalWriteFilter.java | 3 +++ .../java/com/marklogic/client/impl/OkHttpServices.java | 10 ++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 73f4fd5f9..a78759d50 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -188,6 +188,9 @@ protected final DocumentWriteSet filterDocuments(Context context, Function baseHandle) { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index dc28aa80f..dd24b8dab 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -5627,14 +5627,12 @@ private void executeRequest(CallResponseImpl responseImpl) { }; Response response = sendRequestWithRetry(requestBldr, sendRequestFunction, resendableConsumer); - + Objects.requireNonNull(response); if (session != null) { List cookies = new ArrayList<>(); - if (response != null) { - for (String setCookie : response.headers(HEADER_SET_COOKIE)) { - ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); - cookies.add(cookie); - } + for (String setCookie : response.headers(HEADER_SET_COOKIE)) { + ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); + cookies.add(cookie); } ((SessionStateImpl) session).setCookies(cookies); } From e6093815bd082b2137b855b1f46f46dc205200c2 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 16 Jan 2026 12:20:56 -0500 Subject: [PATCH 34/70] MLE-26427 Added unhappy test cases for exclusions And a couple tests for JSON canonicalization --- .../filter/IncrementalWriteFilter.java | 46 ++++++++ ...ApplyExclusionsToIncrementalWriteTest.java | 103 ++++++++++++++++++ ...validExclusionsToIncrementalWriteTest.java | 66 +++++++++++ 3 files changed, 215 insertions(+) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index a78759d50..f7de86a12 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -3,11 +3,13 @@ */ package com.marklogic.client.datamovement.filter; +import com.fasterxml.jackson.core.JsonPointer; import com.marklogic.client.datamovement.DocumentWriteSetFilter; import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.impl.HandleAccessor; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.BaseHandle; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; @@ -16,6 +18,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Instant; @@ -114,11 +118,53 @@ public Builder xmlExclusions(String... xpathExpressions) { } public IncrementalWriteFilter build() { + validateJsonExclusions(); + validateXmlExclusions(); if (useEvalQuery) { return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); } + + private void validateJsonExclusions() { + if (jsonExclusions == null) { + return; + } + for (String jsonPointer : jsonExclusions) { + if (jsonPointer == null || jsonPointer.trim().isEmpty()) { + throw new IllegalArgumentException( + "Empty JSON Pointer expression is not valid for excluding content from incremental write hash calculation; " + + "it would exclude the entire document. JSON Pointer expressions must start with '/'."); + } + try { + JsonPointer.compile(jsonPointer); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format("Invalid JSON Pointer expression '%s' for excluding content from incremental write hash calculation. " + + "JSON Pointer expressions must start with '/'; cause: %s", jsonPointer, e.getMessage()), e); + } + } + } + + private void validateXmlExclusions() { + if (xmlExclusions == null) { + return; + } + XPath xpath = XmlFactories.getXPathFactory().newXPath(); + for (String xpathExpression : xmlExclusions) { + if (xpathExpression == null || xpathExpression.trim().isEmpty()) { + throw new IllegalArgumentException( + "Empty XPath expression is not valid for excluding content from incremental write hash calculation."); + } + try { + xpath.compile(xpathExpression); + } catch (XPathExpressionException e) { + throw new IllegalArgumentException( + String.format("Invalid XPath expression '%s' for excluding content from incremental write hash calculation; cause: %s", + xpathExpression, e.getMessage()), e); + } + } + } } protected final String hashKeyName; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java index 19f5dd339..4c418e578 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java @@ -139,4 +139,107 @@ void xmlExclusions() { assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); } + + /** + * Verifies that JSON Pointer exclusions are only applied to JSON documents and are ignored for XML documents. + * The XML document should use its full content for hashing since no XML exclusions are configured. + */ + @Test + void jsonExclusionsIgnoredForXmlDocuments() { + filter = IncrementalWriteFilter.newBuilder() + .jsonExclusions("/timestamp") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write one JSON doc and one XML doc + docs = new ArrayList<>(); + ObjectNode jsonDoc = objectMapper.createObjectNode(); + jsonDoc.put("id", 1); + jsonDoc.put("timestamp", "2025-01-01T10:00:00Z"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc))); + + String xmlDoc = "12025-01-01T10:00:00Z"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML))); + + writeDocs(docs); + assertEquals(2, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different timestamp values + docs = new ArrayList<>(); + jsonDoc = objectMapper.createObjectNode(); + jsonDoc.put("id", 1); + jsonDoc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc))); + + xmlDoc = "12026-01-02T15:30:00Z"; // Changed + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML))); + + writeDocs(docs); + assertEquals(3, writtenCount.get(), "XML doc should be written since its timestamp changed and no XML exclusions are configured"); + assertEquals(1, skippedCount.get(), "JSON doc should be skipped since only the excluded timestamp field changed"); + } + + /** + * Verifies that when canonicalizeJson is false, documents with logically identical content + * but different key ordering will produce different hashes, causing a write to occur. + */ + @Test + void jsonNotCanonicalizedCausesDifferentHashForReorderedKeys() { + filter = IncrementalWriteFilter.newBuilder() + .canonicalizeJson(false) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial document with keys in a specific order + docs = new ArrayList<>(); + String json1 = "{\"name\":\"Test\",\"id\":1,\"value\":100}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/non-canonical.json", METADATA, + new StringHandle(json1).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with same logical content but different key order + docs = new ArrayList<>(); + String json2 = "{\"id\":1,\"value\":100,\"name\":\"Test\"}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/non-canonical.json", METADATA, + new StringHandle(json2).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(2, writtenCount.get(), "Document should be written because key order differs and JSON is not canonicalized"); + assertEquals(0, skippedCount.get(), "No documents should be skipped"); + } + + /** + * Verifies that with the default canonicalizeJson(true), documents with logically identical content + * but different key ordering will produce the same hash, causing the document to be skipped. + */ + @Test + void jsonCanonicalizedProducesSameHashForReorderedKeys() { + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial document with keys in a specific order + docs = new ArrayList<>(); + String json1 = "{\"name\":\"Test\",\"id\":1,\"value\":100}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/canonical.json", METADATA, + new StringHandle(json1).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with same logical content but different key order + docs = new ArrayList<>(); + String json2 = "{\"id\":1,\"value\":100,\"name\":\"Test\"}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/canonical.json", METADATA, + new StringHandle(json2).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(1, writtenCount.get(), "Document should be skipped because canonicalized JSON produces the same hash"); + assertEquals(1, skippedCount.get(), "One document should be skipped"); + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java new file mode 100644 index 000000000..c71eaa90b --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ApplyInvalidExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest { + + /** + * Verifies that an invalid JSON Pointer expression (missing leading slash) causes the build to fail + * immediately, allowing the user to fix the configuration before any documents are processed. + */ + @Test + void invalidJsonPointerExpression() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder() + .jsonExclusions("timestamp"); // Invalid - missing leading slash + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + + assertTrue(ex.getMessage().contains("Invalid JSON Pointer expression 'timestamp'"), + "Error message should include the invalid expression. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("incremental write"), + "Error message should mention incremental write context. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("must start with '/'"), + "Error message should hint at the fix. Actual: " + ex.getMessage()); + } + + /** + * Verifies that an empty JSON Pointer expression is rejected since it would exclude the entire document, + * leaving nothing to hash. + */ + @Test + void emptyJsonPointerExpression() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder() + .jsonExclusions(""); // Invalid - would exclude entire document + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + + assertTrue(ex.getMessage().contains("Empty JSON Pointer expression"), + "Error message should indicate empty expression. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("would exclude the entire document"), + "Error message should explain why it's invalid. Actual: " + ex.getMessage()); + } + + /** + * Verifies that an invalid XPath expression causes the build to fail immediately, + * allowing the user to fix the configuration before any documents are processed. + */ + @Test + void invalidXPathExpression() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder() + .xmlExclusions("[[[invalid xpath"); // Invalid XPath syntax + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + + assertTrue(ex.getMessage().contains("Invalid XPath expression '[[[invalid xpath'"), + "Error message should include the invalid expression. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("incremental write"), + "Error message should mention incremental write context. Actual: " + ex.getMessage()); + } + +} From cc59358c00117d2f442139f7c3b4b4773d107ab2 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 16 Jan 2026 15:23:46 -0500 Subject: [PATCH 35/70] MLE-26427 Namespace support for exclusions gitflow-feature-stash: namespacaes --- .../filter/ContentExclusionUtil.java | 14 +- .../filter/IncrementalWriteEvalFilter.java | 5 +- .../filter/IncrementalWriteFilter.java | 24 ++- .../filter/IncrementalWriteOpticFilter.java | 4 +- .../filter/SimpleNamespaceContext.java | 46 ++++++ ...ApplyExclusionsToIncrementalWriteTest.java | 149 ++++++++++++++---- 6 files changed, 201 insertions(+), 41 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java index cd2ba9618..39cd5aee2 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java @@ -15,6 +15,7 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import javax.xml.namespace.NamespaceContext; import javax.xml.namespace.QName; import javax.xml.parsers.DocumentBuilder; import javax.xml.transform.OutputKeys; @@ -29,6 +30,8 @@ import java.io.ByteArrayInputStream; import java.io.StringWriter; import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.Map; /** * Utility class for applying content exclusions to documents before hash calculation. @@ -99,23 +102,28 @@ private static void removeNodeAtPointer(String uri, JsonNode rootNode, String js * * @param uri the document URI (used for logging purposes) * @param xmlContent the XML content as a string + * @param namespaces a map of namespace prefixes to URIs for use in XPath expressions, or null * @param xpathExpressions array of XPath expressions identifying elements to exclude * @return the modified XML content with specified elements removed * @throws Exception if the XML content cannot be parsed or serialized */ - static String applyXmlExclusions(String uri, String xmlContent, String... xpathExpressions) throws Exception { + static String applyXmlExclusions(String uri, String xmlContent, Map namespaces, String... xpathExpressions) throws Exception { if (xpathExpressions == null || xpathExpressions.length == 0) { return xmlContent; } DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document document = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8))); - applyXmlExclusions(uri, document, xpathExpressions); + applyXmlExclusions(uri, document, namespaces, xpathExpressions); return serializeDocument(document); } - private static void applyXmlExclusions(String uri, Document document, String[] xpathExpressions) { + private static void applyXmlExclusions(String uri, Document document, Map namespaces, String[] xpathExpressions) { final XPath xpath = XmlFactories.getXPathFactory().newXPath(); + if (namespaces != null && !namespaces.isEmpty()) { + xpath.setNamespaceContext(new SimpleNamespaceContext(namespaces)); + } + for (String xpathExpression : xpathExpressions) { try { XPathExpression expr = xpath.compile(xpathExpression); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index c48d95273..fc0546798 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -12,6 +12,7 @@ import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.io.JacksonHandle; +import java.util.Map; import java.util.function.Consumer; /** @@ -31,8 +32,8 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { """; IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index f7de86a12..730910c0b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -25,6 +25,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; @@ -51,6 +52,7 @@ public static class Builder { private Consumer skippedDocumentsConsumer; private String[] jsonExclusions; private String[] xmlExclusions; + private Map xmlNamespaces; /** * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". @@ -117,13 +119,22 @@ public Builder xmlExclusions(String... xpathExpressions) { return this; } + /** + * @param namespaces a map of namespace prefixes to URIs for use in XPath exclusion expressions. + * For example, Map.of("ns", "http://example.com/ns") allows XPath like "//ns:timestamp". + */ + public Builder xmlNamespaces(Map namespaces) { + this.xmlNamespaces = namespaces; + return this; + } + public IncrementalWriteFilter build() { validateJsonExclusions(); validateXmlExclusions(); if (useEvalQuery) { - return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); + return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); } - return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); + return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); } private void validateJsonExclusions() { @@ -151,6 +162,9 @@ private void validateXmlExclusions() { return; } XPath xpath = XmlFactories.getXPathFactory().newXPath(); + if (xmlNamespaces != null && !xmlNamespaces.isEmpty()) { + xpath.setNamespaceContext(new SimpleNamespaceContext(xmlNamespaces)); + } for (String xpathExpression : xmlExclusions) { if (xpathExpression == null || xpathExpression.trim().isEmpty()) { throw new IllegalArgumentException( @@ -173,18 +187,20 @@ private void validateXmlExclusions() { private final Consumer skippedDocumentsConsumer; private final String[] jsonExclusions; private final String[] xmlExclusions; + private final Map xmlNamespaces; // Hardcoding this for now, with a good general purpose hashing function. // See https://xxhash.com for benchmarks. private final LongHashFunction hashFunction = LongHashFunction.xx3(); - public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { + public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { this.hashKeyName = hashKeyName; this.timestampKeyName = timestampKeyName; this.canonicalizeJson = canonicalizeJson; this.skippedDocumentsConsumer = skippedDocumentsConsumer; this.jsonExclusions = jsonExclusions; this.xmlExclusions = xmlExclusions; + this.xmlNamespaces = xmlNamespaces; } protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { @@ -260,7 +276,7 @@ private String serializeContent(DocumentWriteOperation doc) { } } else if (xmlExclusions != null && xmlExclusions.length > 0) { try { - content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, xmlExclusions); + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, xmlNamespaces, xmlExclusions); } catch (Exception e) { logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java index d760f3ab4..a52d21ad3 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -20,8 +20,8 @@ class IncrementalWriteOpticFilter extends IncrementalWriteFilter { IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions); + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); } @Override diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java new file mode 100644 index 000000000..196b36d71 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import javax.xml.namespace.NamespaceContext; +import java.util.Iterator; +import java.util.Map; + +/** + * A simple implementation of {@link NamespaceContext} backed by a Map of prefix to namespace URI mappings. + * Used for XPath evaluation with namespace-qualified expressions. + * + * @since 8.1.0 + */ +class SimpleNamespaceContext implements NamespaceContext { + + private final Map prefixToNamespaceUri; + + SimpleNamespaceContext(Map prefixToNamespaceUri) { + this.prefixToNamespaceUri = prefixToNamespaceUri; + } + + @Override + public String getNamespaceURI(String prefix) { + return prefixToNamespaceUri.get(prefix); + } + + @Override + public String getPrefix(String namespaceURI) { + for (Map.Entry entry : prefixToNamespaceUri.entrySet()) { + if (entry.getValue().equals(namespaceURI)) { + return entry.getKey(); + } + } + return null; + } + + @Override + public Iterator getPrefixes(String namespaceURI) { + return prefixToNamespaceUri.entrySet().stream() + .filter(entry -> entry.getValue().equals(namespaceURI)) + .map(Map.Entry::getKey) + .iterator(); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java index 4c418e578..1597ef6df 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java @@ -11,8 +11,9 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; class ApplyExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest { @@ -85,15 +86,17 @@ void xmlExclusions() { // Write initial documents docs = new ArrayList<>(); for (int i = 1; i <= 5; i++) { - String xml = "" + - "" + i + "" + - "Document " + i + "" + - "2025-01-01T10:00:00Z" + - "" + - "Test User" + - "2025-01-01T10:00:00Z" + - "" + - ""; + String xml = """ + + %d + Document %d + 2025-01-01T10:00:00Z + + Test User + 2025-01-01T10:00:00Z + + + """.formatted(i, i); docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); } @@ -104,15 +107,17 @@ void xmlExclusions() { // Write again with different values for excluded fields - should be skipped docs = new ArrayList<>(); for (int i = 1; i <= 5; i++) { - String xml = "" + - "" + i + "" + - "Document " + i + "" + - "2026-01-02T15:30:00Z" + // Changed - "" + - "Test User" + - "2026-01-02T15:30:00Z" + // Changed - "" + - ""; + String xml = """ + + %d + Document %d + 2026-01-02T15:30:00Z + + Test User + 2026-01-02T15:30:00Z + + + """.formatted(i, i); docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); } @@ -123,15 +128,17 @@ void xmlExclusions() { // Write again with actual content change - should NOT be skipped docs = new ArrayList<>(); for (int i = 1; i <= 5; i++) { - String xml = "" + - "" + i + "" + - "Modified Document " + i + "" + // Changed - "2026-01-02T16:00:00Z" + - "" + - "Test User" + - "2026-01-02T16:00:00Z" + - "" + - ""; + String xml = """ + + %d + Modified Document %d + 2026-01-02T16:00:00Z + + Test User + 2026-01-02T16:00:00Z + + + """.formatted(i, i); docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); } @@ -158,7 +165,12 @@ void jsonExclusionsIgnoredForXmlDocuments() { jsonDoc.put("timestamp", "2025-01-01T10:00:00Z"); docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc))); - String xmlDoc = "12025-01-01T10:00:00Z"; + String xmlDoc = """ + + 1 + 2025-01-01T10:00:00Z + + """; docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML))); writeDocs(docs); @@ -172,7 +184,12 @@ void jsonExclusionsIgnoredForXmlDocuments() { jsonDoc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc))); - xmlDoc = "12026-01-02T15:30:00Z"; // Changed + xmlDoc = """ + + 1 + 2026-01-02T15:30:00Z + + """; docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML))); writeDocs(docs); @@ -242,4 +259,76 @@ void jsonCanonicalizedProducesSameHashForReorderedKeys() { assertEquals(1, writtenCount.get(), "Document should be skipped because canonicalized JSON produces the same hash"); assertEquals(1, skippedCount.get(), "One document should be skipped"); } + + @Test + void xmlExclusionsWithNamespaces() { + filter = IncrementalWriteFilter.newBuilder() + .xmlExclusions("//ns:timestamp", "//ns:metadata/ns:lastModified") + .xmlNamespaces(Map.of("ns", "http://example.com/ns")) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents with namespaced elements + docs = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + String xml = """ + + %d + Document %d + 2025-01-01T10:00:00Z + + Test User + 2025-01-01T10:00:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/ns-xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(3, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + String xml = """ + + %d + Document %d + 2026-01-02T15:30:00Z + + Test User + 2026-01-02T15:30:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/ns-xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(3, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(3, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + String xml = """ + + %d + Modified Document %d + 2026-01-02T16:00:00Z + + Test User + 2026-01-02T16:00:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/ns-xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(6, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(3, skippedCount.get(), "Skip count should remain at 3"); + } } From 4964dc7a57d422552db25f8cb034c5f8683b9542 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 20 Jan 2026 13:18:05 -0500 Subject: [PATCH 36/70] MLE-26654 Renaming of the retry-on-status-code feature This is currently an implementation detail and not exposed to a user, but doing some renaming here so it's easier for other developers to understand. --- .../marklogic/client/impl/OkHttpServices.java | 117 +++++++++--------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index dd24b8dab..3d367aa2e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -106,10 +106,17 @@ public class OkHttpServices implements RESTServices { * in several places and is slightly different in each place. It's also not possible to implement this logic in an * OkHttp interceptor as the logic needs access to details that are not available to an interceptor. */ - private final Random randRetry = new Random(); - private int maxDelay = DEFAULT_MAX_DELAY; - private int minRetry = DEFAULT_MIN_RETRY; - private final Set retryStatus = new HashSet<>(); + private final Random randomForRetryDelay = new Random(); + + // The maximum amount of time to spend retrying requests. + private int maxDelayForRetries = DEFAULT_MAX_DELAY; + + // The minimum number of retry attempts to make regardless of the max delay. + private int minRetryAttempts = DEFAULT_MIN_RETRY; + + // The HTTP status codes that are retryable. + private static final Set RETRYABLE_STATUS_CODES = + Set.of(STATUS_BAD_GATEWAY, STATUS_SERVICE_UNAVAILABLE, STATUS_GATEWAY_TIMEOUT); private boolean checkFirstRequest = true; @@ -129,10 +136,6 @@ public record ConnectionConfig(String host, int port, String basePath, String da } public OkHttpServices(ConnectionConfig connectionConfig) { - retryStatus.add(STATUS_BAD_GATEWAY); - retryStatus.add(STATUS_SERVICE_UNAVAILABLE); - retryStatus.add(STATUS_GATEWAY_TIMEOUT); - this.okHttpClient = connect(connectionConfig); } @@ -239,13 +242,13 @@ private void configureDelayAndRetry(Properties props) { if (props.containsKey(MAX_DELAY_PROP)) { int max = Utilities.parseInt(props.getProperty(MAX_DELAY_PROP)); if (max > 0) { - maxDelay = max * 1000; + maxDelayForRetries = max * 1000; } } if (props.containsKey(MIN_RETRY_PROP)) { int min = Utilities.parseInt(props.getProperty(MIN_RETRY_PROP)); if (min > 0) { - minRetry = min; + minRetryAttempts = min; } } } @@ -289,7 +292,7 @@ private int makeFirstRequest(int retry) { private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { Response response = sendRequestOnce(setupRequest(requestUri, path, null).head()); int statusCode = response.code(); - if (!retryStatus.contains(statusCode)) { + if (!RETRYABLE_STATUS_CODES.contains(statusCode)) { closeResponse(response); return 0; } @@ -298,7 +301,7 @@ private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { closeResponse(response); int retryAfter = Utilities.parseInt(retryAfterRaw); - return Math.max(retryAfter, calculateDelay(randRetry, retry)); + return Math.max(retryAfter, calculateDelay(retry)); } private RequestParameters addTemporalProtectionParams(RequestParameters params, String uri, ProtectionLevel level, @@ -518,7 +521,7 @@ private Response sendRequestWithRetry( /* * This loop is for retrying the request if the service is unavailable */ - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -537,7 +540,7 @@ private Response sendRequestWithRetry( ); } status = response.code(); - if (!isRetryable || !retryStatus.contains(status)) { + if (!isRetryable || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); /* * If we don't get a service unavailable status or if the request @@ -562,13 +565,13 @@ private Response sendRequestWithRetry( /* * Calculate the delay before which we shouldn't retry */ - nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(randRetry, retry)); + nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(retry)); } /* * If the service is still unavailable after all the retries, we throw a * FailedRetryException indicating that the service is unavailable. */ - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -1198,7 +1201,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -1238,7 +1241,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth status = response.code(); responseHeaders = response.headers(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -1255,9 +1258,9 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -1359,7 +1362,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -1382,7 +1385,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth status = response.code(); responseHeaders = response.headers(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -1397,9 +1400,9 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -2092,7 +2095,7 @@ Response getResponse() { long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -2121,7 +2124,7 @@ Response getResponse() { status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -2132,9 +2135,9 @@ Response getResponse() { closeResponse(response); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -2631,7 +2634,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -2693,7 +2696,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, status = response.code(); - if (!retryStatus.contains(status)) { + if (!RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; } @@ -2708,9 +2711,9 @@ private void putPostValueImpl(RequestLogger reqlog, String method, } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -3064,7 +3067,7 @@ public R putResour long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -3083,7 +3086,7 @@ public R putResour response = doPut(requestBldr, multiPart, hasStreamingPart); status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -3098,9 +3101,9 @@ public R putResour } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -3238,7 +3241,7 @@ public R postResou long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -3257,7 +3260,7 @@ public R postResou response = doPost(requestBldr, multiPart, hasStreamingPart); status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -3272,9 +3275,9 @@ public R postResou } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -3848,7 +3851,7 @@ private U postIt long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -3870,7 +3873,7 @@ private U postIt response = doPost(requestBldr, multiPart, hasStreamingPart); status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -3885,9 +3888,9 @@ private U postIt } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -4453,17 +4456,17 @@ private String stringJoin(Collection collection, String separator, return (builder != null) ? builder.toString() : null; } - private int calculateDelay(Random rand, int i) { + private int calculateDelay(int attempt) { int min = - (i > 6) ? DELAY_CEILING : - (i == 0) ? DELAY_FLOOR : - DELAY_FLOOR + (1 << i) * DELAY_MULTIPLIER; + (attempt > 6) ? DELAY_CEILING : + (attempt == 0) ? DELAY_FLOOR : + DELAY_FLOOR + (1 << attempt) * DELAY_MULTIPLIER; int range = - (i > 6) ? DELAY_FLOOR : - (i == 0) ? 2 * DELAY_MULTIPLIER : - (i == 6) ? DELAY_CEILING - min : - (1 << i) * DELAY_MULTIPLIER; - return min + randRetry.nextInt(range); + (attempt > 6) ? DELAY_FLOOR : + (attempt == 0) ? 2 * DELAY_MULTIPLIER : + (attempt == 6) ? DELAY_CEILING - min : + (1 << attempt) * DELAY_MULTIPLIER; + return min + randomForRetryDelay.nextInt(range); } static class OkHttpResult { @@ -4967,7 +4970,7 @@ public InputStream match(QueryDefinition queryDef, long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -4987,7 +4990,7 @@ public InputStream match(QueryDefinition queryDef, } status = response.code(); - if (!retryStatus.contains(status)) { + if (!RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -4998,9 +5001,9 @@ public InputStream match(QueryDefinition queryDef, closeResponse(response); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( From 95fc3664b2268f0f93ad7a5e8fddac0a25ffaf65 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 20 Jan 2026 15:25:47 -0500 Subject: [PATCH 37/70] MLE-26427 Couple more tests for incremental write Just verifying that text / binary work as expected. Also improved an existing test to use a different hash field name. --- .../filter/IncrementalWriteTest.java | 50 ++++++++++++++++--- .../ml-config/databases/content-database.json | 9 ++++ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 723296b72..0ac60b97a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -6,10 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.marklogic.client.document.*; import com.marklogic.client.impl.DocumentWriteOperationImpl; -import com.marklogic.client.io.DocumentMetadataHandle; -import com.marklogic.client.io.Format; -import com.marklogic.client.io.JacksonHandle; -import com.marklogic.client.io.StringHandle; +import com.marklogic.client.io.*; import com.marklogic.client.test.Common; import org.junit.jupiter.api.Test; @@ -156,7 +153,7 @@ void noRangeIndexForFieldWithEval() { @Test void customTimestampKeyName() { filter = IncrementalWriteFilter.newBuilder() - .hashKeyName("incrementalWriteHash") + .hashKeyName("myWriteHash") .timestampKeyName("myTimestamp") .build(); @@ -165,8 +162,9 @@ void customTimestampKeyName() { DocumentMetadataHandle metadata = Common.client.newDocumentManager().readMetadata("/incremental/test/doc-1.xml", new DocumentMetadataHandle()); + assertNotNull(metadata.getMetadataValues().get("myWriteHash")); assertNotNull(metadata.getMetadataValues().get("myTimestamp")); - assertNotNull(metadata.getMetadataValues().get("incrementalWriteHash")); + assertFalse(metadata.getMetadataValues().containsKey("incrementalWriteHash")); assertFalse(metadata.getMetadataValues().containsKey("incrementalWriteTimestamp")); } @@ -190,6 +188,46 @@ void nullIsIgnoredForKeyNames() { assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); } + @Test + void textDocument() { + final DocumentWriteOperation writeOp = new DocumentWriteOperationImpl("/incremental/test/doc.txt", METADATA, + new StringHandle("Hello world")); + + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write the same text document again + docs = new ArrayList<>(); + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(1, skippedCount.get(), "This is a sanity check to verify that text files work as expected. " + + "Exclusions can't yet be specified for them since we only support JSON Pointer and XPath so far. It may " + + "be worth supporting regex-based exclusions for text files in the future."); + } + + @Test + void binaryDocument() { + byte[] binaryContent = "Binary content example".getBytes(); + final DocumentWriteOperation writeOp = new DocumentWriteOperationImpl("/incremental/test/doc.bin", METADATA, + new BytesHandle(binaryContent).withFormat(Format.BINARY)); + + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write the same binary document again + docs = new ArrayList<>(); + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(1, skippedCount.get(), "Another sanity check to make sure that binary documents work as " + + "expected. Exclusions cannot be specified for them."); + } + private void verifyIncrementalWriteWorks() { writeTenDocuments(); verifyDocumentsHasHashInMetadataKey(); diff --git a/test-app/src/main/ml-config/databases/content-database.json b/test-app/src/main/ml-config/databases/content-database.json index 4e869464f..8ca808124 100644 --- a/test-app/src/main/ml-config/databases/content-database.json +++ b/test-app/src/main/ml-config/databases/content-database.json @@ -197,6 +197,15 @@ "fast-phrase-searches": false, "fast-case-sensitive-searches": false, "fast-diacritic-sensitive-searches": false + }, + { + "field-name": "myWriteHash", + "metadata": "", + "stemmed-searches": "off", + "word-searches": false, + "fast-phrase-searches": false, + "fast-case-sensitive-searches": false, + "fast-diacritic-sensitive-searches": false } ], "range-field-index": [ From c6fe17dc26909ca7321332762e768673a729cc0f Mon Sep 17 00:00:00 2001 From: Steve Biondi Date: Fri, 16 Jan 2026 11:10:49 -0800 Subject: [PATCH 38/70] MLE-25585 implement optic transitive closure in java client. Add generated code with minor changes. Add tests for op.transitiveClosure. Add triple data for tests. Some copyright header fixes. --- .../client/expression/PlanBuilder.java | 32 +- .../client/expression/PlanBuilderBase.java | 11 +- .../client/impl/PlanBuilderImpl.java | 38 +- .../client/impl/PlanBuilderSubImpl.java | 68 +++- .../type/PlanTransitiveClosureOptions.java | 19 + .../test/rows/OpticTransitiveClosureTest.java | 352 ++++++++++++++++++ .../transitive-closure/collections.properties | 1 + .../transitive-closure/permissions.properties | 1 + .../transClosureTripleSet.xml | 107 ++++++ 9 files changed, 625 insertions(+), 4 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/rows/OpticTransitiveClosureTest.java create mode 100644 test-app/src/main/ml-data/optic/transitive-closure/collections.properties create mode 100644 test-app/src/main/ml-data/optic/transitive-closure/permissions.properties create mode 100644 test-app/src/main/ml-data/optic/transitive-closure/transClosureTripleSet.xml diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java index 58a2c533e..7ad4ed0c4 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.expression; @@ -2043,6 +2043,36 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @since 7.2.0; requires MarkLogic 12 */ public abstract ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol path, PlanExprCol length, PlanExprCol weight); +/** + * This method performs a transitive closure operation over a graph-like structure, identifying all reachable node pairs from a given start node to an end node through one or more intermediate steps. A set of (start, end) node pairs where a path exists between them with a length between minLength and maxLength, inclusive. This models the SPARQL one-or-more (+) operator, enabling recursive or chained relationships to be queried efficiently. + * @param start The column is the starting node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + */ + public abstract ModifyPlan transitiveClosure(String start, String end); +/** + * This method performs a transitive closure operation over a graph-like structure, identifying all reachable node pairs from a given start node to an end node through one or more intermediate steps. A set of (start, end) node pairs where a path exists between them with a length between minLength and maxLength, inclusive. This models the SPARQL one-or-more (+) operator, enabling recursive or chained relationships to be queried efficiently. + * @param start The column is the starting node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @return a ModifyPlan object + */ + public abstract ModifyPlan transitiveClosure(PlanExprCol start, PlanExprCol end); +/** + * This method performs a transitive closure operation over a graph-like structure, identifying all reachable node pairs from a given start node to an end node through one or more intermediate steps. A set of (start, end) node pairs where a path exists between them with a length between minLength and maxLength, inclusive. This models the SPARQL one-or-more (+) operator, enabling recursive or chained relationships to be queried efficiently. + * @param start The column is the starting node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @param options This is either an array of strings or an object containing keys and values for the options to this operator. Options include: min-length This option is the minimum number of steps (edges) required in the path. It should be a non-negative integer, and the default is 1.max-length This option Maximum number of steps (edges) allowed in the path. It should be a non-negative integer, and the default is unlimited. + * @return a ModifyPlan object + */ + public abstract ModifyPlan transitiveClosure(String start, String end, PlanTransitiveClosureOptions options); +/** + * This method performs a transitive closure operation over a graph-like structure, identifying all reachable node pairs from a given start node to an end node through one or more intermediate steps. A set of (start, end) node pairs where a path exists between them with a length between minLength and maxLength, inclusive. This models the SPARQL one-or-more (+) operator, enabling recursive or chained relationships to be queried efficiently. + * @param start The column is the starting node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} + * @param options This is either an array of strings or an object containing keys and values for the options to this operator. Options include: min-length This option is the minimum number of steps (edges) required in the path. It should be a non-negative integer, and the default is 1.max-length This option Maximum number of steps (edges) allowed in the path. It should be a non-negative integer, and the default is unlimited. + * @return a ModifyPlan object + */ + public abstract ModifyPlan transitiveClosure(PlanExprCol start, PlanExprCol end, PlanTransitiveClosureOptions options); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java index 61af7fcfa..378f7ae43 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.expression; @@ -356,6 +356,15 @@ public interface PlanBuilderBase { */ PlanSparqlOptions sparqlOptions(); + /** + * Provides a transitive closure option object to configure the execution of the + * {@link PlanBuilder.ModifyPlan#transitiveClosure(PlanExprCol, PlanExprCol, PlanTransitiveClosureOptions)} + * operator. Use the fluent methods of the transitive closure option object + * to set the configuration. + * @return the configuration object + */ + PlanTransitiveClosureOptions transitiveClosureOptions(); + /** * Specifies a JavaScript or XQuery function installed on the server for use * in post-processing in a map() or reduce() operation. diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java index 77c97d02c..61e095d2d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -2268,6 +2268,42 @@ public ModifyPlan shortestPath(PlanExprCol start, PlanExprCol end, PlanExprCol p } + @Override + public ModifyPlan transitiveClosure(String start, String end) { + return transitiveClosure((start == null) ? (PlanExprCol) null : exprCol(start), (end == null) ? null : exprCol(end)); + } + + + @Override + public ModifyPlan transitiveClosure(PlanExprCol start, PlanExprCol end) { + if (start == null) { + throw new IllegalArgumentException("start parameter for transitiveClosure() cannot be null"); + } + if (end == null) { + throw new IllegalArgumentException("end parameter for transitiveClosure() cannot be null"); + } + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "transitive-closure", new Object[]{ start, end }); + } + + + @Override + public ModifyPlan transitiveClosure(String start, String end, PlanTransitiveClosureOptions options) { + return transitiveClosure((start == null) ? null : exprCol(start), (end == null) ? null : exprCol(end), options); + } + + + @Override + public ModifyPlan transitiveClosure(PlanExprCol start, PlanExprCol end, PlanTransitiveClosureOptions options) { + if (start == null) { + throw new IllegalArgumentException("start parameter for transitiveClosure() cannot be null"); + } + if (end == null) { + throw new IllegalArgumentException("end parameter for transitiveClosure() cannot be null"); + } + return new PlanBuilderSubImpl.ModifyPlanSubImpl(this, "op", "transitive-closure", new Object[]{ start, end, PlanBuilderSubImpl.asArg(PlanBuilderSubImpl.makeMap(options)) }); + } + + @Override public ModifyPlan union(ModifyPlan right) { if (right == null) { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java index 9a320ab43..3c11bbf38 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -487,6 +487,10 @@ public PlanSampleByOptions withLimit(XsIntVal limit) { public PlanSparqlOptions sparqlOptions() { return new PlanSparqlOptionsImpl(this); } + @Override + public PlanTransitiveClosureOptions transitiveClosureOptions() { + return new PlanTransitiveClosureOptionsImpl(this); + } static class PlanSparqlOptionsImpl implements PlanSparqlOptions { private PlanBuilderBaseImpl pb; private XsBooleanVal deduplicate; @@ -526,6 +530,45 @@ public PlanSparqlOptions withDeduplicated(XsBooleanVal deduplicate) { } } + static class PlanTransitiveClosureOptionsImpl implements PlanTransitiveClosureOptions { + private PlanBuilderBaseImpl pb; + private XsLongVal minLength; + private XsLongVal maxLength; + PlanTransitiveClosureOptionsImpl(PlanBuilderBaseImpl pb) { + this.pb = pb; + } + PlanTransitiveClosureOptionsImpl(PlanBuilderBaseImpl pb, XsLongVal minLength, XsLongVal maxLength) { + this(pb); + this.minLength = minLength; + this.maxLength = maxLength; + } + + @Override + public XsLongVal getMinLength() { + return minLength; + } + @Override + public PlanTransitiveClosureOptions withMinLength(long minLength) { + return withMinLength(pb.xs.longVal(minLength)); + } + @Override + public PlanTransitiveClosureOptions withMinLength(XsLongVal minLength) { + return new PlanTransitiveClosureOptionsImpl(this.pb, minLength, this.maxLength); + } + @Override + public XsLongVal getMaxLength() { + return maxLength; + } + @Override + public PlanTransitiveClosureOptions withMaxLength(long maxLength) { + return withMaxLength(pb.xs.longVal(maxLength)); + } + @Override + public PlanTransitiveClosureOptions withMaxLength(XsLongVal maxLength) { + return new PlanTransitiveClosureOptionsImpl(this.pb, this.minLength, maxLength); + } + } + @Override public ServerExpression caseExpr(PlanCase... cases) { int lastPos = cases.length - 1; @@ -744,6 +787,29 @@ static Map makeMap(PlanSparqlOptions options) { return mapdef; } + static Map makeMap(PlanTransitiveClosureOptions options) { + if (options == null) { + return null; + } + + Map mapdef = null; + + XsLongVal minLength = options.getMinLength(); + if (minLength != null) { + mapdef = new HashMap<>(); + mapdef.put("minLength", minLength.getLong()); + } + + XsLongVal maxLength = options.getMaxLength(); + if (maxLength != null) { + if (mapdef == null) { + mapdef = new HashMap<>(); + } + mapdef.put("maxLength", maxLength.getLong()); + } + + return mapdef; + } static Map makeMap(String key, String value) { Map map = new HashMap(); if (key != null) { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java new file mode 100644 index 000000000..3772571d9 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.type; + +// IMPORTANT: Do not edit. This file is generated. + +/** + * Options for controlling transitive closure operations, including minimum and maximum + * path lengths. + */ +public interface PlanTransitiveClosureOptions { + XsLongVal getMinLength(); + PlanTransitiveClosureOptions withMinLength(long minLength); + PlanTransitiveClosureOptions withMinLength(XsLongVal minLength); + XsLongVal getMaxLength(); + PlanTransitiveClosureOptions withMaxLength(long maxLength); + PlanTransitiveClosureOptions withMaxLength(XsLongVal maxLength); +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/OpticTransitiveClosureTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/OpticTransitiveClosureTest.java new file mode 100644 index 000000000..d6f04202a --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/OpticTransitiveClosureTest.java @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ + +package com.marklogic.client.test.rows; + +import com.marklogic.client.row.RowRecord; +import com.marklogic.client.row.RowTemplate; +import com.marklogic.client.test.Common; +import com.marklogic.client.type.PlanTransitiveClosureOptions; +import com.marklogic.client.type.PlanTripleOption; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for the transitiveClosure Optic function introduced in MarkLogic 12. + * This test class verifies the transitive closure operation over graph-like structures, + * identifying all reachable node pairs from a given start node to an end node through + * one or more intermediate steps. The tests use a parent-child relationship graph + * loaded from transClosureTripleSet.xml using mlDeploy during setup. + */ +public class OpticTransitiveClosureTest { + + @BeforeAll + public static void setUp() { + Common.connect(); + } + + /** + * Simple full transitive closure without options. + * Expects 21 rows with person and ancestor columns. + */ + @Test + void testSimplePatternFullTransitiveClosure() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("ancestor") + ), + null, + (String) null, + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure(op.col("person"), op.col("ancestor")) + .orderBy(op.sortKeySeq(op.asc("ancestor"), op.asc("person"))), + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + assertEquals(21, rowList.size(), "Expected 21 rows for full transitive closure"); + + // Verify first row has required columns + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist"); + + return null; + }); + } + + /** + * Transitive closure with minLength=2 (grandparents and up). + * This excludes direct parent-child relationships. + * Expects 12 rows. + */ + @Test + void testTransitiveClosureWithMinLength() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> { + // Create options with minLength=2 + PlanTransitiveClosureOptions options = op.transitiveClosureOptions() + .withMinLength(2); + + return op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("ancestor") + ), + null, + (String) null, + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure(op.col("person"), op.col("ancestor"), options); + }, + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + // 2 steps or more excludes direct parent-child relationships + assertEquals(12, rowList.size(), "Expected 12 rows with minLength=2 (grandparents and up)"); + + // Verify columns exist + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist"); + + return null; + }); + } + + /** + * Transitive closure with minLength=2 and maxLength=2 (grandparents only). + * Expects 6 rows. + */ + @Test + void testTransitiveClosureWithMinAndMaxLength() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> { + // Create options with minLength=2 and maxLength=2 + PlanTransitiveClosureOptions options = op.transitiveClosureOptions() + .withMinLength(2) + .withMaxLength(2); + + return op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("ancestor") + ), + null, + "http://test.optic.tc#", + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure(op.col("person"), op.col("ancestor"), options); + }, + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + // 2 steps only is grandparent relationships only + assertEquals(6, rowList.size(), "Expected 6 rows with minLength=2 and maxLength=2 (grandparents only)"); + + // Verify columns exist + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist"); + + return null; + }); + } + + /** + * Transitive closure with column renamed using op.as(). + * Uses "parent" column and renames it to "ancestor". + * Expects 21 rows. + */ + @Test + void testTransitiveClosureWithColumnRename() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("parent") + ), + null, + (String) null, + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure(op.col("person"), op.as("ancestor", op.col("parent"))), + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + assertEquals(21, rowList.size(), "Expected 21 rows with renamed column"); + + // Verify renamed column exists + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist (renamed from parent)"); + + return null; + }); + } + + /** + * Transitive closure with joins to get labels. + * Joins with label triples to get human-readable names. + * Expects 21 rows with person, ancestor, person_name, and ancestor_name columns. + */ + @Test + void testTransitiveClosureWithJoinsForLabels() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> { + var labelIri = op.sem.iri("http://test.optic.tc#label"); + return op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("ancestor") + ) + , (String)null, + "http://test.optic.tc#", + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure(op.col("person"), op.col("ancestor")) + .joinLeftOuter( + op.fromTriples( + op.pattern(op.col("person"), labelIri, op.col("person_name")), + null, + "http://test.optic.tc#", + PlanTripleOption.DEDUPLICATED + ) + ) + .joinLeftOuter( + op.fromTriples( + op.pattern(op.col("ancestor"), labelIri, op.col("ancestor_name")), + null, + "http://test.optic.tc#", + PlanTripleOption.DEDUPLICATED + ) + ); + }, + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + assertEquals(21, rowList.size(), "Expected 21 rows with joined labels"); + + // Verify all columns exist + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist"); + assertNotNull(firstRow.get("person_name"), "person_name column should exist"); + assertNotNull(firstRow.get("ancestor_name"), "ancestor_name column should exist"); + + return null; + }); + } + + /** + * Test 6: Transitive closure using string column names instead of column expressions. + * This tests the convenience overload that accepts String parameters. + * Expects 21 rows. + */ + @Test + void testTransitiveClosureWithStringColumnNames() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("ancestor") + ), + null, + (String) null, + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure("person", "ancestor") + .orderBy(op.sortKeySeq(op.asc("ancestor"), op.asc("person"))), + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + assertEquals(21, rowList.size(), "Expected 21 rows using string column names"); + + // Verify columns exist + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist"); + + return null; + }); + } + + /** + * Test 7: Transitive closure with string column names and options. + * Expects 12 rows with minLength=2. + */ + @Test + void testTransitiveClosureWithStringNamesAndOptions() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> op.fromTriples( + op.pattern( + op.col("person"), + op.sem.iri("http://marklogic.com/transitiveClosure/parent"), + op.col("ancestor") + ), + null, + (String) null, + PlanTripleOption.DEDUPLICATED + ) + .transitiveClosure("person", "ancestor", op.transitiveClosureOptions().withMinLength(2)), + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + assertEquals(12, rowList.size(), "Expected 12 rows with string names and minLength=2"); + + return null; + }); + } + + /** + * Test 8: Transitive closure starting from a SPARQL query with minLength=2 + * Expects 21 rows. This is harder to do in SPARQL directly, so we do the closure in Optic. + */ + @Test + void testTransitiveClosureFromSparql() { + if (Common.getMarkLogicVersion().getMajor() < 12) { + return; + } + + new RowTemplate(Common.client).query(op -> op.fromSparql( + "SELECT ?person ?ancestor WHERE { ?person ?ancestor }" + ) + .transitiveClosure(op.col("person"), op.col("ancestor"), op.transitiveClosureOptions().withMinLength(2)) + .orderBy(op.sortKeySeq(op.asc("ancestor"), op.asc("person"))), + rows -> { + List rowList = new ArrayList<>(); + rows.forEach(rowList::add); + + assertEquals(12, rowList.size(), "Expected 12 rows for transitive closure from SPARQL with minLength=2 (grandparents and farther)"); + + // Verify first row has required columns + RowRecord firstRow = rowList.get(0); + assertNotNull(firstRow.get("person"), "person column should exist"); + assertNotNull(firstRow.get("ancestor"), "ancestor column should exist"); + + return null; + }); + } +} diff --git a/test-app/src/main/ml-data/optic/transitive-closure/collections.properties b/test-app/src/main/ml-data/optic/transitive-closure/collections.properties new file mode 100644 index 000000000..c64199b99 --- /dev/null +++ b/test-app/src/main/ml-data/optic/transitive-closure/collections.properties @@ -0,0 +1 @@ +transClosureTripleSet.xml=http://test.optic.tc#,test-data diff --git a/test-app/src/main/ml-data/optic/transitive-closure/permissions.properties b/test-app/src/main/ml-data/optic/transitive-closure/permissions.properties new file mode 100644 index 000000000..f775de1ee --- /dev/null +++ b/test-app/src/main/ml-data/optic/transitive-closure/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read,rest-writer,update,app-user,read,app-builder,read,app-builder,update diff --git a/test-app/src/main/ml-data/optic/transitive-closure/transClosureTripleSet.xml b/test-app/src/main/ml-data/optic/transitive-closure/transClosureTripleSet.xml new file mode 100644 index 000000000..c2130bdfe --- /dev/null +++ b/test-app/src/main/ml-data/optic/transitive-closure/transClosureTripleSet.xml @@ -0,0 +1,107 @@ + + + + + +http://test.optic.tc#Alice +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Bob + + +http://test.optic.tc#Bob +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Carol + + +http://test.optic.tc#Carol +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#David + + +http://test.optic.tc#David +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Eve + + +http://test.optic.tc#Eve +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Frank + + +http://test.optic.tc#George +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Helen + + +http://test.optic.tc#Helen +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Ian + + +http://test.optic.tc#Alice +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#Cindy + + +http://test.optic.tc#Cindy +http://marklogic.com/transitiveClosure/parent +http://test.optic.tc#John + + +http://test.optic.tc#Alice +http://test.optic.tc#label +Alice + + +http://test.optic.tc#Bob +http://test.optic.tc#label +Bob + + +http://test.optic.tc#Eve +http://test.optic.tc#label +Eve + + +http://test.optic.tc#Cindy +http://test.optic.tc#label +Cindy + + +http://test.optic.tc#Helen +http://test.optic.tc#label +Helen + + +http://test.optic.tc#Ian +http://test.optic.tc#label +Ian + + +http://test.optic.tc#John +http://test.optic.tc#label +John + + +http://test.optic.tc#David +http://test.optic.tc#label +David + + +http://test.optic.tc#George +http://test.optic.tc#label +George + + +http://test.optic.tc#Carol +http://test.optic.tc#label +Carol + + +http://test.optic.tc#Frank +http://test.optic.tc#label +Frank + + + + From 8f7117d03cf924a4dfeb667d6787d9a83fe6adef Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 27 Jan 2026 10:05:32 -0500 Subject: [PATCH 39/70] MLE-26427 Fixing incremental write test Somehow added this without the required range index for tests. --- .../src/main/ml-config/databases/content-database.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-app/src/main/ml-config/databases/content-database.json b/test-app/src/main/ml-config/databases/content-database.json index 8ca808124..53803022d 100644 --- a/test-app/src/main/ml-config/databases/content-database.json +++ b/test-app/src/main/ml-config/databases/content-database.json @@ -229,6 +229,13 @@ "field-name": "incrementalWriteHash", "range-value-positions": false, "invalid-values": "reject" + }, + { + "scalar-type": "string", + "collation": "http://marklogic.com/collation/", + "field-name": "myWriteHash", + "range-value-positions": false, + "invalid-values": "reject" } ], "geospatial-element-index": [ From 92943f6b4fd1d9a8d90495912b34c0c28916b897 Mon Sep 17 00:00:00 2001 From: Steve Biondi Date: Thu, 5 Feb 2026 19:09:32 -0800 Subject: [PATCH 40/70] MLE-26339 Implement vec.trunc and vec.precision in Java Client API --- .../marklogic/client/expression/VecExpr.java | 51 ++++++++++++- .../marklogic/client/impl/VecExprImpl.java | 32 ++++++++- .../client/test/rows/VectorTest.java | 72 ++++++++++++++++++- 3 files changed, 152 insertions(+), 3 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java index d7076612e..7210bcc30 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.expression; @@ -151,6 +151,26 @@ public interface VecExpr { * @return a server expression with the vec:vector server data type */ public ServerExpression normalize(ServerExpression vector1); +/** + * Returns a new vector which is a copy of the input vector with reduced precision. The precision reduction is achieved by clearing the bottom (32 - precision) bits of the mantissa for each dimension's float value. This can be useful for reducing storage requirements or for creating approximate vector representations. + * + * + + *

+ * Provides a client interface to the vec:precision server function. + * @param vector The input vector to reduce precision. Can be a vector or an empty sequence. (of vec:vector) + * @return a server expression with the vec:vector server data type + */ + public ServerExpression precision(ServerExpression vector); +/** + * Returns a new vector which is a copy of the input vector with reduced precision. The precision reduction is achieved by clearing the bottom (32 - precision) bits of the mantissa for each dimension's float value. This can be useful for reducing storage requirements or for creating approximate vector representations. + *

+ * Provides a client interface to the vec:precision server function. + * @param vector The input vector to reduce precision. Can be a vector or an empty sequence. (of vec:vector) + * @param precision The number of mantissa bits to preserve (9-32 inclusive). Default is 16. Higher values preserve more precision. If the value is outside the valid range, throw VEC-INVALIDPRECISION. (of xs:unsignedInt) + * @return a server expression with the vec:vector server data type + */ + public ServerExpression precision(ServerExpression vector, ServerExpression precision); /** * Returns the difference of two vectors. The vectors must be of the same dimension. * @@ -185,6 +205,35 @@ public interface VecExpr { * @return a server expression with the vec:vector server data type */ public ServerExpression subvector(ServerExpression vector, ServerExpression start, ServerExpression length); +/** + * Returns a new vector which is a copy of the input vector with each element truncated to a specific number of digits. + * + * + + *

+ * Provides a client interface to the vec:trunc server function. + * @param vector The input vector to truncate. (of vec:vector) + * @return a server expression with the vec:vector server data type + */ + public ServerExpression trunc(ServerExpression vector); +/** + * Returns a new vector which is a copy of the input vector with each element truncated to a specific number of digits. + *

+ * Provides a client interface to the vec:trunc server function. + * @param vector The input vector to truncate. (of vec:vector) + * @param n The numbers of decimal places to truncate to. The default is 0. Negative values cause that many digits to the left of the decimal point to be truncated. (of xs:int) + * @return a server expression with the vec:vector server data type + */ + public ServerExpression trunc(ServerExpression vector, int n); +/** + * Returns a new vector which is a copy of the input vector with each element truncated to a specific number of digits. + *

+ * Provides a client interface to the vec:trunc server function. + * @param vector The input vector to truncate. (of vec:vector) + * @param n The numbers of decimal places to truncate to. The default is 0. Negative values cause that many digits to the left of the decimal point to be truncated. (of xs:int) + * @return a server expression with the vec:vector server data type + */ + public ServerExpression trunc(ServerExpression vector, ServerExpression n); /** * Returns a vector value. * diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java index b09b1afa4..5a8ac10d7 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -93,6 +93,18 @@ public ServerExpression normalize(ServerExpression vector1) { } + @Override + public ServerExpression precision(ServerExpression vector) { + return new VectorCallImpl("vec", "precision", new Object[]{ vector }); + } + + + @Override + public ServerExpression precision(ServerExpression vector, ServerExpression precision) { + return new VectorCallImpl("vec", "precision", new Object[]{ vector, precision }); + } + + @Override public ServerExpression subtract(ServerExpression vector1, ServerExpression vector2) { return new VectorCallImpl("vec", "subtract", new Object[]{ vector1, vector2 }); @@ -111,6 +123,24 @@ public ServerExpression subvector(ServerExpression vector, ServerExpression star } + @Override + public ServerExpression trunc(ServerExpression vector) { + return new VectorCallImpl("vec", "trunc", new Object[]{ vector }); + } + + + @Override + public ServerExpression trunc(ServerExpression vector, int n) { + return trunc(vector, xs.intVal(n)); + } + + + @Override + public ServerExpression trunc(ServerExpression vector, ServerExpression n) { + return new VectorCallImpl("vec", "trunc", new Object[]{ vector, n }); + } + + @Override public ServerExpression vector(ServerExpression values) { return new VectorCallImpl("vec", "vector", new Object[]{ values }); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java index 06b7c0950..d41e4bfee 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.rows; @@ -57,6 +57,8 @@ void vectorFunctionsHappyPath() { .bind(op.as("base64Encode", op.vec.base64Encode(op.col("sampleVector")))) .bind(op.as("base64Decode", op.vec.base64Decode(op.col("base64Encode")))) .bind(op.as("subVector", op.vec.subvector(op.col("sampleVector"), op.xs.integer(1), op.xs.integer(1)))) + .bind(op.as("precision", op.vec.precision(op.col("sampleVector"), op.xs.unsignedInt(16)))) + .bind(op.as("trunc", op.vec.trunc(op.col("sampleVector"), 1))) .bind(op.as("vectorScore", op.vec.vectorScore(op.xs.unsignedInt(1), op.xs.doubleVal(0.5)))) .bind(op.as("simpleVectorScore", op.vec.vectorScore(op.xs.unsignedInt(1), 0.5, 1))) .bind(op.as("simplestVectorScore", op.vec.vectorScore(op.xs.unsignedInt(1), 0.5))); @@ -82,6 +84,8 @@ void vectorFunctionsHappyPath() { assertEquals(3, ((ArrayNode) row.get("base64Decode")).size()); assertEquals(5.6, row.getDouble("get")); assertEquals(1, ((ArrayNode) row.get("subVector")).size()); + assertEquals(3, ((ArrayNode) row.get("precision")).size()); + assertEquals(3, ((ArrayNode) row.get("trunc")).size()); assertEquals(333333.0, row.getDouble("vectorScore")); assertEquals(666666.0, row.getDouble("simpleVectorScore")); assertEquals(333333.0, row.getDouble("simplestVectorScore")); @@ -184,4 +188,70 @@ void dslAnnTopK() { List rows = resultRows(plan); assertEquals(2, rows.size(), "Just verifying that 'annTopK' works via the DSL and v1/rows."); } + + @Test + void precision() { + // Test vec.precision with default precision (16 bits) + PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") + .limit(1) + .bind(op.as("testVector", op.vec.vector(op.xs.doubleSeq(3.14159265, 2.71828182, 1.41421356)))) + .bind(op.as("precisionDefault", op.vec.precision(op.col("testVector")))) + .bind(op.as("precision10", op.vec.precision(op.col("testVector"), op.xs.unsignedInt(10)))); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + RowRecord row = rows.get(0); + + // Verify that precision returns a vector + ArrayNode precisionDefault = (ArrayNode) row.get("precisionDefault"); + assertNotNull(precisionDefault); + assertEquals(3, precisionDefault.size()); + + // Verify precision with 10 bits - should truncate values + ArrayNode precision10 = (ArrayNode) row.get("precision10"); + assertNotNull(precision10); + assertEquals(3, precision10.size()); + assertEquals(3, precision10.get(0).asInt(), "First element should be truncated to 3"); + assertEquals(2, precision10.get(1).asInt(), "Second element should be truncated to 2"); + assertEquals(1, precision10.get(2).asInt(), "Third element should be truncated to 1"); + } + + @Test + void trunc() { + // Test vec.trunc with different decimal places + PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") + .limit(1) + .bind(op.as("testVector", op.vec.vector(op.xs.doubleSeq(1.123456789, 2.123456789, 3.123456789)))) + .bind(op.as("truncDefault", op.vec.trunc(op.col("testVector")))) + .bind(op.as("trunc1", op.vec.trunc(op.col("testVector"), 1))) + .bind(op.as("trunc2", op.vec.trunc(op.col("testVector"), op.xs.intVal(2)))); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + RowRecord row = rows.get(0); + + // Verify truncation with default (0 decimal places) + ArrayNode truncDefault = (ArrayNode) row.get("truncDefault"); + assertNotNull(truncDefault); + assertEquals(3, truncDefault.size()); + assertEquals(1, truncDefault.get(0).asInt(), "First element should be truncated to 1"); + assertEquals(2, truncDefault.get(1).asInt(), "Second element should be truncated to 2"); + assertEquals(3, truncDefault.get(2).asInt(), "Third element should be truncated to 3"); + + // Verify truncation with 1 decimal place + ArrayNode trunc1 = (ArrayNode) row.get("trunc1"); + assertNotNull(trunc1); + assertEquals(3, trunc1.size()); + assertEquals(1.1, trunc1.get(0).asDouble(), 0.001, "First element should be truncated to 1.1"); + assertEquals(2.1, trunc1.get(1).asDouble(), 0.001, "Second element should be truncated to 2.1"); + assertEquals(3.1, trunc1.get(2).asDouble(), 0.001, "Third element should be truncated to 3.1"); + + // Verify truncation with 2 decimal places + ArrayNode trunc2 = (ArrayNode) row.get("trunc2"); + assertNotNull(trunc2); + assertEquals(3, trunc2.size()); + assertEquals(1.12, trunc2.get(0).asDouble(), 0.001, "First element should be truncated to 1.12"); + assertEquals(2.12, trunc2.get(1).asDouble(), 0.001, "Second element should be truncated to 2.12"); + assertEquals(3.12, trunc2.get(2).asDouble(), 0.001, "Third element should be truncated to 3.12"); + } } From d7c6ad5832dcf5c5fb9368e8499ddd616daa2b7c Mon Sep 17 00:00:00 2001 From: Steve Biondi Date: Fri, 6 Feb 2026 10:29:47 -0800 Subject: [PATCH 41/70] MLE-27018 update cosine, cosineDistance and vectorScore to match codegen The documentation for cosine function changed to just "return the cosine of the angle between two vectors". This is the same thing as similarity, but change the text to match codegen. Fix test to enforce correct range of values it should return ([-1,1]) to avoid confusion. Changed javadoc for cosineDistance to state explicitly "returns the cosine distance between two vectors" to match codegen. Test that 1 - cosine(v1,v2) == cosineDistance(v1,v2) explicitly in test (within a floating point delta) to document the relationship in the test. Update vectorScore methods, change the similarity parameter name to distance to match codegen. Add two new methods that have another weight param for the ANN portion of the final hybrid score. --- .../marklogic/client/expression/VecExpr.java | 95 +++++++++++-------- .../marklogic/client/impl/VecExprImpl.java | 28 ++++-- .../client/test/rows/VectorTest.java | 83 +++++++++++++++- 3 files changed, 156 insertions(+), 50 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java index 7210bcc30..87753efb5 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java @@ -57,30 +57,26 @@ public interface VecExpr { */ public ServerExpression base64Encode(ServerExpression vector1); - /** - * Returns the cosine similarity between two vectors. The vectors must be of the same dimension. - * - * - * - *

- * Provides a client interface to the vec:cosine server function. - * - * @param vector1 The vector from which to calculate the cosine similarity with vector2. (of vec:vector) - * @param vector2 The vector from which to calculate the cosine similarity with vector1. (of vec:vector) - * @return a server expression with the xs:double server data type - * @since 7.2.0 - */ - public ServerExpression cosine(ServerExpression vector1, ServerExpression vector2); +/** + * Returns the cosine of the angle between two vectors. The vectors must be of the same dimension. + *

+ * Provides a client interface to the vec:cosine server function. + * @param vector1 The vector from which to calculate the cosine with vector2. (of vec:vector) + * @param vector2 The vector from which to calculate the cosine with vector1. (of vec:vector) + * @return a server expression with the xs:double server data type + * @since 7.2.0 + */ + public ServerExpression cosine(ServerExpression vector1, ServerExpression vector2); - /** - * Return the distance between two vectors. The vectors must be of the same dimension. - * - * @param vector1 The vector from which to calculate the cosine distance with vector2. (of vec:vector) - * @param vector2 The vector from which to calculate the cosine distance with vector1. (of vec:vector) - * @return a server expression with the xs:double server data type - * @since 7.2.0 - */ - public ServerExpression cosineDistance(ServerExpression vector1, ServerExpression vector2); +/** + * Returns the cosine distance between two vectors. The vectors must be of the same dimension. + * + * @param vector1 The vector from which to calculate the cosine distance with vector2. (of vec:vector) + * @param vector2 The vector from which to calculate the cosine distance with vector1. (of vec:vector) + * @return a server expression with the xs:double server data type + * @since 7.2.0 + */ + public ServerExpression cosineDistance(ServerExpression vector1, ServerExpression vector2); /** * Returns the dimension of the vector passed in. @@ -246,44 +242,63 @@ public interface VecExpr { */ public ServerExpression vector(ServerExpression values); /** - * A helper function that returns a hybrid score using a cts score and a vector similarity calculation result. You can tune the effect of the vector similarity on the score using the similarityWeight option. The ideal value for similarityWeight depends on your application. + * A helper function that returns a hybrid score using a cts score and a vector distance calculation result. You can tune the effect of the vector distance on the score using the distanceWeight option. The ideal value for distanceWeight depends on your application. The hybrid score is calculated using the formula: score = weight * annScore + (1 - weight) * ctsScore. - annScore is derived from the distance and distanceWeight, where a larger distanceWeight reduces the annScore for the same distance. - weight determines the contribution of the annScore and ctsScore to the final score. A weight of 0.5 balances both equally. This formula allows you to combine traditional cts scoring with vector-based distance scoring, providing a flexible way to rank results. *

* Provides a client interface to the vec:vector-score server function. * @param score The cts:score of the matching document. (of xs:unsignedInt) - * @param similarity The similarity between the vector in the matching document and the query vector. The result of a call to ovec:cosine(). In the case that the vectors are normalized, pass ovec:dot-product(). Note that vec:euclidean-distance() should not be used here. (of xs:double) + * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) * @return a server expression with the xs:unsignedLong server data type */ - public ServerExpression vectorScore(ServerExpression score, double similarity); + public ServerExpression vectorScore(ServerExpression score, double distance); /** - * A helper function that returns a hybrid score using a cts score and a vector similarity calculation result. You can tune the effect of the vector similarity on the score using the similarityWeight option. The ideal value for similarityWeight depends on your application. - * - * - + * A helper function that returns a hybrid score using a cts score and a vector distance calculation result. You can tune the effect of the vector distance on the score using the distanceWeight option. The ideal value for distanceWeight depends on your application. The hybrid score is calculated using the formula: score = weight * annScore + (1 - weight) * ctsScore. - annScore is derived from the distance and distanceWeight, where a larger distanceWeight reduces the annScore for the same distance. - weight determines the contribution of the annScore and ctsScore to the final score. A weight of 0.5 balances both equally. This formula allows you to combine traditional cts scoring with vector-based distance scoring, providing a flexible way to rank results. + *

+ * Provides a client interface to the vec:vector-score server function. + * @param score The cts:score of the matching document. (of xs:unsignedInt) + * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) + * @return a server expression with the xs:unsignedLong server data type + */ + public ServerExpression vectorScore(ServerExpression score, ServerExpression distance); +/** + * A helper function that returns a hybrid score using a cts score and a vector distance calculation result. You can tune the effect of the vector distance on the score using the distanceWeight option. The ideal value for distanceWeight depends on your application. The hybrid score is calculated using the formula: score = weight * annScore + (1 - weight) * ctsScore. - annScore is derived from the distance and distanceWeight, where a larger distanceWeight reduces the annScore for the same distance. - weight determines the contribution of the annScore and ctsScore to the final score. A weight of 0.5 balances both equally. This formula allows you to combine traditional cts scoring with vector-based distance scoring, providing a flexible way to rank results. + *

+ * Provides a client interface to the vec:vector-score server function. + * @param score The cts:score of the matching document. (of xs:unsignedInt) + * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) + * @param distanceWeight The weight of the vector distance on the annScore. This value is a positive coefficient that scales the distance. A larger distanceWeight produces a lower annScore for the same distance. The default value is 1. (of xs:double) + * @return a server expression with the xs:unsignedLong server data type + */ + public ServerExpression vectorScore(ServerExpression score, double distance, double distanceWeight); +/** + * A helper function that returns a hybrid score using a cts score and a vector distance calculation result. You can tune the effect of the vector distance on the score using the distanceWeight option. The ideal value for distanceWeight depends on your application. The hybrid score is calculated using the formula: score = weight * annScore + (1 - weight) * ctsScore. - annScore is derived from the distance and distanceWeight, where a larger distanceWeight reduces the annScore for the same distance. - weight determines the contribution of the annScore and ctsScore to the final score. A weight of 0.5 balances both equally. This formula allows you to combine traditional cts scoring with vector-based distance scoring, providing a flexible way to rank results. *

* Provides a client interface to the vec:vector-score server function. * @param score The cts:score of the matching document. (of xs:unsignedInt) - * @param similarity The similarity between the vector in the matching document and the query vector. The result of a call to ovec:cosine(). In the case that the vectors are normalized, pass ovec:dot-product(). Note that vec:euclidean-distance() should not be used here. (of xs:double) + * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) + * @param distanceWeight The weight of the vector distance on the annScore. This value is a positive coefficient that scales the distance. A larger distanceWeight produces a lower annScore for the same distance. The default value is 1. (of xs:double) * @return a server expression with the xs:unsignedLong server data type */ - public ServerExpression vectorScore(ServerExpression score, ServerExpression similarity); + public ServerExpression vectorScore(ServerExpression score, ServerExpression distance, ServerExpression distanceWeight); /** - * A helper function that returns a hybrid score using a cts score and a vector similarity calculation result. You can tune the effect of the vector similarity on the score using the similarityWeight option. The ideal value for similarityWeight depends on your application. + * A helper function that returns a hybrid score using a cts score and a vector distance calculation result. You can tune the effect of the vector distance on the score using the distanceWeight option. The ideal value for distanceWeight depends on your application. The hybrid score is calculated using the formula: score = weight * annScore + (1 - weight) * ctsScore. - annScore is derived from the distance and distanceWeight, where a larger distanceWeight reduces the annScore for the same distance. - weight determines the contribution of the annScore and ctsScore to the final score. A weight of 0.5 balances both equally. This formula allows you to combine traditional cts scoring with vector-based distance scoring, providing a flexible way to rank results. *

* Provides a client interface to the vec:vector-score server function. * @param score The cts:score of the matching document. (of xs:unsignedInt) - * @param similarity The similarity between the vector in the matching document and the query vector. The result of a call to ovec:cosine(). In the case that the vectors are normalized, pass ovec:dot-product(). Note that vec:euclidean-distance() should not be used here. (of xs:double) - * @param similarityWeight The weight of the vector similarity on the score. The default value is 0.1. If 0.0 is passed in, vector similarity has no effect. If passed a value less than 0.0 or greater than 1.0, throw VEC-VECTORSCORE. (of xs:double) + * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) + * @param distanceWeight The weight of the vector distance on the annScore. This value is a positive coefficient that scales the distance. A larger distanceWeight produces a lower annScore for the same distance. The default value is 1. (of xs:double) + * @param weight The weight of the annScore in the final hybrid score. This value is a coefficient between 0 and 1, where 0 gives full weight to the cts score and 1 gives full weight to the annScore. The default value is 0.5. (of xs:double) * @return a server expression with the xs:unsignedLong server data type */ - public ServerExpression vectorScore(ServerExpression score, double similarity, double similarityWeight); + public ServerExpression vectorScore(ServerExpression score, double distance, double distanceWeight, double weight); /** - * A helper function that returns a hybrid score using a cts score and a vector similarity calculation result. You can tune the effect of the vector similarity on the score using the similarityWeight option. The ideal value for similarityWeight depends on your application. + * A helper function that returns a hybrid score using a cts score and a vector distance calculation result. You can tune the effect of the vector distance on the score using the distanceWeight option. The ideal value for distanceWeight depends on your application. The hybrid score is calculated using the formula: score = weight * annScore + (1 - weight) * ctsScore. - annScore is derived from the distance and distanceWeight, where a larger distanceWeight reduces the annScore for the same distance. - weight determines the contribution of the annScore and ctsScore to the final score. A weight of 0.5 balances both equally. This formula allows you to combine traditional cts scoring with vector-based distance scoring, providing a flexible way to rank results. *

* Provides a client interface to the vec:vector-score server function. * @param score The cts:score of the matching document. (of xs:unsignedInt) - * @param similarity The similarity between the vector in the matching document and the query vector. The result of a call to ovec:cosine(). In the case that the vectors are normalized, pass ovec:dot-product(). Note that vec:euclidean-distance() should not be used here. (of xs:double) - * @param similarityWeight The weight of the vector similarity on the score. The default value is 0.1. If 0.0 is passed in, vector similarity has no effect. If passed a value less than 0.0 or greater than 1.0, throw VEC-VECTORSCORE. (of xs:double) + * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) + * @param distanceWeight The weight of the vector distance on the annScore. This value is a positive coefficient that scales the distance. A larger distanceWeight produces a lower annScore for the same distance. The default value is 1. (of xs:double) + * @param weight The weight of the annScore in the final hybrid score. This value is a coefficient between 0 and 1, where 0 gives full weight to the cts score and 1 gives full weight to the annScore. The default value is 0.5. (of xs:double) * @return a server expression with the xs:unsignedLong server data type */ - public ServerExpression vectorScore(ServerExpression score, ServerExpression similarity, ServerExpression similarityWeight); + public ServerExpression vectorScore(ServerExpression score, ServerExpression distance, ServerExpression distanceWeight, ServerExpression weight); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java index 5a8ac10d7..79e901872 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/VecExprImpl.java @@ -148,26 +148,38 @@ public ServerExpression vector(ServerExpression values) { @Override - public ServerExpression vectorScore(ServerExpression score, double similarity) { - return vectorScore(score, xs.doubleVal(similarity)); + public ServerExpression vectorScore(ServerExpression score, double distance) { + return vectorScore(score, xs.doubleVal(distance)); } @Override - public ServerExpression vectorScore(ServerExpression score, ServerExpression similarity) { - return new XsExprImpl.UnsignedLongCallImpl("vec", "vector-score", new Object[]{ score, similarity }); + public ServerExpression vectorScore(ServerExpression score, ServerExpression distance) { + return new XsExprImpl.UnsignedLongCallImpl("vec", "vector-score", new Object[]{ score, distance }); } @Override - public ServerExpression vectorScore(ServerExpression score, double similarity, double similarityWeight) { - return vectorScore(score, xs.doubleVal(similarity), xs.doubleVal(similarityWeight)); + public ServerExpression vectorScore(ServerExpression score, double distance, double distanceWeight) { + return vectorScore(score, xs.doubleVal(distance), xs.doubleVal(distanceWeight)); } @Override - public ServerExpression vectorScore(ServerExpression score, ServerExpression similarity, ServerExpression similarityWeight) { - return new XsExprImpl.UnsignedLongCallImpl("vec", "vector-score", new Object[]{ score, similarity, similarityWeight }); + public ServerExpression vectorScore(ServerExpression score, ServerExpression distance, ServerExpression distanceWeight) { + return new XsExprImpl.UnsignedLongCallImpl("vec", "vector-score", new Object[]{ score, distance, distanceWeight }); + } + + + @Override + public ServerExpression vectorScore(ServerExpression score, double distance, double distanceWeight, double weight) { + return vectorScore(score, xs.doubleVal(distance), xs.doubleVal(distanceWeight), xs.doubleVal(weight)); + } + + + @Override + public ServerExpression vectorScore(ServerExpression score, ServerExpression distance, ServerExpression distanceWeight, ServerExpression weight) { + return new XsExprImpl.UnsignedLongCallImpl("vec", "vector-score", new Object[]{ score, distance, distanceWeight, weight }); } static class VectorSeqCallImpl extends BaseTypeImpl.ServerExpressionCallImpl { diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java index d41e4bfee..0f609ad04 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/VectorTest.java @@ -45,6 +45,7 @@ void vectorFunctionsHappyPath() { .limit(1) .bind(op.as("sampleVector", op.vec.vector(sampleVector))) .bind(op.as("cosine", op.vec.cosine(op.col("embedding"), op.col("sampleVector")))) + .bind(op.as("cosineDistanceEmbedding", op.vec.cosineDistance(op.col("embedding"), op.col("sampleVector")))) .bind(op.as("cosineDistance", op.vec.cosineDistance(op.col("sampleVector"), op.col("sampleVector")))) .bind(op.as("dotProduct", op.vec.dotProduct(op.col("embedding"), op.col("sampleVector")))) .bind(op.as("euclideanDistance", op.vec.euclideanDistance(op.col("embedding"), op.col("sampleVector")))) @@ -67,9 +68,16 @@ void vectorFunctionsHappyPath() { assertEquals(1, rows.size()); RowRecord row = rows.get(0); - // Simple sanity checks to verify that the functions ran. + // Simple sanity checks to verify that the functions ran and produce reasonable values. double cosine = row.getDouble("cosine"); - assertTrue((cosine > 0) && (cosine < 1), "Unexpected value: " + cosine); + assertTrue((cosine >= -1) && (cosine <= 1), "Cosine must be between -1 and 1, got: " + cosine); + + double cosineDistanceEmbedding = row.getDouble("cosineDistanceEmbedding"); + assertTrue(cosineDistanceEmbedding >= 0 && cosineDistanceEmbedding <= 2, "Cosine distance must be between 0 and 2, got: " + cosineDistanceEmbedding); + + // this identity (cosine distance = 1 - cosine) should be true for doubles within a small delta, but we won't require exact equality due to inexact floating point math. + assertEquals(1 - cosine, cosineDistanceEmbedding, 0.0001, "Cosine distance should be 1 - cosine"); + double dotProduct = row.getDouble("dotProduct"); Assertions.assertTrue(dotProduct > 0, "Unexpected value: " + dotProduct); double euclideanDistance = row.getDouble("euclideanDistance"); @@ -254,4 +262,75 @@ void trunc() { assertEquals(2.12, trunc2.get(1).asDouble(), 0.001, "Second element should be truncated to 2.12"); assertEquals(3.12, trunc2.get(2).asDouble(), 0.001, "Third element should be truncated to 3.12"); } + + @Test + void vectorScoreWithWeight_primitive() { + // Test vec.vectorScore with all 4 parameters using primitive doubles + PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") + .limit(1) + .bind(op.as("vectorScore1", op.vec.vectorScore(op.xs.unsignedInt(100), 0.3, 0.5, 0.5))) + .bind(op.as("vectorScore2", op.vec.vectorScore(op.xs.unsignedInt(100), 0.3, 0.8, 0.7))) + .bind(op.as("vectorScore3", op.vec.vectorScore(op.xs.unsignedInt(100), 0.3, 0.5, 0.3))); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + RowRecord row = rows.get(0); + + // Verify that vector scores are calculated + double score1 = row.getDouble("vectorScore1"); + assertTrue(score1 > 0, "Vector score should be positive, got: " + score1); + + double score2 = row.getDouble("vectorScore2"); + assertTrue(score2 > 0, "Vector score should be positive, got: " + score2); + + double score3 = row.getDouble("vectorScore3"); + assertTrue(score3 > 0, "Vector score should be positive, got: " + score3); + + // Different weight parameters should produce different scores + assertNotEquals(score1, score2, "Different distanceWeight values should produce different scores"); + assertNotEquals(score1, score3, "Different weight values should produce different scores"); + } + + @Test + void vectorScoreWithWeight_serverExpression() { + // Test vec.vectorScore with all 4 parameters using ServerExpression + PlanBuilder.ModifyPlan plan = op.fromView("vectors", "persons") + .limit(1) + .bind(op.as("vectorScore1", op.vec.vectorScore( + op.xs.unsignedInt(100), + op.xs.doubleVal(0.3), + op.xs.doubleVal(0.5), + op.xs.doubleVal(0.5) + ))) + .bind(op.as("vectorScore2", op.vec.vectorScore( + op.xs.unsignedInt(100), + op.xs.doubleVal(0.3), + op.xs.doubleVal(0.8), + op.xs.doubleVal(0.7) + ))) + .bind(op.as("vectorScore3", op.vec.vectorScore( + op.xs.unsignedInt(100), + op.xs.doubleVal(0.3), + op.xs.doubleVal(0.5), + op.xs.doubleVal(0.3) + ))); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + RowRecord row = rows.get(0); + + // Verify that vector scores are calculated + double score1 = row.getDouble("vectorScore1"); + assertTrue(score1 > 0, "Vector score should be positive, got: " + score1); + + double score2 = row.getDouble("vectorScore2"); + assertTrue(score2 > 0, "Vector score should be positive, got: " + score2); + + double score3 = row.getDouble("vectorScore3"); + assertTrue(score3 > 0, "Vector score should be positive, got: " + score3); + + // Different weight parameters should produce different scores + assertNotEquals(score1, score2, "Different distanceWeight values should produce different scores"); + assertNotEquals(score1, score3, "Different weight values should produce different scores"); + } } From 06d2f80c94a138a3fdc05efb6cc9bfa1f1f43213 Mon Sep 17 00:00:00 2001 From: nagalakshmi Date: Thu, 29 Jan 2026 20:20:19 +0530 Subject: [PATCH 42/70] Added support to execute regressions on arm infrastructure --- Jenkinsfile | 294 ++++++++++++++++++++++++++++++++++++-------- docker-compose.yaml | 6 +- 2 files changed, 245 insertions(+), 55 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5225054e6..329bd558f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,39 +1,56 @@ @Library('shared-libraries') _ -def getJavaHomePath() { - if (env.JAVA_VERSION == "JAVA21") { - return "/home/builder/java/jdk-21.0.1" - } else { - return "/home/builder/java/jdk-17.0.2" - } +def getJavaHomePath(isArm = false) { + if (isArm) { + def version = (env.JAVA_VERSION == "JAVA21") ? "21" : "17" + def path = "/usr/lib/jvm/java-${version}-amazon-corretto.aarch64" + return path + } else { + if (env.JAVA_VERSION == "JAVA21") { + return "/home/builder/java/jdk-21.0.1" + } else { + return "/home/builder/java/jdk-17.0.2" + } + } +} + +def getPlatform(isArm = false) { + return isArm ? "linux/arm64" : "linux/amd64" +} + +def setConverters(isArm = false) { + return isArm ? "false" :"true" } def setupDockerMarkLogic(String image) { - cleanupDocker() - sh label: 'mlsetup', script: '''#!/bin/bash - echo "Removing any running MarkLogic server and clean up MarkLogic data directory" - sudo /usr/local/sbin/mladmin remove - sudo /usr/local/sbin/mladmin cleandata - cd java-client-api - docker compose down -v || true - docker volume prune -f - echo "Using image: "''' + image + ''' - docker pull ''' + image + ''' - MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$JAVA_HOME/bin:$PATH - ./gradlew -i mlWaitTillReady - sleep 3 - ./gradlew -i mlWaitTillReady - ./gradlew mlTestConnections - ./gradlew -i mlDeploy mlReloadSchemas - ''' + cleanupDocker() + sh label: 'mlsetup', script: '''#!/bin/bash + echo "Removing any running MarkLogic server and clean up MarkLogic data directory" + sudo /usr/local/sbin/mladmin remove + sudo /usr/local/sbin/mladmin cleandata + cd java-client-api + export PLATFORM=$PLATFORM + export MARKLOGIC_INSTALL_CONVERTERS=$MARKLOGIC_INSTALL_CONVERTERS + docker compose down -v || true + docker volume prune -f + + echo "Using image: "''' + image + ''' + docker pull ''' + image + ''' + + MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs \ + docker compose up -d --build + echo "Waiting for MarkLogic server to initialize." + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + ./gradlew -i mlWaitTillReady + ./gradlew mlTestConnections + ./gradlew -i mlDeploy mlReloadSchemas + ''' } def runTests(String image) { setupDockerMarkLogic(image) - sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR @@ -45,7 +62,7 @@ def runTests(String image) { rm -rf ~/.m2/repository/com/squareup/okhttp3/ echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + ./gradlew clean build -x test ./gradlew marklogic-client-api:test || true ''' @@ -100,9 +117,9 @@ def runTestsWithReverseProxy(String image) { rm -rf ~/.m2/repository/com/squareup/okhttp3/ echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + ./gradlew clean build -x test - echo "Running marklogic-client-api tests with reverse proxy." + echo "Running marklogic-client-api tests with reverse proxy." ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test || true ''' @@ -180,37 +197,38 @@ pipeline { GRADLE_DIR = ".gradle" DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') + PLATFORM = getPlatform() + MARKLOGIC_INSTALL_CONVERTERS = setConverters() } stages { - stage('pull-request-tests') { when { - not { - expression { return params.regressions } + expression { + return !params.regressions } } steps { setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api - echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." - ls -la ~/.m2/repository/com/squareup + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup rm -rf ~/.m2/repository/com/squareup/okhttp3/ - echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test - echo "Run a sufficient number of tests to verify the PR." + echo "Run a sufficient number of tests to verify the PR." ./gradlew marklogic-client-api:test --tests ReadDocumentPageTest || true echo "Run a test with the reverse proxy server to ensure it's fine." ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:test --tests SearchWithPageLengthTest || true - ''' + ''' } post { always { @@ -225,18 +243,20 @@ pipeline { when { branch 'develop' not { - expression { return params.regressions } + anyOf { + expression { return params.regressions } + } } } steps { sh label: 'publish', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cp ~/.gradle/gradle.properties $GRADLE_USER_HOME; - cd java-client-api - ./gradlew publish - ''' + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cp ~/.gradle/gradle.properties $GRADLE_USER_HOME; + cd java-client-api + ./gradlew publish + ''' } } @@ -259,7 +279,7 @@ pipeline { stage(stageName) { try { - runTests(fullImage) + runTests(fullImage, false) } finally { junit '**/build/**/TEST*.xml' updateWorkspacePermissions() @@ -270,5 +290,175 @@ pipeline { } } } + + stage('provisionInfrastructure'){ + when { + branch 'develop' + expression { return !params.regressions } + } + agent {label 'javaClientLinuxPool'} + + steps{ + script { + withCredentials([ + string(credentialsId: 'aws-region-us-west', variable: 'AWS_REGION'), + string(credentialsId: 'aws-role-headless-testing', variable: 'AWS_ROLE'), + string(credentialsId: 'aws-role-account-headless', variable: 'AWS_ROLE_ACCOUNT') + ]) { + def deploymentResult = deployAWSInstance([ + instanceName: "java-client-instance-${BUILD_NUMBER}", + region: env.AWS_REGION, + credentialsId: 'headlessDbUserEC2', + role: env.AWS_ROLE, + roleAccount: env.AWS_ROLE_ACCOUNT, + branch: 'master' + ]) + + echo "✅ Instance deployed: ${deploymentResult.privateIp}" + echo "✅ Terraform directory: ${deploymentResult.terraformDir}" + echo "✅ Workspace: ${deploymentResult.workspace}" + echo "✅ Status: ${deploymentResult.status}" + + // Store deployment info for cleanup + env.DEPLOYMENT_INSTANCE_NAME = deploymentResult.instanceName + env.DEPLOYMENT_REGION = deploymentResult.region + env.DEPLOYMENT_TERRAFORM_DIR = deploymentResult.terraformDir + env.EC2_PRIVATE_IP = deploymentResult.privateIp + + def nodeName = "java-client-agent-${BUILD_NUMBER}" + def remoteFS = "/space/jenkins_home" + def labels = "java-client-agent-${BUILD_NUMBER}" + def instanceIp = env.EC2_PRIVATE_IP + + // Attach volumes + def volumeResult = attachInstanceVolumes([ + instanceIp: instanceIp, + remoteFS: remoteFS, + branch: 'master' + ]) + + echo "✅ Volume attachment completed: ${volumeResult.volumeAttached}" + echo "✅ Java installed: ${volumeResult.javaInstalled}" + + //Install dependencies AND run init scripts + def depsResult = installDependenciesAndInitScripts([ + instanceIp: instanceIp, + packageFile: 'Packagedependencies', + packageDir: 'terraform-templates/java-client-api', + initScriptsDir: 'terraform-templates/java-client-api/scripts', + initScriptsFile: 'terraform-templates/java-client-api/initscripts' + ]) + + echo "✅ Dependencies installed: ${depsResult.dependenciesInstalled}" + if (depsResult.initScriptsExecuted) { + echo "✅ Init scripts executed: ${depsResult.initScriptsCount} scripts" + } else { + echo "ℹ️ No init scripts configured or executed" + } + + // Use shared library to create Jenkins agent + def agentResult = createJenkinsAgent([ + nodeName: nodeName, + instanceIp: instanceIp, + remoteFS: remoteFS, + labels: labels, + timeoutMinutes: 5, + credentialsId: 'qa-builder-aws' + ]) + + echo "✅ Jenkins agent created: ${agentResult.nodeName}" + echo "✅ Agent status: ${agentResult.status}" + } + } + } + } + + stage('regressions-11 arm infrastructure') { + when { + beforeAgent true + branch 'develop' + expression { return !params.regressions } + expression { return env.EC2_PRIVATE_IP != null } + } + agent { label "java-client-agent-${BUILD_NUMBER}" } + environment { + JAVA_HOME_DIR = getJavaHomePath(true) + PLATFORM = getPlatform(true) + MARKLOGIC_INSTALL_CONVERTERS = setConverters(true) + } + steps { + checkout([$class: 'GitSCM', + branches: scm.branches, + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'java-client-api']], + submoduleCfg: [], + userRemoteConfigs: scm.userRemoteConfigs]) + + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-arm:latest-11") + } + post { + always { + archiveArtifacts artifacts: 'java-client-api/**/build/reports/**/*.html' + junit '**/build/**/TEST*.xml' + updateWorkspacePermissions() + tearDownDocker() + } + } + } + } + + post{ + always { + script { + echo "🧹 Starting cleanup process..." + + try { + // Cleanup Terraform infrastructure + if (env.EC2_PRIVATE_IP) { + echo "🗑️ Cleaning up Terraform resources..." + node('javaClientLinuxPool') { + try { + //`sleep 60` allows AWS resources to stabilize before Terraform destroys them, preventing "resource in use" errors + sleep 60 + unstash "terraform-${BUILD_NUMBER}" + withCredentials([ + string(credentialsId: 'aws-region-us-west', variable: 'AWS_REGION'), + string(credentialsId: 'aws-role-headless-testing', variable: 'AWS_ROLE'), + string(credentialsId: 'aws-role-account-headless', variable: 'AWS_ROLE_ACCOUNT') + ]) { + withAWS(credentials: 'headlessDbUserEC2', region: env.AWS_REGION, role: env.AWS_ROLE, roleAccount: env.AWS_ROLE_ACCOUNT, duration: 3600) { + sh '''#!/bin/bash + export PATH=/home/builder/terraform:$PATH + cd ${WORKSPACE}/${DEPLOYMENT_TERRAFORM_DIR} + terraform workspace select dev + terraform destroy -auto-approve + ''' + } + } + echo "✅ Terraform resources destroyed successfully." + // Cleanup Jenkins agent using shared library function + def nodeName = "java-client-agent-${BUILD_NUMBER}" + echo "🗑️ Cleaning up Jenkins agent: ${nodeName}" + try { + def cleanupResult = cleanupJenkinsAgent(nodeName) + echo "✅ Cleanup result: ${cleanupResult.status} for node: ${cleanupResult.nodeName}" + } catch (Exception jenkinsCleanupException) { + echo "⚠️ Warning: Jenkins agent cleanup failed: ${jenkinsCleanupException.message}" + } + echo "✅ Pipeline cleanup completed successfully." + } catch (Exception terraformException) { + echo "⚠️ Warning: Terraform cleanup failed: ${terraformException.message}" + } + } + } else { + echo "ℹ️ No EC2 instance IP found, skipping Terraform cleanup" + } + } catch (Exception cleanupException) { + echo "⚠️ Warning: Cleanup encountered an error: ${cleanupException.message}" + echo "📋 Continuing with pipeline completion despite cleanup issues..." + } + } + } } } + diff --git a/docker-compose.yaml b/docker-compose.yaml index 9d1dab27e..2a7583e07 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,9 +4,9 @@ services: marklogic: image: "${MARKLOGIC_IMAGE}" - platform: linux/amd64 + platform: "${PLATFORM:-linux/amd64}" environment: - - INSTALL_CONVERTERS=true + - INSTALL_CONVERTERS=${MARKLOGIC_INSTALL_CONVERTERS:-true} - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME=admin - MARKLOGIC_ADMIN_PASSWORD=admin @@ -21,4 +21,4 @@ services: - "8010-8015:8010-8015" # Range of ports used by app servers, at least one of which - 8015 - is created by a test. volumes: - marklogicLogs: + marklogicLogs: \ No newline at end of file From 541b63e64b70311bedd2e232b5da561aeef7c6f5 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 10 Feb 2026 11:06:00 -0500 Subject: [PATCH 43/70] MLE-26918 Added fromView support for incremental write The plan is to dump eval support and just offer fromLexicons and fromView, but need to test out fromView a bit first. Did some refactoring too because the constructors had gotten so ugly - there's now an IncrementalWriteConfig class that holds all the inputs from the Builder, so that filter constructors only need that as an arg. --- .../filter/IncrementalWriteConfig.java | 79 ++++++++++++++++++ .../filter/IncrementalWriteEvalFilter.java | 10 +-- .../filter/IncrementalWriteFilter.java | 80 ++++++++++++------- .../filter/IncrementalWriteOpticFilter.java | 8 +- .../filter/IncrementalWriteViewFilter.java | 57 +++++++++++++ .../filter/IncrementalWriteTest.java | 51 ++++++++++++ .../ml-schemas/tde/incrementalWriteHash.json | 25 ++++++ 7 files changed, 271 insertions(+), 39 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java create mode 100644 test-app/src/main/ml-schemas/tde/incrementalWriteHash.json diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java new file mode 100644 index 000000000..d350f71f1 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.document.DocumentWriteOperation; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Configuration for incremental write filtering. + * + * @since 8.1.0 + */ +public class IncrementalWriteConfig { + + private final String hashKeyName; + private final String timestampKeyName; + private final boolean canonicalizeJson; + private final Consumer skippedDocumentsConsumer; + private final String[] jsonExclusions; + private final String[] xmlExclusions; + private final Map xmlNamespaces; + private final String schemaName; + private final String viewName; + + public IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, + Consumer skippedDocumentsConsumer, + String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces, + String schemaName, String viewName) { + this.hashKeyName = hashKeyName; + this.timestampKeyName = timestampKeyName; + this.canonicalizeJson = canonicalizeJson; + this.skippedDocumentsConsumer = skippedDocumentsConsumer; + this.jsonExclusions = jsonExclusions; + this.xmlExclusions = xmlExclusions; + this.xmlNamespaces = xmlNamespaces != null ? Collections.unmodifiableMap(xmlNamespaces) : null; + this.schemaName = schemaName; + this.viewName = viewName; + } + + public String getHashKeyName() { + return hashKeyName; + } + + public String getTimestampKeyName() { + return timestampKeyName; + } + + public boolean isCanonicalizeJson() { + return canonicalizeJson; + } + + public Consumer getSkippedDocumentsConsumer() { + return skippedDocumentsConsumer; + } + + public String[] getJsonExclusions() { + return jsonExclusions; + } + + public String[] getXmlExclusions() { + return xmlExclusions; + } + + public Map getXmlNamespaces() { + return xmlNamespaces != null ? xmlNamespaces : Collections.emptyMap(); + } + + public String getSchemaName() { + return schemaName; + } + + public String getViewName() { + return viewName; + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index fc0546798..838087203 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -12,9 +12,6 @@ import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.io.JacksonHandle; -import java.util.Map; -import java.util.function.Consumer; - /** * Uses server-side JavaScript code to get the existing hash values for a set of URIs. * @@ -31,9 +28,8 @@ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { response """; - IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + IncrementalWriteEvalFilter(IncrementalWriteConfig config) { + super(config); } @Override @@ -47,7 +43,7 @@ public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { try { JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) - .addVariable("hashKeyName", hashKeyName) + .addVariable("hashKeyName", getConfig().getHashKeyName()) .addVariable("uris", new JacksonHandle(uris)) .evalAs(JsonNode.class); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 730910c0b..61df914a9 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -53,6 +53,8 @@ public static class Builder { private String[] jsonExclusions; private String[] xmlExclusions; private Map xmlNamespaces; + private String schemaName; + private String viewName; /** * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". @@ -128,13 +130,43 @@ public Builder xmlNamespaces(Map namespaces) { return this; } + /** + * Configures the filter to use a TDE view for retrieving hash values instead of field range indexes. + * This approach requires a TDE template to be deployed that extracts the URI and hash metadata. + * + * @param schemaName the schema name of the TDE view + * @param viewName the view name of the TDE view + * @return this builder + */ + public Builder fromView(String schemaName, String viewName) { + boolean schemaEmpty = schemaName == null || schemaName.trim().isEmpty(); + boolean viewEmpty = viewName == null || viewName.trim().isEmpty(); + + if (schemaEmpty && !viewEmpty) { + throw new IllegalArgumentException("Schema name cannot be null or empty when view name is provided"); + } + if (!schemaEmpty && viewEmpty) { + throw new IllegalArgumentException("View name cannot be null or empty when schema name is provided"); + } + + this.schemaName = schemaName; + this.viewName = viewName; + return this; + } + public IncrementalWriteFilter build() { validateJsonExclusions(); validateXmlExclusions(); + IncrementalWriteConfig config = new IncrementalWriteConfig(hashKeyName, timestampKeyName, canonicalizeJson, + skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces, schemaName, viewName); + + if (schemaName != null && viewName != null) { + return new IncrementalWriteViewFilter(config); + } if (useEvalQuery) { - return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + return new IncrementalWriteEvalFilter(config); } - return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + return new IncrementalWriteOpticFilter(config); } private void validateJsonExclusions() { @@ -181,26 +213,18 @@ private void validateXmlExclusions() { } } - protected final String hashKeyName; - private final String timestampKeyName; - private final boolean canonicalizeJson; - private final Consumer skippedDocumentsConsumer; - private final String[] jsonExclusions; - private final String[] xmlExclusions; - private final Map xmlNamespaces; + private final IncrementalWriteConfig config; // Hardcoding this for now, with a good general purpose hashing function. // See https://xxhash.com for benchmarks. private final LongHashFunction hashFunction = LongHashFunction.xx3(); - public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { - this.hashKeyName = hashKeyName; - this.timestampKeyName = timestampKeyName; - this.canonicalizeJson = canonicalizeJson; - this.skippedDocumentsConsumer = skippedDocumentsConsumer; - this.jsonExclusions = jsonExclusions; - this.xmlExclusions = xmlExclusions; - this.xmlNamespaces = xmlNamespaces; + public IncrementalWriteFilter(IncrementalWriteConfig config) { + this.config = config; + } + + public IncrementalWriteConfig getConfig() { + return config; } protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { @@ -230,19 +254,19 @@ protected final DocumentWriteSet filterDocuments(Context context, Function 0) { - content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, jsonExclusions); + if (config.getJsonExclusions() != null && config.getJsonExclusions().length > 0) { + content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, config.getJsonExclusions()); } jc = new JsonCanonicalizer(content); return jc.getEncodedString(); @@ -274,9 +298,9 @@ private String serializeContent(DocumentWriteOperation doc) { logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); } - } else if (xmlExclusions != null && xmlExclusions.length > 0) { + } else if (config.getXmlExclusions() != null && config.getXmlExclusions().length > 0) { try { - content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, xmlNamespaces, xmlExclusions); + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, config.getXmlNamespaces(), config.getXmlExclusions()); } catch (Exception e) { logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); @@ -316,4 +340,6 @@ protected static DocumentWriteOperation addHashToMetadata(DocumentWriteOperation return new DocumentWriteOperationImpl(op.getUri(), newMetadata, op.getContent(), op.getTemporalDocumentURI()); } + + } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java index a52d21ad3..b7fed3099 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -10,7 +10,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.function.Consumer; /** * Uses an Optic query to get the existing hash values for a set of URIs. @@ -19,9 +18,8 @@ */ class IncrementalWriteOpticFilter extends IncrementalWriteFilter { - IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { - super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + IncrementalWriteOpticFilter(IncrementalWriteConfig config) { + super(config); } @Override @@ -39,7 +37,7 @@ public DocumentWriteSet apply(Context context) { Map existingHashes = rowTemplate.query(op -> op.fromLexicons(Map.of( "uri", op.cts.uriReference(), - "hash", op.cts.fieldReference(super.hashKeyName) + "hash", op.cts.fieldReference(getConfig().getHashKeyName()) )).where( op.cts.documentQuery(op.xs.stringSeq(uris)) ), diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java new file mode 100644 index 000000000..02dc75439 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.FailedRequestException; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.row.RowTemplate; + +import java.util.HashMap; +import java.util.Map; + +/** + * Uses an Optic query with fromView to get the existing hash values for a set of URIs from a TDE view. + * This implementation requires a TDE template to be deployed that extracts the URI and hash metadata. + * + * @since 8.1.0 + */ +class IncrementalWriteViewFilter extends IncrementalWriteFilter { + + IncrementalWriteViewFilter(IncrementalWriteConfig config) { + super(config); + } + + @Override + public DocumentWriteSet apply(Context context) { + final String[] uris = context.getDocumentWriteSet().stream() + .filter(op -> DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(op.getOperationType())) + .map(DocumentWriteOperation::getUri) + .toArray(String[]::new); + + RowTemplate rowTemplate = new RowTemplate(context.getDatabaseClient()); + + try { + Map existingHashes = rowTemplate.query(op -> + op.fromView(getConfig().getSchemaName(), getConfig().getViewName()) + .where(op.in(op.col("uri"), op.xs.stringSeq(uris))), + + rows -> { + Map map = new HashMap<>(); + rows.forEach(row -> { + String uri = row.getString("uri"); + String existingHash = row.getString("hash"); + map.put(uri, existingHash); + }); + return map; + } + ); + + return filterDocuments(context, uri -> existingHashes.get(uri)); + } catch (FailedRequestException e) { + String message = "Unable to query for existing incremental write hashes from view " + getConfig().getSchemaName() + "." + getConfig().getViewName() + "; cause: " + e.getMessage(); + throw new FailedRequestException(message, e.getFailedRequest()); + } + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 0ac60b97a..0d0817ed3 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -228,6 +228,57 @@ void binaryDocument() { "expected. Exclusions cannot be specified for them."); } + @Test + void fromView() { + filter = IncrementalWriteFilter.newBuilder() + .fromView("javaClient", "incrementalWriteHash") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + verifyIncrementalWriteWorks(); + } + + @Test + void emptyValuesForFromView() { + filter = IncrementalWriteFilter.newBuilder() + // Empty/null values are ignored, as long as both schema/view are empty/null. This makes life a little + // easier for a connector in that the connector does not need to check for empty/null values. + .fromView("", null) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + verifyIncrementalWriteWorks(); + } + + @Test + void invalidSchemaArg() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder(); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder.fromView(null, "theView")); + assertEquals("Schema name cannot be null or empty when view name is provided", ex.getMessage()); + } + + @Test + void invalidViewArg() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder(); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> builder.fromView("javaClient", null)); + assertEquals("View name cannot be null or empty when schema name is provided", ex.getMessage()); + } + + @Test + void invalidView() { + filter = IncrementalWriteFilter.newBuilder() + .fromView("javaClient", "this-view-doesnt-exist") + .build(); + + writeTenDocuments(); + + assertNotNull(batchFailure.get()); + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("SQL-TABLENOTFOUND"), + "When the user tries to use the incremental write feature with an invalid view, " + + "we should fail with a helpful error message. Actual message: " + message); + } + private void verifyIncrementalWriteWorks() { writeTenDocuments(); verifyDocumentsHasHashInMetadataKey(); diff --git a/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json b/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json new file mode 100644 index 000000000..d5044414c --- /dev/null +++ b/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json @@ -0,0 +1,25 @@ +{ + "template": { + "description": "For incremental write that uses op.fromView instead of op.fromLexicons", + "context": "/doc", + "rows": [ + { + "schemaName": "javaClient", + "viewName": "incrementalWriteHash", + "columns": [ + { + "name": "uri", + "scalarType": "string", + "val": "xdmp:node-uri(.)" + }, + { + "name": "hash", + "scalarType": "string", + "val": "xdmp:node-metadata-value(., 'incrementalWriteHash')", + "nullable": true + } + ] + } + ] + } +} From c505138acc2acb423bb7a4639101643e488b8bd1 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 11 Feb 2026 05:55:25 -0500 Subject: [PATCH 44/70] MLE-27077 Added fix for invalid header for empty doc --- .copyrightconfig | 2 +- .../marklogic/client/impl/OkHttpServices.java | 42 +++++++++++++- .../test/document/ReadDocumentPageTest.java | 56 +++++++++---------- .../src/main/ml-data/sample/empty-file.txt | 0 4 files changed, 65 insertions(+), 35 deletions(-) create mode 100644 test-app/src/main/ml-data/sample/empty-file.txt diff --git a/.copyrightconfig b/.copyrightconfig index c87b8a91b..018996db6 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS, *.txt diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 3d367aa2e..57ec71dc0 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -34,7 +34,9 @@ import jakarta.mail.BodyPart; import jakarta.mail.Header; import jakarta.mail.MessagingException; +import jakarta.mail.internet.ContentDisposition; import jakarta.mail.internet.MimeMultipart; +import jakarta.mail.internet.ParseException; import jakarta.mail.util.ByteArrayDataSource; import jakarta.xml.bind.DatatypeConverter; import okhttp3.*; @@ -1808,16 +1810,50 @@ static private long getHeaderLength(String length) { static private String getHeaderUri(BodyPart part) { try { - if (part != null) { - return part.getFileName(); + if (part == null) { + return null; } - // if it's not found, just return null + + try { + String filename = part.getFileName(); + if (filename != null) { + return filename; + } + } catch (ParseException e) { + // Jakarta Mail's parser failed due to malformed Content-Disposition header. + // Check if MarkLogic sent a malformed "format=" parameter at the end, which violates RFC 2183. + String contentDisposition = getHeader(part, "Content-Disposition"); + if (contentDisposition != null && contentDisposition.matches(".*;\\s*format\\s*=\\s*$")) { + // Remove the trailing "; format=" to fix the malformed header + String cleaned = contentDisposition.replaceFirst(";\\s*format\\s*=\\s*$", "").trim(); + logger.debug("Removed trailing 'format=' from malformed Content-Disposition header: {} -> {}", contentDisposition, cleaned); + return extractFilenameFromContentDisposition(cleaned); + } + throw e; + } + return null; } catch (MessagingException e) { throw new MarkLogicIOException(e); } } + static private String extractFilenameFromContentDisposition(String contentDisposition) { + if (contentDisposition == null) { + return null; + } + try { + // Use Jakarta Mail's ContentDisposition parser to extract the filename parameter. This is the class + // that throws an error when "format=" exists in the value, but that has been removed already. + ContentDisposition cd = new ContentDisposition(contentDisposition); + return cd.getParameter("filename"); + } catch (ParseException e) { + logger.warn("Failed to parse cleaned Content-Disposition header: {}; cause: {}", + contentDisposition, e.getMessage()); + return null; + } + } + static private void updateVersion(DocumentDescriptor descriptor, Headers headers) { updateVersion(descriptor, extractVersion(headers.get(HEADER_ETAG))); } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadDocumentPageTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadDocumentPageTest.java index fdd4a06c1..421aa41cb 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadDocumentPageTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadDocumentPageTest.java @@ -1,28 +1,30 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.document; import com.marklogic.client.DatabaseClient; -import com.marklogic.client.document.*; +import com.marklogic.client.document.DocumentPage; +import com.marklogic.client.document.DocumentRecord; +import com.marklogic.client.document.JSONDocumentManager; import com.marklogic.client.io.BytesHandle; -import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.StructuredQueryBuilder; import com.marklogic.client.query.StructuredQueryDefinition; +import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; -class ReadDocumentPageTest { +class ReadDocumentPageTest extends AbstractClientTest { + /** + * Verifies that the jakarta.mail library, instead of javax.mail, can probably read the URI. + * See MLE-15748, which pertains to issues with javax.mail only allowing US-ASCII characters. + */ @Test - void test() { - Common.deleteUrisWithPattern("/aaa-page/*"); - + void uriWithNonUsAsciiCharacters() { final String uri = "/aaa-page/太田佳伸のＸＭＬファイル.xml"; DocumentRecord documentRecord; try (DatabaseClient client = Common.newClient()) { @@ -38,35 +40,27 @@ void test() { } @Test - @Disabled("Disabling for now because this seems to be a server bug.") - void testEmptyDocWithNoExtension() { - final String collection = "empty-binary-test"; + void emptyTextDocument() { + final String uri = "/sample/empty-file.txt"; try (DatabaseClient client = Common.newClient()) { - writeEmptyDocWithNoFileExtension(client, collection); - JSONDocumentManager documentManager = client.newJSONDocumentManager(); - StructuredQueryDefinition query = new StructuredQueryBuilder().collection(collection); + StructuredQueryDefinition query = new StructuredQueryBuilder().document(uri); DocumentRecord documentRecord; try (DocumentPage documentPage = documentManager.search(query, 1)) { assertTrue(documentPage.hasNext(), "Expected a document in the page, but none was found."); documentRecord = documentPage.next(); } - String uri = documentRecord.getUri(); - assertEquals("/test/empty", uri, "The URI of the empty document should match the one written."); - } - } + String actualUri = documentRecord.getUri(); + assertEquals(uri, actualUri, "The URI of the empty document should match the one written."); - protected void writeEmptyDocWithNoFileExtension(DatabaseClient client, String... collections) { - DocumentMetadataHandle metadata = new DocumentMetadataHandle() - .withCollections(collections) - .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); - // This needs to be a JSON document manager because the empty document is written without a format. - JSONDocumentManager mgr = client.newJSONDocumentManager(); - DocumentWriteSet set = mgr.newWriteSet(); - BytesHandle emptyBytesHandle = new BytesHandle(new byte[0]); - String uri = "/test/empty"; - set.add(uri, metadata, emptyBytesHandle); - mgr.write(set); + IllegalStateException ex = assertThrows(IllegalStateException.class, + () -> documentRecord.getContent(new BytesHandle())); + assertEquals("No bytes to write", ex.getMessage(), + "This assertion is documenting existing behavior, where an empty doc will result in an " + + "exception being thrown when an attempt is made to retrieve its content. " + + "This doesn't seem ideal - returning null seems preferable - but it's the " + + "behavior that has likely always existed."); + } } } diff --git a/test-app/src/main/ml-data/sample/empty-file.txt b/test-app/src/main/ml-data/sample/empty-file.txt new file mode 100644 index 000000000..e69de29bb From e4bd0826dadf15b6981fa70a8c3261ab94ad3c71 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 11 Feb 2026 09:15:27 -0500 Subject: [PATCH 45/70] MLE-27077 Disabling ARM tests These are running on a non-regression build, which we don't want. Modified the branch so they don't run at all for now. --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 329bd558f..bdeffd6f3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ def getPlatform(isArm = false) { return isArm ? "linux/arm64" : "linux/amd64" } -def setConverters(isArm = false) { +def shouldInstallConverters(isArm = false) { return isArm ? "false" :"true" } @@ -198,7 +198,7 @@ pipeline { DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') PLATFORM = getPlatform() - MARKLOGIC_INSTALL_CONVERTERS = setConverters() + MARKLOGIC_INSTALL_CONVERTERS = shouldInstallConverters() } stages { @@ -293,7 +293,7 @@ pipeline { stage('provisionInfrastructure'){ when { - branch 'develop' + branch 'develop-arm' expression { return !params.regressions } } agent {label 'javaClientLinuxPool'} @@ -376,7 +376,7 @@ pipeline { stage('regressions-11 arm infrastructure') { when { beforeAgent true - branch 'develop' + branch 'develop-arm' expression { return !params.regressions } expression { return env.EC2_PRIVATE_IP != null } } @@ -384,7 +384,7 @@ pipeline { environment { JAVA_HOME_DIR = getJavaHomePath(true) PLATFORM = getPlatform(true) - MARKLOGIC_INSTALL_CONVERTERS = setConverters(true) + MARKLOGIC_INSTALL_CONVERTERS = shouldInstallConverters(true) } steps { checkout([$class: 'GitSCM', From 0f9b30cafaf8a527b3acc37690ebacc510ff3c9b Mon Sep 17 00:00:00 2001 From: Steve Biondi Date: Tue, 10 Feb 2026 12:43:48 -0800 Subject: [PATCH 46/70] MLE-23473 Implement fromDocs Add implementation of optic fromDocs to java client. Includes new context method to indicate current row for expressions. a --- .../client/expression/PlanBuilder.java | 84 ++++++ .../marklogic/client/expression/VecExpr.java | 5 + .../client/impl/ColumnBuilderImpl.java | 85 ++++++ .../client/impl/PlanBuilderImpl.java | 112 ++++++++ .../client/impl/PlanBuilderSubImpl.java | 1 + .../client/type/PlanColumnBuilder.java | 85 ++++++ .../client/type/PlanContextExprCall.java | 13 + .../client/test/rows/FromDocsTest.java | 250 ++++++++++++++++++ .../optic/locations/collections.properties | 1 + .../ml-data/optic/locations/new-york.json | 7 + .../optic/locations/permissions.properties | 1 + .../ml-data/optic/locations/portland.json | 7 + .../optic/locations/san-francisco.json | 7 + .../main/ml-data/optic/locations/seattle.json | 7 + .../src/main/ml-data/optic/widgets/alpha.json | 7 + .../src/main/ml-data/optic/widgets/beta.json | 7 + .../optic/widgets/collections.properties | 1 + .../optic/widgets/permissions.properties | 1 + 18 files changed, 681 insertions(+) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/ColumnBuilderImpl.java create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromDocsTest.java create mode 100644 test-app/src/main/ml-data/optic/locations/collections.properties create mode 100644 test-app/src/main/ml-data/optic/locations/new-york.json create mode 100644 test-app/src/main/ml-data/optic/locations/permissions.properties create mode 100644 test-app/src/main/ml-data/optic/locations/portland.json create mode 100644 test-app/src/main/ml-data/optic/locations/san-francisco.json create mode 100644 test-app/src/main/ml-data/optic/locations/seattle.json create mode 100644 test-app/src/main/ml-data/optic/widgets/alpha.json create mode 100644 test-app/src/main/ml-data/optic/widgets/beta.json create mode 100644 test-app/src/main/ml-data/optic/widgets/collections.properties create mode 100644 test-app/src/main/ml-data/optic/widgets/permissions.properties diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java index 7ad4ed0c4..14c91d5e7 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java @@ -247,6 +247,11 @@ protected PlanBuilder( */ public abstract PatchBuilder patchBuilder(XsStringVal contextPath, Map namespaces); /** + * Create column definitions which can be used in op:from-docs. Below functions are used to create column definitions. op:add-column, op:type, op:xpath, op:expr, op:nullable, op:default, op:dimension, op:coordinate-system, op:units, op:collation. + * @return a PlanColumnBuilder object + */ + public abstract PlanColumnBuilder columnBuilder(); + /** * This function creates a placeholder for a literal value in an expression or as the offset or max for a limit. The op:result function throws in an error if the binding parameter does not specify a literal value for the parameter. *

* Provides a client interface to the op:param server function. @@ -403,6 +408,58 @@ protected PlanBuilder( */ public abstract AccessPlan fromView(XsStringVal schema, XsStringVal view, XsStringVal qualifierName, PlanSystemColumn sysCols); /** + * This function dynamically maps semi-structured data (JSON/XML) into rows and columns without deploying a TDE template. It enables ad-hoc queries similar to Virtual Template Views but with additional flexibility, such as node output and advanced column customization. + * @param query Query to select documents for row generation. The query can be a cts:query or a string as a shortcut for a cts:word-query. + * @param contextPath XPath applied to each matched document; each result becomes a row. + * @param columnSpec The column definitions created by using op:column-builder. + * @return a AccessPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. + */ + public abstract AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec); + /** + * This function dynamically maps semi-structured data (JSON/XML) into rows and columns without deploying a TDE template. It enables ad-hoc queries similar to Virtual Template Views but with additional flexibility, such as node output and advanced column customization. + * @param query Query to select documents for row generation. The query can be a cts:query or a string as a shortcut for a cts:word-query. + * @param contextPath XPath applied to each matched document; each result becomes a row. + * @param columnSpec The column definitions created by using op:column-builder. + * @param qualifier Specifies a name for qualifying the column names in place of the combination of the schema and view names. Use cases for the qualifier include self joins. Using an empty string removes all qualification from the column names. + * @return a AccessPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. + */ + public abstract AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec, String qualifier); + /** + * This function dynamically maps semi-structured data (JSON/XML) into rows and columns without deploying a TDE template. It enables ad-hoc queries similar to Virtual Template Views but with additional flexibility, such as node output and advanced column customization. + * @param query Query to select documents for row generation. The query can be a cts:query or a string as a shortcut for a cts:word-query. + * @param contextPath XPath applied to each matched document; each result becomes a row. + * @param columnSpec The column definitions created by using op:column-builder. + * @param qualifier Specifies a name for qualifying the column names in place of the combination of the schema and view names. Use cases for the qualifier include self joins. Using an empty string removes all qualification from the column names. + * @return a AccessPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. + */ + public abstract AccessPlan fromDocs(CtsQueryExpr query, String contextPath, PlanColumnBuilder columnSpec, String qualifier); + /** + * This function dynamically maps semi-structured data (JSON/XML) into rows and columns without deploying a TDE template. It enables ad-hoc queries similar to Virtual Template Views but with additional flexibility, such as node output and advanced column customization. + * @param query Query to select documents for row generation. The query can be a cts:query or a string as a shortcut for a cts:word-query. + * @param contextPath XPath applied to each matched document; each result becomes a row. + * @param columnSpec The column definitions created by using op:column-builder. + * @param qualifier Specifies a name for qualifying the column names in place of the combination of the schema and view names. Use cases for the qualifier include self joins. Using an empty string removes all qualification from the column names. + * @param systemCol An optional named fragment id column returned by op:fragment-id-col. One use case for fragment ids is in joins with lexicons or document content. + * @return a AccessPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. + */ + public abstract AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec, String qualifier, PlanSystemColumn systemCol); + /** + * This function dynamically maps semi-structured data (JSON/XML) into rows and columns without deploying a TDE template. It enables ad-hoc queries similar to Virtual Template Views but with additional flexibility, such as node output and advanced column customization. + * @param query Query to select documents for row generation. The query can be a cts:query or a string as a shortcut for a cts:word-query. + * @param contextPath XPath applied to each matched document; each result becomes a row. + * @param columnSpec The column definitions created by using op:column-builder. + * @param qualifier Specifies a name for qualifying the column names in place of the combination of the schema and view names. Use cases for the qualifier include self joins. Using an empty string removes all qualification from the column names. + * @param systemCol An optional named fragment id column returned by op:fragment-id-col. One use case for fragment ids is in joins with lexicons or document content. + * @param namespaces Namespaces prefix (key) and uri (value). + * @return a AccessPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. + */ + public abstract AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec, String qualifier, PlanSystemColumn systemCol, PlanNamespaceBindingsSeq namespaces); + /** * This function factory returns a new function that takes a name parameter and returns a sem:iri, prepending the specified base URI onto the name. * @param base The base URI to be prepended to the name. * @return a PlanPrefixer object @@ -1130,6 +1187,11 @@ protected PlanBuilder( */ public abstract PlanCase when(ServerExpression condition, ServerExpression... value); /** + * This helper function returns the node from the current processing row. It is to be used in op:xpath, to reference the 'current item' instead of a doc column. + * @return a PlanContextExprCall object + */ + public abstract PlanContextExprCall context(); + /** * This function extracts a sequence of child nodes from a column with node values -- especially, the document nodes from a document join. The path is an XPath (specified as a string) to apply to each node to generate a sequence of nodes as an expression value. *

* Provides a client interface to the op:xpath server function. @@ -1167,6 +1229,24 @@ protected PlanBuilder( * @return a server expression with the node server data type */ public abstract ServerExpression xpath(PlanColumn column, ServerExpression path, PlanNamespaceBindingsSeq namespaceBindings); + /** + * This function extracts a sequence of child nodes from a server expression (such as op:context()) with node values. The path is an XPath (specified as a string) to apply to each node to generate a sequence of nodes as an expression value. + *

+ * Provides a client interface to the op:xpath server function. + * @param expression The server expression (such as op:context()) from which to extract the child nodes. + * @param path An XPath (specified as a string) to apply to each node. (of xs:string) + * @return a server expression with the node server data type + */ + public abstract ServerExpression xpath(ServerExpression expression, String path); + /** + * This function extracts a sequence of child nodes from a server expression (such as op:context()) with node values. The path is an XPath to apply to each node to generate a sequence of nodes as an expression value. + *

+ * Provides a client interface to the op:xpath server function. + * @param expression The server expression (such as op:context()) from which to extract the child nodes. + * @param path An XPath to apply to each node. (of xs:string) + * @return a server expression with the node server data type + */ +public abstract ServerExpression xpath(ServerExpression expression, ServerExpression path); /** * This function constructs a JSON document with the root content, which must be exactly one JSON object or array node. *

@@ -2048,6 +2128,7 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @param start The column is the starting node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} * @return a ModifyPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public abstract ModifyPlan transitiveClosure(String start, String end); /** @@ -2055,6 +2136,7 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @param start The column is the starting node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} * @return a ModifyPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public abstract ModifyPlan transitiveClosure(PlanExprCol start, PlanExprCol end); /** @@ -2063,6 +2145,7 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} * @param options This is either an array of strings or an object containing keys and values for the options to this operator. Options include: min-length This option is the minimum number of steps (edges) required in the path. It should be a non-negative integer, and the default is 1.max-length This option Maximum number of steps (edges) allowed in the path. It should be a non-negative integer, and the default is unlimited. * @return a ModifyPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public abstract ModifyPlan transitiveClosure(String start, String end, PlanTransitiveClosureOptions options); /** @@ -2071,6 +2154,7 @@ public interface ModifyPlan extends PreparePlan, PlanBuilderBase.ModifyPlanBase * @param end The column is the end node of the traversal. The column can be named with a string or a column function such as op:col, op:view-col, or op:schema-col, or constructed from an expression with the op:as function. See {@link PlanBuilder#col(XsStringVal)} * @param options This is either an array of strings or an object containing keys and values for the options to this operator. Options include: min-length This option is the minimum number of steps (edges) required in the path. It should be a non-negative integer, and the default is 1.max-length This option Maximum number of steps (edges) allowed in the path. It should be a non-negative integer, and the default is unlimited. * @return a ModifyPlan object + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public abstract ModifyPlan transitiveClosure(PlanExprCol start, PlanExprCol end, PlanTransitiveClosureOptions options); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java index 87753efb5..3942c4117 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java @@ -156,6 +156,7 @@ public interface VecExpr { * Provides a client interface to the vec:precision server function. * @param vector The input vector to reduce precision. Can be a vector or an empty sequence. (of vec:vector) * @return a server expression with the vec:vector server data type + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public ServerExpression precision(ServerExpression vector); /** @@ -165,6 +166,7 @@ public interface VecExpr { * @param vector The input vector to reduce precision. Can be a vector or an empty sequence. (of vec:vector) * @param precision The number of mantissa bits to preserve (9-32 inclusive). Default is 16. Higher values preserve more precision. If the value is outside the valid range, throw VEC-INVALIDPRECISION. (of xs:unsignedInt) * @return a server expression with the vec:vector server data type + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public ServerExpression precision(ServerExpression vector, ServerExpression precision); /** @@ -210,6 +212,7 @@ public interface VecExpr { * Provides a client interface to the vec:trunc server function. * @param vector The input vector to truncate. (of vec:vector) * @return a server expression with the vec:vector server data type + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public ServerExpression trunc(ServerExpression vector); /** @@ -219,6 +222,7 @@ public interface VecExpr { * @param vector The input vector to truncate. (of vec:vector) * @param n The numbers of decimal places to truncate to. The default is 0. Negative values cause that many digits to the left of the decimal point to be truncated. (of xs:int) * @return a server expression with the vec:vector server data type + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public ServerExpression trunc(ServerExpression vector, int n); /** @@ -228,6 +232,7 @@ public interface VecExpr { * @param vector The input vector to truncate. (of vec:vector) * @param n The numbers of decimal places to truncate to. The default is 0. Negative values cause that many digits to the left of the decimal point to be truncated. (of xs:int) * @return a server expression with the vec:vector server data type + * @since 8.1.0; requires MarkLogic 12.1 or higher. */ public ServerExpression trunc(ServerExpression vector, ServerExpression n); /** diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/ColumnBuilderImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/ColumnBuilderImpl.java new file mode 100644 index 000000000..2acb3e593 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/ColumnBuilderImpl.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl; + +import com.marklogic.client.type.PlanColumnBuilder; +import com.marklogic.client.type.ServerExpression; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@SuppressWarnings("unchecked") +class ColumnBuilderImpl extends BaseTypeImpl.BaseCallImpl implements PlanColumnBuilder, BaseTypeImpl.BaseArgImpl { + + ColumnBuilderImpl() { + super("op", "suboperators", + new BaseTypeImpl.BaseArgImpl[]{ + new BaseTypeImpl.BaseCallImpl("op", "column-builder", new BaseTypeImpl.BaseArgImpl[]{}) + }); + } + + private ColumnBuilderImpl(List args) { + super("op", "suboperators", args.toArray(new BaseTypeImpl.BaseArgImpl[]{})); + } + + public PlanColumnBuilder addColumn(String name) { + return addArg("add-column", name); + } + + public PlanColumnBuilder xpath(String path) { + return addArg("xpath", path); + } + + public PlanColumnBuilder type(String type) { + return addArg("type", type); + } + + public PlanColumnBuilder nullable(boolean nullable) { + return addArg("nullable", new XsValueImpl.BooleanValImpl(nullable)); + } + + public PlanColumnBuilder expr(ServerExpression expression) { + return addArg("expr", expression); + } + + public PlanColumnBuilder defaultValue(String value) { + return addArg("default", value); + } + + public PlanColumnBuilder collation(String collation) { + return addArg("collation", collation); + } + + public PlanColumnBuilder dimension(int dimension) { + return addArg("dimension", dimension); + } + + public PlanColumnBuilder coordinateSystem(String coordinateSystem) { + return addArg("coordinate-system", coordinateSystem); + } + + private PlanColumnBuilder addArg(String functionName, Object... args) { + BaseTypeImpl.BaseArgImpl newArg = new BaseTypeImpl.BaseCallImpl( + "op", functionName, makeArgs(args) + ); + List newArgs = new ArrayList<>(); + newArgs.addAll(Arrays.asList(getArgsImpl())); + newArgs.add(newArg); + return new ColumnBuilderImpl(newArgs); + } + + private BaseTypeImpl.BaseArgImpl[] makeArgs(Object... args) { + List argList = new ArrayList<>(); + for (Object arg : args) { + if (arg instanceof BaseTypeImpl.BaseArgImpl) { + argList.add((BaseTypeImpl.BaseArgImpl) arg); + } else { + // Use Literal for plain values (strings, numbers, etc.) + argList.add(new BaseTypeImpl.Literal(arg)); + } + } + return argList.toArray(new BaseTypeImpl.BaseArgImpl[]{}); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java index 61e095d2d..4adb266d4 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderImpl.java @@ -184,6 +184,18 @@ public PlanExprColSeq colSeq(PlanExprCol... col) { } + @Override + public PlanColumnBuilder columnBuilder() { + return new ColumnBuilderImpl(); + } + + + @Override + public PlanContextExprCall context() { + return new ContextExprCallCallImpl("op", "context", new Object[]{ }); + } + + @Override public PlanAggregateCol count(String name) { return count((name == null) ? (PlanColumn) null : col(name)); @@ -508,6 +520,81 @@ public AccessPlan fromView(XsStringVal schema, XsStringVal view, XsStringVal qua } + @Override + public AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec) { + if (query == null) { + throw new IllegalArgumentException("query parameter for fromDocs() cannot be null"); + } + if (contextPath == null) { + throw new IllegalArgumentException("contextPath parameter for fromDocs() cannot be null"); + } + if (columnSpec == null) { + throw new IllegalArgumentException("columnSpec parameter for fromDocs() cannot be null"); + } + return new PlanBuilderSubImpl.AccessPlanSubImpl("op", "from-docs", new Object[]{ cts.wordQuery(query), xs.string(contextPath), columnSpec }); + } + + + @Override + public AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec, String qualifier) { + if (query == null) { + throw new IllegalArgumentException("query parameter for fromDocs() cannot be null"); + } + if (contextPath == null) { + throw new IllegalArgumentException("contextPath parameter for fromDocs() cannot be null"); + } + if (columnSpec == null) { + throw new IllegalArgumentException("columnSpec parameter for fromDocs() cannot be null"); + } + return new PlanBuilderSubImpl.AccessPlanSubImpl("op", "from-docs", new Object[]{ cts.wordQuery(query), xs.string(contextPath), columnSpec, (qualifier == null) ? null : xs.string(qualifier) }); + } + + + @Override + public AccessPlan fromDocs(CtsQueryExpr query, String contextPath, PlanColumnBuilder columnSpec, String qualifier) { + if (query == null) { + throw new IllegalArgumentException("query parameter for fromDocs() cannot be null"); + } + if (contextPath == null) { + throw new IllegalArgumentException("contextPath parameter for fromDocs() cannot be null"); + } + if (columnSpec == null) { + throw new IllegalArgumentException("columnSpec parameter for fromDocs() cannot be null"); + } + return new PlanBuilderSubImpl.AccessPlanSubImpl("op", "from-docs", new Object[]{ query, xs.string(contextPath), columnSpec, (qualifier == null) ? null : xs.string(qualifier) }); + } + + + @Override + public AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec, String qualifier, PlanSystemColumn systemCol) { + if (query == null) { + throw new IllegalArgumentException("query parameter for fromDocs() cannot be null"); + } + if (contextPath == null) { + throw new IllegalArgumentException("contextPath parameter for fromDocs() cannot be null"); + } + if (columnSpec == null) { + throw new IllegalArgumentException("columnSpec parameter for fromDocs() cannot be null"); + } + return new PlanBuilderSubImpl.AccessPlanSubImpl("op", "from-docs", new Object[]{ cts.wordQuery(query), xs.string(contextPath), columnSpec, (qualifier == null) ? null : xs.string(qualifier), systemCol }); + } + + + @Override + public AccessPlan fromDocs(String query, String contextPath, PlanColumnBuilder columnSpec, String qualifier, PlanSystemColumn systemCol, PlanNamespaceBindingsSeq namespaces) { + if (query == null) { + throw new IllegalArgumentException("query parameter for fromDocs() cannot be null"); + } + if (contextPath == null) { + throw new IllegalArgumentException("contextPath parameter for fromDocs() cannot be null"); + } + if (columnSpec == null) { + throw new IllegalArgumentException("columnSpec parameter for fromDocs() cannot be null"); + } + return new PlanBuilderSubImpl.AccessPlanSubImpl("op", "from-docs", new Object[]{ cts.wordQuery(query), xs.string(contextPath), columnSpec, (qualifier == null) ? null : xs.string(qualifier), systemCol, namespaces }); + } + + @Override public ServerExpression ge(ServerExpression left, ServerExpression right) { if (left == null) { @@ -1285,6 +1372,24 @@ public ServerExpression xpath(PlanColumn column, ServerExpression path, PlanName } + @Override + public ServerExpression xpath(ServerExpression expression, String path) { + return xpath(expression, (path == null) ? (ServerExpression) null : xs.string(path)); + } + + + @Override + public ServerExpression xpath(ServerExpression expression, ServerExpression path) { + if (expression == null) { + throw new IllegalArgumentException("expression parameter for xpath() cannot be null"); + } + if (path == null) { + throw new IllegalArgumentException("path parameter for xpath() cannot be null"); + } + return new BaseTypeImpl.NodeSeqCallImpl("op", "xpath", new Object[]{ expression, path }); + } + + // external type implementations static class AggregateColSeqListImpl extends PlanSeqListImpl implements PlanAggregateColSeq { @@ -1371,6 +1476,13 @@ static class ConditionCallImpl extends PlanCallImpl implements PlanCondition { } + static class ContextExprCallCallImpl extends PlanCallImpl implements PlanContextExprCall { + ContextExprCallCallImpl(String fnPrefix, String fnName, Object[] fnArgs) { + super(fnPrefix, fnName, fnArgs); + } + } + + static class DocColsIdentifierSeqListImpl extends PlanSeqListImpl implements PlanDocColsIdentifierSeq { DocColsIdentifierSeqListImpl(Object[] items) { super(items); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java index 3c11bbf38..2d6580458 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/PlanBuilderSubImpl.java @@ -1246,6 +1246,7 @@ static class AccessPlanSubImpl case "from-doc-uris": case "from-param": case "from-doc-descriptors": + case "from-docs": if (fnArgs.length < 1) { throw new IllegalArgumentException("accessor constructor without parameters: "+fnArgs.length); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java new file mode 100644 index 000000000..fe39ec616 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.type; + +// IMPORTANT: Do not edit. This file is generated. + +/** + * An instance of a column builder returned by the columnBuilder() method + * in a row pipeline. Used to create column definitions for op:from-docs. + */ +public interface PlanColumnBuilder extends ServerExpression { + + /** + * Add a column definition. + * + * @param name The name of the column + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder addColumn(String name); + + /** + * Set the XPath expression for the current column. + * + * @param path The XPath expression + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder xpath(String path); + + /** + * Set the data type for the current column. + * + * @param type The data type (e.g., "string", "integer", "decimal", "vector", "point") + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder type(String type); + + /** + * Specify whether the column can be null. + * + * @param nullable Whether the column can be null + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder nullable(boolean nullable); + + /** + * Set an expression to compute the column value. + * + * @param expression The expression to compute the value + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder expr(ServerExpression expression); + + /** + * Set a default value for the column. + * + * @param value The default value + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder defaultValue(String value); + + /** + * Set the collation for the column. + * + * @param collation The collation URI + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder collation(String collation); + + /** + * Set the dimension for a vector column. + * + * @param dimension The vector dimension + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder dimension(int dimension); + + /** + * Set the coordinate system for a geospatial column. + * + * @param coordinateSystem The coordinate system (e.g., "wgs84") + * @return a PlanColumnBuilder object + */ + PlanColumnBuilder coordinateSystem(String coordinateSystem); +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java new file mode 100644 index 000000000..128e30f38 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.type; + +// IMPORTANT: Do not edit. This file is generated. + +/** + * An instance of a context expression call returned by the context() method + * in a row pipeline. + */ +public interface PlanContextExprCall extends ServerExpression { +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromDocsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromDocsTest.java new file mode 100644 index 000000000..0087d5f3e --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/FromDocsTest.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.test.rows; + +import com.marklogic.client.expression.PlanBuilder; +import com.marklogic.client.expression.PlanBuilder.Plan; +import com.marklogic.client.row.RowManager; +import com.marklogic.client.row.RowRecord; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import com.marklogic.client.test.junit5.RequiresML12; +import com.marklogic.client.type.PlanColumnBuilder; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for the new fromDocs ModifyPlan method that dynamically maps semi-structured + * data (JSON/XML) into rows and columns without deploying a TDE template. + */ +@ExtendWith(RequiresML12.class) +public class FromDocsTest extends AbstractClientTest { + + protected RowManager rowManager; + protected PlanBuilder op; + + @BeforeEach + public void setup() { + Common.client = Common.newClientBuilder().withUsername("rest-reader").build(); + rowManager = Common.client.newRowManager(); + op = rowManager.newPlanBuilder(); + } + + @Test + public void fromDocsBasic() { + PlanColumnBuilder columnSpec = op.columnBuilder() + .addColumn("lastName").xpath("./lastName").type("string") + .addColumn("firstName").xpath("./firstName").type("string") + .addColumn("dob").xpath("./dob").type("string") + .addColumn("instrument").xpath("./instrument").type("string"); + + PlanBuilder.AccessPlan plan = op.fromDocs( + op.cts.wordQuery("Coltrane"), + "/musician", + columnSpec, + "MusicianView" + ); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + + RowRecord row = rows.get(0); + assertEquals("Coltrane", row.getString("MusicianView.lastName")); + assertEquals("John", row.getString("MusicianView.firstName")); + assertEquals("1926-09-23", row.getString("MusicianView.dob")); + assertEquals("saxophone", row.getString("MusicianView.instrument")); + } + + @Test + public void fromDocsWithDefault() { + PlanColumnBuilder columnSpec = op.columnBuilder() + .addColumn("lastName").xpath("./lastName").type("string") + .collation("http://marklogic.com/collation/") + .addColumn("firstName").xpath("./firstName").type("string") + .collation("http://marklogic.com/collation/") + .addColumn("birthDate").xpath("./birthDate").type("string") + .defaultValue("Unknown") + .addColumn("instrument").xpath("./instrument").type("string") + .addColumn("genre").xpath("./genre").type("string") + .defaultValue("Jazz"); + + PlanBuilder.AccessPlan plan = op.fromDocs( + op.cts.wordQuery("Coltrane"), + "/musician", + columnSpec, + "MusicianView" + ); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + + RowRecord row = rows.get(0); + assertEquals("Coltrane", row.getString("MusicianView.lastName")); + assertEquals("John", row.getString("MusicianView.firstName")); + assertEquals("Unknown", row.getString("MusicianView.birthDate")); + assertEquals("saxophone", row.getString("MusicianView.instrument")); + assertEquals("Jazz", row.getString("MusicianView.genre")); + } + + @Test + public void fromDocsWithContextAndXpath() { + PlanColumnBuilder columnSpec = op.columnBuilder() + .addColumn("name").xpath("./name").type("string") + .addColumn("quantity").xpath("./quantity").type("integer") + .addColumn("price").xpath("./price").type("decimal") + .addColumn("totalCost") + .nullable(true) + .expr( + op.multiply( + op.xs.decimal(op.xpath(op.context(), op.xs.string("./price"))), + op.xs.integer(op.xpath(op.context(), op.xs.string("./quantity"))) + ) + ) + .type("decimal"); + + PlanBuilder.ModifyPlan plan = op.fromDocs( + op.cts.wordQuery("Widget"), + "/product", + columnSpec, + "ProductView" + ).orderBy(op.viewCol("ProductView", "name")); + + List rows = resultRows(plan); + assertEquals(2, rows.size()); + + RowRecord firstRow = rows.get(0); + assertEquals("Widget Alpha", firstRow.getString("ProductView.name")); + assertEquals(25.50, firstRow.getDouble("ProductView.price"), 0.01); + assertEquals(100, firstRow.getInt("ProductView.quantity")); + assertEquals(2550, firstRow.getDouble("ProductView.totalCost"), 0.01); + + RowRecord secondRow = rows.get(1); + assertEquals("Widget Beta", secondRow.getString("ProductView.name")); + assertEquals(42.99, secondRow.getDouble("ProductView.price"), 0.01); + assertEquals(250, secondRow.getInt("ProductView.quantity")); + assertEquals(10747.5, secondRow.getDouble("ProductView.totalCost"), 0.01); + } + + @Test + public void fromDocsWithGeospatialQuery() { + PlanColumnBuilder columnSpec = op.columnBuilder() + .addColumn("city").xpath("./city").type("string") + .addColumn("location-wgs84").xpath("./latLong").type("point") + .coordinateSystem("wgs84") + .addColumn("description").xpath("./description").type("string"); + + // find cities within 650 miles of Portland, OR (45.52, -122.68) + // use existing geospatial element index on latLong property, which is defined as a point with wgs84 coordinate system + PlanBuilder.ModifyPlan plan = op.fromDocs( + op.cts.collectionQuery("/optic/locations"), + "/location", + columnSpec, + "LocationView" + ).where( + op.cts.jsonPropertyGeospatialQuery( + "latLong", + op.cts.circle(650, op.cts.point(45.52, -122.68)), + "coordinate-system=wgs84" + ) + ); + + List rows = resultRows(plan); + + assertEquals(3, rows.size()); + + Set cities = rows.stream() + .map(row -> row.getString("LocationView.city")) + .collect(Collectors.toSet()); + + assertTrue(cities.contains("Portland")); + assertTrue(cities.contains("Seattle")); + assertTrue(cities.contains("San Francisco")); + assertFalse(cities.contains("New York")); + } + + @Test + public void fromDocsWithVectorAndDimension() { + PlanColumnBuilder columnSpec = op.columnBuilder() + .addColumn("name").xpath("./name").type("string") + .addColumn("summary").xpath("./summary").type("string") + .addColumn("embedding").xpath("vec:vector(./embedding)") + .type("vector") + .dimension(3) + .addColumn("cosineDistance") + .nullable(true) + .expr( + op.vec.cosineDistance( + op.vec.vector(op.xs.doubleSeq(1, 2, 3)), + op.vec.vector(op.xpath(op.context(), op.xs.string("./embedding"))) + ) + ) + .type("double") + .addColumn("cosine") + .nullable(true) + .expr( + op.vec.cosine( + op.vec.vector(op.xs.doubleSeq(1, 2, 3)), + op.vec.vector(op.xpath(op.context(), op.xs.string("./embedding"))) + ) + ) + .type("double") + .addColumn("euclideanDistance") + .nullable(true) + .expr( + op.math.trunc( + op.vec.euclideanDistance( + op.vec.vector(op.xs.doubleSeq(1, 2, 3)), + op.vec.vector(op.xpath(op.context(), op.xs.string("./embedding"))) + ), + 4 + ) + ) + .type("double"); + + PlanBuilder.ModifyPlan plan = op.fromDocs( + op.cts.wordQuery("*"), + "/person", + columnSpec, + "PersonView" + ).where( + op.lt( + op.vec.cosineDistance( + op.vec.vector(op.xs.doubleSeq(1, 2, 3)), + op.viewCol("PersonView", "embedding") + ), + op.xs.doubleVal(0.1) + ) + ).orderBy(op.viewCol("PersonView", "euclideanDistance")); + + List rows = resultRows(plan); + assertEquals(1, rows.size()); + + // Alice should return. + assertEquals(0.3741, rows.get(0).getDouble("PersonView.euclideanDistance"), 0.0001); + assertEquals(1, rows.get(0).getDouble("PersonView.cosine"), 0.0001); + assertEquals(0, rows.get(0).getDouble("PersonView.cosineDistance"), 0.0001); + + List names = rows.stream() + .map(row -> row.getString("PersonView.name")) + .toList(); + + assertTrue(names.contains("Alice")); + assertFalse(names.contains("Bob")); + } + + /** + * Convenience method for executing a plan and getting the rows back as a list. + */ + protected final List resultRows(Plan plan) { + return rowManager.resultRows(plan).stream().toList(); + } +} diff --git a/test-app/src/main/ml-data/optic/locations/collections.properties b/test-app/src/main/ml-data/optic/locations/collections.properties new file mode 100644 index 000000000..fa7e1d999 --- /dev/null +++ b/test-app/src/main/ml-data/optic/locations/collections.properties @@ -0,0 +1 @@ +*=/optic/locations,test-data diff --git a/test-app/src/main/ml-data/optic/locations/new-york.json b/test-app/src/main/ml-data/optic/locations/new-york.json new file mode 100644 index 000000000..6d7134d69 --- /dev/null +++ b/test-app/src/main/ml-data/optic/locations/new-york.json @@ -0,0 +1,7 @@ +{ + "location": { + "city": "New York", + "latLong": "40.71, -74.01", + "description": "The Big Apple" + } +} diff --git a/test-app/src/main/ml-data/optic/locations/permissions.properties b/test-app/src/main/ml-data/optic/locations/permissions.properties new file mode 100644 index 000000000..c97785496 --- /dev/null +++ b/test-app/src/main/ml-data/optic/locations/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read,rest-writer,update diff --git a/test-app/src/main/ml-data/optic/locations/portland.json b/test-app/src/main/ml-data/optic/locations/portland.json new file mode 100644 index 000000000..48df20d8b --- /dev/null +++ b/test-app/src/main/ml-data/optic/locations/portland.json @@ -0,0 +1,7 @@ +{ + "location": { + "city": "Portland", + "latLong": "45.52, -122.68", + "description": "City of Roses" + } +} diff --git a/test-app/src/main/ml-data/optic/locations/san-francisco.json b/test-app/src/main/ml-data/optic/locations/san-francisco.json new file mode 100644 index 000000000..5ebfc0361 --- /dev/null +++ b/test-app/src/main/ml-data/optic/locations/san-francisco.json @@ -0,0 +1,7 @@ +{ + "location": { + "city": "San Francisco", + "latLong": "37.77, -122.42", + "description": "City by the Bay" + } +} diff --git a/test-app/src/main/ml-data/optic/locations/seattle.json b/test-app/src/main/ml-data/optic/locations/seattle.json new file mode 100644 index 000000000..404773972 --- /dev/null +++ b/test-app/src/main/ml-data/optic/locations/seattle.json @@ -0,0 +1,7 @@ +{ + "location": { + "city": "Seattle", + "latLong": "47.61, -122.33", + "description": "Emerald City" + } +} diff --git a/test-app/src/main/ml-data/optic/widgets/alpha.json b/test-app/src/main/ml-data/optic/widgets/alpha.json new file mode 100644 index 000000000..d15651f9e --- /dev/null +++ b/test-app/src/main/ml-data/optic/widgets/alpha.json @@ -0,0 +1,7 @@ +{ + "product": { + "name": "Widget Alpha", + "price": 25.50, + "quantity": 100 + } +} diff --git a/test-app/src/main/ml-data/optic/widgets/beta.json b/test-app/src/main/ml-data/optic/widgets/beta.json new file mode 100644 index 000000000..1da0fb694 --- /dev/null +++ b/test-app/src/main/ml-data/optic/widgets/beta.json @@ -0,0 +1,7 @@ +{ + "product": { + "name": "Widget Beta", + "price": 42.99, + "quantity": 250 + } +} diff --git a/test-app/src/main/ml-data/optic/widgets/collections.properties b/test-app/src/main/ml-data/optic/widgets/collections.properties new file mode 100644 index 000000000..5676dbf5a --- /dev/null +++ b/test-app/src/main/ml-data/optic/widgets/collections.properties @@ -0,0 +1 @@ +*=/optic/widgets,test-data diff --git a/test-app/src/main/ml-data/optic/widgets/permissions.properties b/test-app/src/main/ml-data/optic/widgets/permissions.properties new file mode 100644 index 000000000..c97785496 --- /dev/null +++ b/test-app/src/main/ml-data/optic/widgets/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read,rest-writer,update From 157641ceb288d538bc08dfa2acb2ccb1a17431fd Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 12 Feb 2026 11:33:25 -0500 Subject: [PATCH 47/70] Revert "MLE-27077 Disabling ARM tests" This reverts commit f10d0377908b60c005a46f222f7f27adecc7ec50. --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bdeffd6f3..329bd558f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ def getPlatform(isArm = false) { return isArm ? "linux/arm64" : "linux/amd64" } -def shouldInstallConverters(isArm = false) { +def setConverters(isArm = false) { return isArm ? "false" :"true" } @@ -198,7 +198,7 @@ pipeline { DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') PLATFORM = getPlatform() - MARKLOGIC_INSTALL_CONVERTERS = shouldInstallConverters() + MARKLOGIC_INSTALL_CONVERTERS = setConverters() } stages { @@ -293,7 +293,7 @@ pipeline { stage('provisionInfrastructure'){ when { - branch 'develop-arm' + branch 'develop' expression { return !params.regressions } } agent {label 'javaClientLinuxPool'} @@ -376,7 +376,7 @@ pipeline { stage('regressions-11 arm infrastructure') { when { beforeAgent true - branch 'develop-arm' + branch 'develop' expression { return !params.regressions } expression { return env.EC2_PRIVATE_IP != null } } @@ -384,7 +384,7 @@ pipeline { environment { JAVA_HOME_DIR = getJavaHomePath(true) PLATFORM = getPlatform(true) - MARKLOGIC_INSTALL_CONVERTERS = shouldInstallConverters(true) + MARKLOGIC_INSTALL_CONVERTERS = setConverters(true) } steps { checkout([$class: 'GitSCM', From b1d1edfd10ff15939291f4ce722185e1250229cf Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 12 Feb 2026 11:33:38 -0500 Subject: [PATCH 48/70] Revert "Added support to execute regressions on arm infrastructure" This reverts commit 06d2f80c94a138a3fdc05efb6cc9bfa1f1f43213. --- Jenkinsfile | 294 ++++++++------------------------------------ docker-compose.yaml | 6 +- 2 files changed, 55 insertions(+), 245 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 329bd558f..5225054e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,56 +1,39 @@ @Library('shared-libraries') _ -def getJavaHomePath(isArm = false) { - if (isArm) { - def version = (env.JAVA_VERSION == "JAVA21") ? "21" : "17" - def path = "/usr/lib/jvm/java-${version}-amazon-corretto.aarch64" - return path - } else { - if (env.JAVA_VERSION == "JAVA21") { - return "/home/builder/java/jdk-21.0.1" - } else { - return "/home/builder/java/jdk-17.0.2" - } - } -} - -def getPlatform(isArm = false) { - return isArm ? "linux/arm64" : "linux/amd64" -} - -def setConverters(isArm = false) { - return isArm ? "false" :"true" +def getJavaHomePath() { + if (env.JAVA_VERSION == "JAVA21") { + return "/home/builder/java/jdk-21.0.1" + } else { + return "/home/builder/java/jdk-17.0.2" + } } def setupDockerMarkLogic(String image) { - cleanupDocker() - sh label: 'mlsetup', script: '''#!/bin/bash - echo "Removing any running MarkLogic server and clean up MarkLogic data directory" - sudo /usr/local/sbin/mladmin remove - sudo /usr/local/sbin/mladmin cleandata - cd java-client-api - export PLATFORM=$PLATFORM - export MARKLOGIC_INSTALL_CONVERTERS=$MARKLOGIC_INSTALL_CONVERTERS - docker compose down -v || true - docker volume prune -f - - echo "Using image: "''' + image + ''' - docker pull ''' + image + ''' - - MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs \ - docker compose up -d --build - echo "Waiting for MarkLogic server to initialize." - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - ./gradlew -i mlWaitTillReady - ./gradlew mlTestConnections - ./gradlew -i mlDeploy mlReloadSchemas - ''' + cleanupDocker() + sh label: 'mlsetup', script: '''#!/bin/bash + echo "Removing any running MarkLogic server and clean up MarkLogic data directory" + sudo /usr/local/sbin/mladmin remove + sudo /usr/local/sbin/mladmin cleandata + cd java-client-api + docker compose down -v || true + docker volume prune -f + echo "Using image: "''' + image + ''' + docker pull ''' + image + ''' + MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$JAVA_HOME/bin:$PATH + ./gradlew -i mlWaitTillReady + sleep 3 + ./gradlew -i mlWaitTillReady + ./gradlew mlTestConnections + ./gradlew -i mlDeploy mlReloadSchemas + ''' } def runTests(String image) { setupDockerMarkLogic(image) + sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR @@ -62,7 +45,7 @@ def runTests(String image) { rm -rf ~/.m2/repository/com/squareup/okhttp3/ echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + ./gradlew clean build -x test ./gradlew marklogic-client-api:test || true ''' @@ -117,9 +100,9 @@ def runTestsWithReverseProxy(String image) { rm -rf ~/.m2/repository/com/squareup/okhttp3/ echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + ./gradlew clean build -x test - echo "Running marklogic-client-api tests with reverse proxy." + echo "Running marklogic-client-api tests with reverse proxy." ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test || true ''' @@ -197,38 +180,37 @@ pipeline { GRADLE_DIR = ".gradle" DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') - PLATFORM = getPlatform() - MARKLOGIC_INSTALL_CONVERTERS = setConverters() } stages { + stage('pull-request-tests') { when { - expression { - return !params.regressions + not { + expression { return params.regressions } } } steps { setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api - echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." - ls -la ~/.m2/repository/com/squareup + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup rm -rf ~/.m2/repository/com/squareup/okhttp3/ - echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test - echo "Run a sufficient number of tests to verify the PR." + echo "Run a sufficient number of tests to verify the PR." ./gradlew marklogic-client-api:test --tests ReadDocumentPageTest || true echo "Run a test with the reverse proxy server to ensure it's fine." ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:test --tests SearchWithPageLengthTest || true - ''' + ''' } post { always { @@ -243,20 +225,18 @@ pipeline { when { branch 'develop' not { - anyOf { - expression { return params.regressions } - } + expression { return params.regressions } } } steps { sh label: 'publish', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cp ~/.gradle/gradle.properties $GRADLE_USER_HOME; - cd java-client-api - ./gradlew publish - ''' + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cp ~/.gradle/gradle.properties $GRADLE_USER_HOME; + cd java-client-api + ./gradlew publish + ''' } } @@ -279,7 +259,7 @@ pipeline { stage(stageName) { try { - runTests(fullImage, false) + runTests(fullImage) } finally { junit '**/build/**/TEST*.xml' updateWorkspacePermissions() @@ -290,175 +270,5 @@ pipeline { } } } - - stage('provisionInfrastructure'){ - when { - branch 'develop' - expression { return !params.regressions } - } - agent {label 'javaClientLinuxPool'} - - steps{ - script { - withCredentials([ - string(credentialsId: 'aws-region-us-west', variable: 'AWS_REGION'), - string(credentialsId: 'aws-role-headless-testing', variable: 'AWS_ROLE'), - string(credentialsId: 'aws-role-account-headless', variable: 'AWS_ROLE_ACCOUNT') - ]) { - def deploymentResult = deployAWSInstance([ - instanceName: "java-client-instance-${BUILD_NUMBER}", - region: env.AWS_REGION, - credentialsId: 'headlessDbUserEC2', - role: env.AWS_ROLE, - roleAccount: env.AWS_ROLE_ACCOUNT, - branch: 'master' - ]) - - echo "✅ Instance deployed: ${deploymentResult.privateIp}" - echo "✅ Terraform directory: ${deploymentResult.terraformDir}" - echo "✅ Workspace: ${deploymentResult.workspace}" - echo "✅ Status: ${deploymentResult.status}" - - // Store deployment info for cleanup - env.DEPLOYMENT_INSTANCE_NAME = deploymentResult.instanceName - env.DEPLOYMENT_REGION = deploymentResult.region - env.DEPLOYMENT_TERRAFORM_DIR = deploymentResult.terraformDir - env.EC2_PRIVATE_IP = deploymentResult.privateIp - - def nodeName = "java-client-agent-${BUILD_NUMBER}" - def remoteFS = "/space/jenkins_home" - def labels = "java-client-agent-${BUILD_NUMBER}" - def instanceIp = env.EC2_PRIVATE_IP - - // Attach volumes - def volumeResult = attachInstanceVolumes([ - instanceIp: instanceIp, - remoteFS: remoteFS, - branch: 'master' - ]) - - echo "✅ Volume attachment completed: ${volumeResult.volumeAttached}" - echo "✅ Java installed: ${volumeResult.javaInstalled}" - - //Install dependencies AND run init scripts - def depsResult = installDependenciesAndInitScripts([ - instanceIp: instanceIp, - packageFile: 'Packagedependencies', - packageDir: 'terraform-templates/java-client-api', - initScriptsDir: 'terraform-templates/java-client-api/scripts', - initScriptsFile: 'terraform-templates/java-client-api/initscripts' - ]) - - echo "✅ Dependencies installed: ${depsResult.dependenciesInstalled}" - if (depsResult.initScriptsExecuted) { - echo "✅ Init scripts executed: ${depsResult.initScriptsCount} scripts" - } else { - echo "ℹ️ No init scripts configured or executed" - } - - // Use shared library to create Jenkins agent - def agentResult = createJenkinsAgent([ - nodeName: nodeName, - instanceIp: instanceIp, - remoteFS: remoteFS, - labels: labels, - timeoutMinutes: 5, - credentialsId: 'qa-builder-aws' - ]) - - echo "✅ Jenkins agent created: ${agentResult.nodeName}" - echo "✅ Agent status: ${agentResult.status}" - } - } - } - } - - stage('regressions-11 arm infrastructure') { - when { - beforeAgent true - branch 'develop' - expression { return !params.regressions } - expression { return env.EC2_PRIVATE_IP != null } - } - agent { label "java-client-agent-${BUILD_NUMBER}" } - environment { - JAVA_HOME_DIR = getJavaHomePath(true) - PLATFORM = getPlatform(true) - MARKLOGIC_INSTALL_CONVERTERS = setConverters(true) - } - steps { - checkout([$class: 'GitSCM', - branches: scm.branches, - doGenerateSubmoduleConfigurations: false, - extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'java-client-api']], - submoduleCfg: [], - userRemoteConfigs: scm.userRemoteConfigs]) - - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-arm:latest-11") - } - post { - always { - archiveArtifacts artifacts: 'java-client-api/**/build/reports/**/*.html' - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() - } - } - } - } - - post{ - always { - script { - echo "🧹 Starting cleanup process..." - - try { - // Cleanup Terraform infrastructure - if (env.EC2_PRIVATE_IP) { - echo "🗑️ Cleaning up Terraform resources..." - node('javaClientLinuxPool') { - try { - //`sleep 60` allows AWS resources to stabilize before Terraform destroys them, preventing "resource in use" errors - sleep 60 - unstash "terraform-${BUILD_NUMBER}" - withCredentials([ - string(credentialsId: 'aws-region-us-west', variable: 'AWS_REGION'), - string(credentialsId: 'aws-role-headless-testing', variable: 'AWS_ROLE'), - string(credentialsId: 'aws-role-account-headless', variable: 'AWS_ROLE_ACCOUNT') - ]) { - withAWS(credentials: 'headlessDbUserEC2', region: env.AWS_REGION, role: env.AWS_ROLE, roleAccount: env.AWS_ROLE_ACCOUNT, duration: 3600) { - sh '''#!/bin/bash - export PATH=/home/builder/terraform:$PATH - cd ${WORKSPACE}/${DEPLOYMENT_TERRAFORM_DIR} - terraform workspace select dev - terraform destroy -auto-approve - ''' - } - } - echo "✅ Terraform resources destroyed successfully." - // Cleanup Jenkins agent using shared library function - def nodeName = "java-client-agent-${BUILD_NUMBER}" - echo "🗑️ Cleaning up Jenkins agent: ${nodeName}" - try { - def cleanupResult = cleanupJenkinsAgent(nodeName) - echo "✅ Cleanup result: ${cleanupResult.status} for node: ${cleanupResult.nodeName}" - } catch (Exception jenkinsCleanupException) { - echo "⚠️ Warning: Jenkins agent cleanup failed: ${jenkinsCleanupException.message}" - } - echo "✅ Pipeline cleanup completed successfully." - } catch (Exception terraformException) { - echo "⚠️ Warning: Terraform cleanup failed: ${terraformException.message}" - } - } - } else { - echo "ℹ️ No EC2 instance IP found, skipping Terraform cleanup" - } - } catch (Exception cleanupException) { - echo "⚠️ Warning: Cleanup encountered an error: ${cleanupException.message}" - echo "📋 Continuing with pipeline completion despite cleanup issues..." - } - } - } } } - diff --git a/docker-compose.yaml b/docker-compose.yaml index 2a7583e07..9d1dab27e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,9 +4,9 @@ services: marklogic: image: "${MARKLOGIC_IMAGE}" - platform: "${PLATFORM:-linux/amd64}" + platform: linux/amd64 environment: - - INSTALL_CONVERTERS=${MARKLOGIC_INSTALL_CONVERTERS:-true} + - INSTALL_CONVERTERS=true - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME=admin - MARKLOGIC_ADMIN_PASSWORD=admin @@ -21,4 +21,4 @@ services: - "8010-8015:8010-8015" # Range of ports used by app servers, at least one of which - 8015 - is created by a test. volumes: - marklogicLogs: \ No newline at end of file + marklogicLogs: From f30e2596a9b24fb13dda54bf6afd15a3d593bee7 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 12 Feb 2026 11:46:39 -0500 Subject: [PATCH 49/70] Restoring all PR tests --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5225054e6..6227a7da4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -206,7 +206,7 @@ pipeline { ./gradlew clean build -x test echo "Run a sufficient number of tests to verify the PR." - ./gradlew marklogic-client-api:test --tests ReadDocumentPageTest || true + ./gradlew marklogic-client-api:test echo "Run a test with the reverse proxy server to ensure it's fine." ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:test --tests SearchWithPageLengthTest || true From aa665631337831408bd05baba6fe6a310a4beb28 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 16 Feb 2026 13:47:06 -0500 Subject: [PATCH 50/70] MLE-27077 Refactor: Reformatted script blocks in Jenkinsfile Also removed the unused block for running all tests with the reverse proxy, will revisit enabling that later. --- Jenkinsfile | 193 ++++++++++++++++++++-------------------------------- 1 file changed, 73 insertions(+), 120 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6227a7da4..7caaa8d2f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,23 +11,23 @@ def getJavaHomePath() { def setupDockerMarkLogic(String image) { cleanupDocker() sh label: 'mlsetup', script: '''#!/bin/bash - echo "Removing any running MarkLogic server and clean up MarkLogic data directory" - sudo /usr/local/sbin/mladmin remove - sudo /usr/local/sbin/mladmin cleandata - cd java-client-api - docker compose down -v || true - docker volume prune -f - echo "Using image: "''' + image + ''' - docker pull ''' + image + ''' - MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build + echo "Removing any running MarkLogic server and clean up MarkLogic data directory" + sudo /usr/local/sbin/mladmin remove + sudo /usr/local/sbin/mladmin cleandata + cd java-client-api + docker compose down -v || true + docker volume prune -f + echo "Using image: "''' + image + ''' + docker pull ''' + image + ''' + MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$JAVA_HOME/bin:$PATH ./gradlew -i mlWaitTillReady - sleep 3 - ./gradlew -i mlWaitTillReady - ./gradlew mlTestConnections - ./gradlew -i mlDeploy mlReloadSchemas + sleep 3 + ./gradlew -i mlWaitTillReady + ./gradlew mlTestConnections + ./gradlew -i mlDeploy mlReloadSchemas ''' } @@ -35,99 +35,52 @@ def runTests(String image) { setupDockerMarkLogic(image) sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api - echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." - ls -la ~/.m2/repository/com/squareup - rm -rf ~/.m2/repository/com/squareup/okhttp3/ + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup + rm -rf ~/.m2/repository/com/squareup/okhttp3/ - echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test - ./gradlew marklogic-client-api:test || true + ./gradlew marklogic-client-api:test || true ''' sh label: 'run ml-development-tools tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew ml-development-tools:test || true + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew ml-development-tools:test || true ''' sh label: 'run fragile functional tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew mlDeploy -PmlForestDataDirectory=/space - ./gradlew marklogic-client-api-functionaltests:runFragileTests || true + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew mlDeploy -PmlForestDataDirectory=/space + ./gradlew marklogic-client-api-functionaltests:runFragileTests || true ''' sh label: 'run fast functional tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew marklogic-client-api-functionaltests:runFastFunctionalTests || true + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew marklogic-client-api-functionaltests:runFastFunctionalTests || true ''' sh label: 'run slow functional tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew marklogic-client-api-functionaltests:runSlowFunctionalTests || true - ''' - - postProcessTestResults() -} - -def runTestsWithReverseProxy(String image) { - setupDockerMarkLogic(image) - - sh label: 'run marklogic-client-api tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - - echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." - ls -la ~/.m2/repository/com/squareup - rm -rf ~/.m2/repository/com/squareup/okhttp3/ - - echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test - - echo "Running marklogic-client-api tests with reverse proxy." - ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api:test || true - ''' - - sh label: 'run fragile functional tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runFragileTests || true - ''' - - sh label: 'run fast functional tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runFastFunctionalTests || true - ''' - - sh label: 'run slow functional tests with reverse proxy', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api - ./gradlew -PtestUseReverseProxyServer=true runReverseProxyServer marklogic-client-api-functionaltests:runSlowFunctionalTests || true + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api + ./gradlew marklogic-client-api-functionaltests:runSlowFunctionalTests || true ''' postProcessTestResults() @@ -135,20 +88,20 @@ def runTestsWithReverseProxy(String image) { def postProcessTestResults() { sh label: 'post-test-process', script: ''' - cd java-client-api - mkdir -p marklogic-client-api-functionaltests/build/test-results/runFragileTests - mkdir -p marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests - mkdir -p marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests - cd $WORKSPACE/java-client-api/marklogic-client-api/build/test-results/test/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/ml-development-tools/build/test-results/test/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFragileTests/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml - cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests/ - sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd java-client-api + mkdir -p marklogic-client-api-functionaltests/build/test-results/runFragileTests + mkdir -p marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests + mkdir -p marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests + cd $WORKSPACE/java-client-api/marklogic-client-api/build/test-results/test/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/ml-development-tools/build/test-results/test/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFragileTests/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runFastFunctionalTests/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml + cd $WORKSPACE/java-client-api/marklogic-client-api-functionaltests/build/test-results/runSlowFunctionalTests/ + sed -i "s/classname=\\"/classname=\\"${STAGE_NAME}-/g" TEST*.xml ''' } @@ -193,19 +146,19 @@ pipeline { steps { setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cd java-client-api + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd java-client-api - echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." - ls -la ~/.m2/repository/com/squareup + echo "Temporary fix for mysterious issue with okhttp3 being corrupted in local Maven cache." + ls -la ~/.m2/repository/com/squareup rm -rf ~/.m2/repository/com/squareup/okhttp3/ - echo "Ensure all subprojects can be built first." - ./gradlew clean build -x test + echo "Ensure all subprojects can be built first." + ./gradlew clean build -x test - echo "Run a sufficient number of tests to verify the PR." + echo "Run a sufficient number of tests to verify the PR." ./gradlew marklogic-client-api:test echo "Run a test with the reverse proxy server to ensure it's fine." @@ -230,12 +183,12 @@ pipeline { } steps { sh label: 'publish', script: '''#!/bin/bash - export JAVA_HOME=$JAVA_HOME_DIR - export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - cp ~/.gradle/gradle.properties $GRADLE_USER_HOME; - cd java-client-api - ./gradlew publish + export JAVA_HOME=$JAVA_HOME_DIR + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cp ~/.gradle/gradle.properties $GRADLE_USER_HOME; + cd java-client-api + ./gradlew publish ''' } } From fcc5cc0b1ad514f2f3234764b81d6a1f108762bd Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 16 Feb 2026 14:05:16 -0500 Subject: [PATCH 51/70] PDP-817 Modifying Jenkinsfile to run ARM tests in regressions --- Jenkinsfile | 178 +++++++++++++++++++++++++++++++++++++++++++- docker-compose.yaml | 4 +- 2 files changed, 179 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7caaa8d2f..85ada6c25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,11 @@ def getJavaHomePath() { } } +def getJavaHomePathForARM() { + def version = (env.JAVA_VERSION == "JAVA21") ? "21" : "17" + return "/usr/lib/jvm/java-${version}-amazon-corretto.aarch64" +} + def setupDockerMarkLogic(String image) { cleanupDocker() sh label: 'mlsetup', script: '''#!/bin/bash @@ -24,7 +29,6 @@ def setupDockerMarkLogic(String image) { export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$JAVA_HOME/bin:$PATH ./gradlew -i mlWaitTillReady - sleep 3 ./gradlew -i mlWaitTillReady ./gradlew mlTestConnections ./gradlew -i mlDeploy mlReloadSchemas @@ -133,6 +137,8 @@ pipeline { GRADLE_DIR = ".gradle" DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') + PLATFORM = "linux/amd64" + MARKLOGIC_INSTALL_CONVERTERS = "true" } stages { @@ -223,5 +229,175 @@ pipeline { } } } + + stage('provisionInfrastructure') { + when { + branch 'develop' + expression { return params.regressions } + } + agent { label 'javaClientLinuxPool' } + + steps { + script { + withCredentials([ + string(credentialsId: 'aws-region-us-west', variable: 'AWS_REGION'), + string(credentialsId: 'aws-role-headless-testing', variable: 'AWS_ROLE'), + string(credentialsId: 'aws-role-account-headless', variable: 'AWS_ROLE_ACCOUNT') + ]) { + def deploymentResult = deployAWSInstance([ + instanceName : "java-client-instance-${BUILD_NUMBER}", + region : env.AWS_REGION, + credentialsId: 'headlessDbUserEC2', + role : env.AWS_ROLE, + roleAccount : env.AWS_ROLE_ACCOUNT, + branch : 'master' + ]) + + echo "✅ Instance deployed: ${deploymentResult.privateIp}" + echo "✅ Terraform directory: ${deploymentResult.terraformDir}" + echo "✅ Workspace: ${deploymentResult.workspace}" + echo "✅ Status: ${deploymentResult.status}" + + // Store deployment info for cleanup + env.DEPLOYMENT_INSTANCE_NAME = deploymentResult.instanceName + env.DEPLOYMENT_REGION = deploymentResult.region + env.DEPLOYMENT_TERRAFORM_DIR = deploymentResult.terraformDir + env.EC2_PRIVATE_IP = deploymentResult.privateIp + + def nodeName = "java-client-agent-${BUILD_NUMBER}" + def remoteFS = "/space/jenkins_home" + def labels = "java-client-agent-${BUILD_NUMBER}" + def instanceIp = env.EC2_PRIVATE_IP + + // Attach volumes + def volumeResult = attachInstanceVolumes([ + instanceIp: instanceIp, + remoteFS : remoteFS, + branch : 'master' + ]) + + echo "✅ Volume attachment completed: ${volumeResult.volumeAttached}" + echo "✅ Java installed: ${volumeResult.javaInstalled}" + + //Install dependencies AND run init scripts + def depsResult = installDependenciesAndInitScripts([ + instanceIp : instanceIp, + packageFile : 'Packagedependencies', + packageDir : 'terraform-templates/java-client-api', + initScriptsDir : 'terraform-templates/java-client-api/scripts', + initScriptsFile: 'terraform-templates/java-client-api/initscripts' + ]) + + echo "✅ Dependencies installed: ${depsResult.dependenciesInstalled}" + if (depsResult.initScriptsExecuted) { + echo "✅ Init scripts executed: ${depsResult.initScriptsCount} scripts" + } else { + echo "ℹ️ No init scripts configured or executed" + } + + // Use shared library to create Jenkins agent + def agentResult = createJenkinsAgent([ + nodeName : nodeName, + instanceIp : instanceIp, + remoteFS : remoteFS, + labels : labels, + timeoutMinutes: 5, + credentialsId : 'qa-builder-aws' + ]) + + echo "✅ Jenkins agent created: ${agentResult.nodeName}" + echo "✅ Agent status: ${agentResult.status}" + } + } + } + } + + stage('regressions-11 arm infrastructure') { + when { + beforeAgent true + branch 'develop' + expression { return params.regressions } + expression { return env.EC2_PRIVATE_IP != null } + } + agent { label "java-client-agent-${BUILD_NUMBER}" } + environment { + JAVA_HOME_DIR = getJavaHomePathForARM() + PLATFORM = "linux/arm64" + MARKLOGIC_INSTALL_CONVERTERS = "false" + } + steps { + checkout([$class : 'GitSCM', + branches : scm.branches, + doGenerateSubmoduleConfigurations: false, + extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'java-client-api']], + submoduleCfg : [], + userRemoteConfigs : scm.userRemoteConfigs]) + + runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi9-arm:latest-11") + } + post { + always { + archiveArtifacts artifacts: 'java-client-api/**/build/reports/**/*.html' + junit '**/build/**/TEST*.xml' + updateWorkspacePermissions() + tearDownDocker() + } + } + } + } + + post { + always { + script { + echo "🧹 Starting cleanup process..." + + try { + // Cleanup Terraform infrastructure + if (env.EC2_PRIVATE_IP) { + echo "🗑️ Cleaning up Terraform resources..." + node('javaClientLinuxPool') { + try { + //`sleep 60` allows AWS resources to stabilize before Terraform destroys them, preventing "resource in use" errors + sleep 60 + unstash "terraform-${BUILD_NUMBER}" + withCredentials([ + string(credentialsId: 'aws-region-us-west', variable: 'AWS_REGION'), + string(credentialsId: 'aws-role-headless-testing', variable: 'AWS_ROLE'), + string(credentialsId: 'aws-role-account-headless', variable: 'AWS_ROLE_ACCOUNT') + ]) { + withAWS(credentials: 'headlessDbUserEC2', region: env.AWS_REGION, role: env.AWS_ROLE, roleAccount: env.AWS_ROLE_ACCOUNT, duration: 3600) { + sh '''#!/bin/bash + export PATH=/home/builder/terraform:$PATH + cd ${WORKSPACE}/${DEPLOYMENT_TERRAFORM_DIR} + terraform workspace select dev + terraform destroy -auto-approve + ''' + } + } + echo "✅ Terraform resources destroyed successfully." + // Cleanup Jenkins agent using shared library function + def nodeName = "java-client-agent-${BUILD_NUMBER}" + echo "🗑️ Cleaning up Jenkins agent: ${nodeName}" + try { + def cleanupResult = cleanupJenkinsAgent(nodeName) + echo "✅ Cleanup result: ${cleanupResult.status} for node: ${cleanupResult.nodeName}" + } catch (Exception jenkinsCleanupException) { + echo "⚠️ Warning: Jenkins agent cleanup failed: ${jenkinsCleanupException.message}" + } + echo "✅ Pipeline cleanup completed successfully." + } catch (Exception terraformException) { + echo "⚠️ Warning: Terraform cleanup failed: ${terraformException.message}" + } + } + } else { + echo "ℹ️ No EC2 instance IP found, skipping Terraform cleanup" + } + } catch (Exception cleanupException) { + echo "⚠️ Warning: Cleanup encountered an error: ${cleanupException.message}" + echo "📋 Continuing with pipeline completion despite cleanup issues..." + } + } + } } } + diff --git a/docker-compose.yaml b/docker-compose.yaml index 9d1dab27e..f8f19d23b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,9 +4,9 @@ services: marklogic: image: "${MARKLOGIC_IMAGE}" - platform: linux/amd64 + platform: "${PLATFORM:-linux/amd64}" environment: - - INSTALL_CONVERTERS=true + - INSTALL_CONVERTERS=${MARKLOGIC_INSTALL_CONVERTERS:-true} - MARKLOGIC_INIT=true - MARKLOGIC_ADMIN_USERNAME=admin - MARKLOGIC_ADMIN_PASSWORD=admin From b721b7180454a62cd7767c494a5095797a0bf2a8 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 16 Feb 2026 17:21:20 -0500 Subject: [PATCH 52/70] MLE-26918 Added FilterException to identify filter-related errors Important for the batch retrier in the Spark connector so it doesn't retry a batch when the filter fails. Improved the code for closing handles as well in BatchWriter. --- .../datamovement/filter/FilterException.java | 18 +++++ .../filter/IncrementalWriteOpticFilter.java | 4 ++ .../filter/IncrementalWriteViewFilter.java | 4 ++ .../client/datamovement/impl/BatchWriter.java | 65 +++++++++++-------- .../com/marklogic/client/impl/IoUtil.java | 18 ++++- .../filter/IncrementalWriteTest.java | 19 +++++- 6 files changed, 96 insertions(+), 32 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java new file mode 100644 index 000000000..712f52cdf --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.datamovement.DataMovementException; + +/** + * Any exception thrown by execution of a {@code DocumentWriteSetFilter} will be wrapped in this exception and + * rethrown by the {@code WriteBatcher}, allowing failure listeners to distinguish filter exceptions from other + * exceptions that may occur during batch processing. + */ +public class FilterException extends DataMovementException { + + public FilterException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java index b7fed3099..7d0d709a6 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -53,6 +53,10 @@ public DocumentWriteSet apply(Context context) { } ); + if (logger.isDebugEnabled()) { + logger.debug("Retrieved {} existing hashes for batch of size {}", existingHashes.size(), uris.length); + } + return filterDocuments(context, uri -> existingHashes.get(uri)); } catch (FailedRequestException e) { String message = "Unable to query for existing incremental write hashes; cause: " + e.getMessage(); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java index 02dc75439..994512f54 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java @@ -48,6 +48,10 @@ public DocumentWriteSet apply(Context context) { } ); + if (logger.isDebugEnabled()) { + logger.debug("Retrieved {} existing hashes for batch of size {}", existingHashes.size(), uris.length); + } + return filterDocuments(context, uri -> existingHashes.get(uri)); } catch (FailedRequestException e) { String message = "Unable to query for existing incremental write hashes from view " + getConfig().getSchemaName() + "." + getConfig().getViewName() + "; cause: " + e.getMessage(); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java index a2ebe835d..7c8594a89 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java @@ -1,12 +1,14 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; import com.marklogic.client.datamovement.DocumentWriteSetFilter; +import com.marklogic.client.datamovement.filter.FilterException; import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.IoUtil; import com.marklogic.client.io.Format; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,29 +27,36 @@ public void run() { return; } - try { + if (logger.isTraceEnabled()) { logger.trace("Begin write batch {} to forest on host '{}'", batchWriteSet.getBatchNumber(), batchWriteSet.getClient().getHost()); + } + + DocumentWriteSet documentWriteSet = batchWriteSet.getDocumentWriteSet(); - DocumentWriteSet documentWriteSet = batchWriteSet.getDocumentWriteSet(); - if (filter != null) { + if (filter != null) { + try { documentWriteSet = filter.apply(batchWriteSet); - if (documentWriteSet == null || documentWriteSet.isEmpty()) { - logger.debug("Filter returned empty write set for batch {}, skipping write", batchWriteSet.getBatchNumber()); - closeAllHandles(); - return; - } - batchWriteSet.updateWithFilteredDocumentWriteSet(documentWriteSet); + } catch (Exception e) { + closeAllHandles(); + String message = String.format("Unable to apply filter to batch %d; cause: %s", batchWriteSet.getBatchNumber(), e.getMessage()); + onFailure(new FilterException(message, e)); + return; + } + if (documentWriteSet == null || documentWriteSet.isEmpty()) { + closeAllHandles(); + logger.debug("Filter returned empty write set for batch {}, skipping write", batchWriteSet.getBatchNumber()); + return; } + batchWriteSet.updateWithFilteredDocumentWriteSet(documentWriteSet); + } + try { writeDocuments(documentWriteSet); - - // This seems like it should be part of a finally block - but it's able to throw an exception. Which implies - // that onFailure() should occur when this fails, which seems odd??? - closeAllHandles(); - onSuccess(); } catch (Throwable t) { onFailure(t); + } finally { + closeAllHandles(); } } @@ -79,21 +88,21 @@ private void onFailure(Throwable t) { } } - private void closeAllHandles() throws Throwable { - Throwable lastThrowable = null; + /** + * This used to throw a Throwable... but it's not clear what a user would ever do with that if a content handle + * cannot be closed. Instead, this has been altered to use closeQuietly. + */ + private void closeAllHandles() { for (DocumentWriteOperation doc : batchWriteSet.getDocumentWriteSet()) { - try { - if (doc.getContent() instanceof Closeable closeable) { - closeable.close(); - } - if (doc.getMetadata() instanceof Closeable closeable) { - closeable.close(); - } - } catch (Throwable t) { - logger.error("Error closing all handles in BatchWriter", t); - lastThrowable = t; + if (doc == null) { + continue; + } + if (doc.getContent() instanceof Closeable closeable) { + IoUtil.closeQuietly(closeable); + } + if (doc.getMetadata() instanceof Closeable closeable) { + IoUtil.closeQuietly(closeable); } } - if (lastThrowable != null) throw lastThrowable; } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java index 17a910a99..c84f08c18 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/IoUtil.java @@ -1,9 +1,12 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; +import org.slf4j.LoggerFactory; + import java.io.ByteArrayOutputStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -23,4 +26,17 @@ static byte[] streamToBytes(InputStream stream) throws IOException { buffer.flush(); return buffer.toByteArray(); } + + static void closeQuietly(Closeable closeable) { + // Reinvented here as we don't yet have a dependency on a 3rd party library that provides this method, and it's + // not worth bringing in a dependency just for this. + if (closeable != null) { + try { + closeable.close(); + } catch (IOException e) { + LoggerFactory.getLogger(IoUtil.class) + .warn("Unexpected exception while closing stream: %s".formatted(e.getMessage()), e); + } + } + } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 0d0817ed3..90258ec78 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -4,6 +4,7 @@ package com.marklogic.client.datamovement.filter; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.FailedRequestException; import com.marklogic.client.document.*; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.io.*; @@ -128,10 +129,22 @@ void noRangeIndexForField() { writeTenDocuments(); assertNotNull(batchFailure.get()); - String message = batchFailure.get().getMessage(); - assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("XDMP-FIELDRIDXNOTFOUND"), + assertTrue(batchFailure.get() instanceof FilterException, + "If the filter fails, the exception should be wrapped in a FilterException so that WriteBatcher " + + "failure listeners can distinguish the difference between a filter failure and a write failure. " + + "If the filter fails, there is likely no reason to retry the request. Actual exception class: " + batchFailure.get().getClass()); + + FilterException filterException = (FilterException) batchFailure.get(); + String message = filterException.getMessage(); + assertTrue(message.startsWith("Unable to apply filter to batch 1; cause: "), + "The filter exception message should provide context that the error happened when a filter was applied. " + + "Actual message: " + message); + + assertTrue(filterException.getCause() instanceof FailedRequestException); + String causeMessage = filterException.getCause().getMessage(); + assertTrue(causeMessage.contains("Unable to query for existing incremental write hashes") && causeMessage.contains("XDMP-FIELDRIDXNOTFOUND"), "When the user tries to use the incremental write feature without the required range index, we should " + - "fail with a helpful error message. Actual message: " + message); + "fail with a helpful error message. Actual message: " + causeMessage); } @Test From 5e003350af114b456502c377ab70be0afb9111ac Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 17 Feb 2026 16:41:58 -0500 Subject: [PATCH 53/70] MLE-27078 Added incorrect test to capture doc format error --- .copyrightconfig | 2 +- .../test/document/ReadJsonAsBinaryTest.java | 59 +++++++++++++++++++ .../main/ml-modules/transforms/toBinary.xqy | 16 +++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java create mode 100644 test-app/src/main/ml-modules/transforms/toBinary.xqy diff --git a/.copyrightconfig b/.copyrightconfig index 018996db6..024c7ca7f 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS, *.txt +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS, *.txt, *.xqy diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java new file mode 100644 index 000000000..124c005da --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.test.document; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.document.*; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.InputStreamHandle; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ReadJsonAsBinaryTest extends AbstractClientTest { + + @Test + @Disabled("See MLE-27191 for description of the server bug") + void test() { + try (DatabaseClient client = Common.newClient()) { + final String uri = "/a.json"; + writeJsonAsBinary(uri, client); + + GenericDocumentManager docManager = client.newDocumentManager(); + + // A multipart request returns the correct document format. + try (DocumentPage page = docManager.read(uri)) { + DocumentRecord record = page.next(); + assertEquals(uri, record.getUri()); + assertEquals("application/json", record.getMimetype()); + assertEquals(Format.BINARY, record.getFormat()); + } + + // But a request for a single URI does not! + try (InputStreamHandle handle = docManager.read(uri, new InputStreamHandle())) { + assertEquals("application/json", handle.getMimetype()); + assertEquals(Format.BINARY, handle.getFormat(), "Unfortunately due to MLE-27191, the " + + "format is JSON instead of binary. So this assertion will fail until that bug is fixed. " + + "For a Java Client user, the workaround is to use the read method above and get a DocumentPage " + + "back, as a multipart response seems to identify the correct headers."); + } + + } + } + + private void writeJsonAsBinary(String uri, DatabaseClient client) { + JSONDocumentManager jsonDocManager = client.newJSONDocumentManager(); + DocumentMetadataHandle metadata = new DocumentMetadataHandle() + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + jsonDocManager.setWriteTransform(new ServerTransform("toBinary")); + jsonDocManager.write(uri, metadata, new JacksonHandle(new ObjectMapper().createObjectNode().put("a", 1))); + } +} diff --git a/test-app/src/main/ml-modules/transforms/toBinary.xqy b/test-app/src/main/ml-modules/transforms/toBinary.xqy new file mode 100644 index 000000000..52229049c --- /dev/null +++ b/test-app/src/main/ml-modules/transforms/toBinary.xqy @@ -0,0 +1,16 @@ +xquery version "1.0-ml"; + +module namespace transform = "http://marklogic.com/rest-api/transform/toBinary"; + +(: +Demonstrates how a transform can be used to store JSON or XML as a binary document. This is useful for a use case of +a document with a URI that ends in e.g. ".json" or ".xml" but the user wants to treat the content as binary data so +it is not indexed. +:) +declare function transform($context as map:map, $params as map:map, $content as document-node()) as document-node() +{ + let $node := $content/node() + let $enc := xdmp:base64-encode(xdmp:quote($node)) + let $bin := xs:hexBinary(xs:base64Binary($enc)) + return document { binary { $bin } } +}; From 04aef10b3774a42703cd4d0d17e101a733c7dc8c Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 18 Feb 2026 10:02:49 -0500 Subject: [PATCH 54/70] MLE-27078 More tests for this bug Made a very confusing private field easier to understand in DocumentManagerImpl. --- .env | 2 +- Jenkinsfile | 3 +- .../client/impl/DocumentManagerImpl.java | 45 +++++++------- .../test/document/ReadJsonAsBinaryTest.java | 60 +++++++++++++------ test-app/build.gradle | 7 ++- 5 files changed, 75 insertions(+), 42 deletions(-) diff --git a/.env b/.env index c67e5ae05..1eaaf27ef 100644 --- a/.env +++ b/.env @@ -4,4 +4,4 @@ MARKLOGIC_LOGS_VOLUME=./docker/marklogic/logs MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12 # Latest public release -#MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest +# MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest diff --git a/Jenkinsfile b/Jenkinsfile index 85ada6c25..3243eb7b8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -150,7 +150,8 @@ pipeline { } } steps { - setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") + setupDockerMarkLogic("progressofficial/marklogic-db:latest") + sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DocumentManagerImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DocumentManagerImpl.java index f7ce18d9a..0eaf82402 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DocumentManagerImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DocumentManagerImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -48,24 +48,29 @@ abstract class DocumentManagerImpl processedMetadata = new HashSet() { + // UGH. This state variable is used to track if the user modifies the metadataCategories set. That's because the + // default value of "ALL" means something different than if the user sets it to "ALL". Specifically, on a bulk read, + // the expectation is that no metadata is retrieved. So if the value is "ALL" and the user didn't set that, we + // don't get metadata. But if the value is "ALL" and the user did set that, we get all the metadata. UGH! + private boolean isMetadataCategoriesModified; + + final private Set metadataCategories = new HashSet<>() { @Override public boolean add(Metadata e) { - isProcessedMetadataModified = true; + isMetadataCategoriesModified = true; return super.add(e); } @Override public boolean addAll(Collection c) { - isProcessedMetadataModified = true; + isMetadataCategoriesModified = true; return super.addAll(c); } }; { - processedMetadata.add(Metadata.ALL); + metadataCategories.add(Metadata.ALL); // we need to know if the user modifies after us - isProcessedMetadataModified = false; + isMetadataCategoriesModified = false; } private RESTServices services; @@ -113,24 +118,24 @@ public void setContentFormat(Format format) { @Override public void setMetadataCategories(Set categories) { clearMetadataCategories(); - processedMetadata.addAll(categories); + metadataCategories.addAll(categories); } @Override public void setMetadataCategories(Metadata... categories) { clearMetadataCategories(); for (Metadata category : categories) - processedMetadata.add(category); + metadataCategories.add(category); } @Override public Set getMetadataCategories() { - return processedMetadata; + return metadataCategories; } @Override public void clearMetadataCategories() { - processedMetadata.clear(); + metadataCategories.clear(); } @Override @@ -374,7 +379,7 @@ public T read(DocumentDescriptor desc, requestLogger, desc, transaction, - (metadataHandle != null) ? processedMetadata : null, + (metadataHandle != null) ? metadataCategories : null, mergeTransformParameters((transform != null) ? transform : getReadTransform(), extraParams), metadataHandle, contentHandle); @@ -448,7 +453,7 @@ public DocumentPage read(long serverTimestamp, ServerTransform transform, Transa transaction, // the default for bulk is no metadata, which differs from the normal // default of ALL - (isProcessedMetadataModified || !withContent) ? processedMetadata : null, + (isMetadataCategoriesModified || !withContent) ? metadataCategories : null, nonDocumentFormat, mergeTransformParameters((transform != null) ? transform : getReadTransform(), extraParams), withContent, @@ -544,7 +549,7 @@ private DocumentPage search(SearchQueryDefinition querydef, long start, // the default for bulk is no metadata, which differs from the normal // default of ALL - Set metadata = isProcessedMetadataModified ? processedMetadata + Set metadata = isMetadataCategoriesModified ? metadataCategories : null; return services.getBulkDocuments(requestLogger, serverTimestamp, querydef, start, getPageLength(), transaction, searchHandle, searchView, metadata, @@ -943,7 +948,7 @@ protected TemporalDescriptor write(DocumentDescriptor desc, String temporalDocum requestLogger, desc, transaction, - (metadataHandle != null) ? processedMetadata : null, + (metadataHandle != null) ? metadataCategories : null, mergeTransformParameters((transform != null) ? transform : getWriteTransform(), extraParams), metadataHandle, contentHandle); } @@ -1266,7 +1271,7 @@ protected DocumentDescriptorImpl create(DocumentUriTemplate template, requestLogger, template, transaction, - (metadataHandle != null) ? processedMetadata : null, + (metadataHandle != null) ? metadataCategories : null, mergeTransformParameters((transform != null) ? transform : getWriteTransform(), extraParams), metadataHandle, contentHandle); } @@ -1326,7 +1331,7 @@ public void patch(DocumentDescriptor desc, DocumentPatchHandle patch, DocumentPatchHandleImpl builtPatch = (patch instanceof DocumentPatchHandleImpl) ? (DocumentPatchHandleImpl) patch : null; services.patchDocument(requestLogger, desc, transaction, - (builtPatch != null) ? builtPatch.getMetadata() : processedMetadata, + (builtPatch != null) ? builtPatch.getMetadata() : metadataCategories, (builtPatch != null) ? builtPatch.isOnContent() : true, patch); } @@ -1360,7 +1365,7 @@ public void patch(String uri, String temporalDocumentURI, String temporalCollec extraParams = addTemporalParams(extraParams, temporalCollection, temporalDocumentURI, null); DocumentPatchHandleImpl builtPatch = (patch instanceof DocumentPatchHandleImpl) ? (DocumentPatchHandleImpl) patch : null; - services.patchDocument(requestLogger, new DocumentDescriptorImpl(uri, true), transaction, (builtPatch != null) ? builtPatch.getMetadata() : processedMetadata, + services.patchDocument(requestLogger, new DocumentDescriptorImpl(uri, true), transaction, (builtPatch != null) ? builtPatch.getMetadata() : metadataCategories, (builtPatch != null) ? builtPatch.isOnContent() : true, extraParams, sourceDocumentURI, patch); } @@ -1416,7 +1421,7 @@ public void writeDefaultMetadata(String uri, Transaction transaction) logger.info("Resetting metadata for {}", uri); services.deleteDocument(requestLogger, - new DocumentDescriptorImpl(uri, true), transaction, processedMetadata, + new DocumentDescriptorImpl(uri, true), transaction, metadataCategories, getWriteParams()); } @@ -1433,7 +1438,7 @@ public void writeDefaultMetadata(Transaction transaction, String... uris) if (uris.length == 0) throw new IllegalArgumentException( "Resetting document metadata with empty identifier list"); - services.delete(requestLogger, transaction, processedMetadata, uris); + services.delete(requestLogger, transaction, metadataCategories, uris); } @Override diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java index 124c005da..ee6f42601 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/document/ReadJsonAsBinaryTest.java @@ -17,34 +17,37 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@Disabled("See MLE-27191 for description of the server bug") class ReadJsonAsBinaryTest extends AbstractClientTest { + private static final String URI = "/a.json"; + @Test - @Disabled("See MLE-27191 for description of the server bug") - void test() { + void genericDocumentManager() { try (DatabaseClient client = Common.newClient()) { - final String uri = "/a.json"; - writeJsonAsBinary(uri, client); + writeJsonAsBinary(URI, client); GenericDocumentManager docManager = client.newDocumentManager(); + // A bulk read works... + verifyBulkRead(docManager); + // But a request for a single URI does not! + verifySingleRead(docManager); + } + } - // A multipart request returns the correct document format. - try (DocumentPage page = docManager.read(uri)) { - DocumentRecord record = page.next(); - assertEquals(uri, record.getUri()); - assertEquals("application/json", record.getMimetype()); - assertEquals(Format.BINARY, record.getFormat()); - } + @Test + void managerWithModifiedMetadataCategories() { + try (DatabaseClient client = Common.newClient()) { + writeJsonAsBinary(URI, client); - // But a request for a single URI does not! - try (InputStreamHandle handle = docManager.read(uri, new InputStreamHandle())) { - assertEquals("application/json", handle.getMimetype()); - assertEquals(Format.BINARY, handle.getFormat(), "Unfortunately due to MLE-27191, the " + - "format is JSON instead of binary. So this assertion will fail until that bug is fixed. " + - "For a Java Client user, the workaround is to use the read method above and get a DocumentPage " + - "back, as a multipart response seems to identify the correct headers."); - } + GenericDocumentManager docManager = client.newDocumentManager(); + // With a default manager, metadata shouldn't be read by default, so this should work. + verifyBulkRead(docManager); + + // Setting metadata categories to anything will cause the bug. + docManager.setMetadataCategories(DocumentManager.Metadata.PERMISSIONS); + verifyBulkRead(docManager); } } @@ -56,4 +59,23 @@ private void writeJsonAsBinary(String uri, DatabaseClient client) { jsonDocManager.setWriteTransform(new ServerTransform("toBinary")); jsonDocManager.write(uri, metadata, new JacksonHandle(new ObjectMapper().createObjectNode().put("a", 1))); } + + private void verifySingleRead(GenericDocumentManager docManager) { + try (InputStreamHandle handle = docManager.read(URI, new InputStreamHandle())) { + assertEquals("application/json", handle.getMimetype()); + assertEquals(Format.BINARY, handle.getFormat(), "Unfortunately due to MLE-27191, the " + + "format is JSON instead of binary. So this assertion will fail until that bug is fixed. " + + "For a Java Client user, the workaround is to use the read method above and get a DocumentPage " + + "back, as a multipart response seems to identify the correct headers."); + } + } + + private void verifyBulkRead(GenericDocumentManager docManager) { + try (DocumentPage page = docManager.read(URI)) { + DocumentRecord record = page.next(); + assertEquals(URI, record.getUri()); + assertEquals("application/json", record.getMimetype()); + assertEquals(Format.BINARY, record.getFormat()); + } + } } diff --git a/test-app/build.gradle b/test-app/build.gradle index f8d21f472..a5a70c96e 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ buildscript { @@ -25,6 +25,11 @@ plugins { apply plugin: "com.marklogic.ml-gradle" +// Sometimes, 3 x 20 isn't enough on Jenkins. +mlWaitTillReady { + maxAttempts = 40 +} + dependencies { implementation "io.undertow:undertow-core:2.3.20.Final" implementation "io.undertow:undertow-servlet:2.3.20.Final" From fb5f639f858f07363afe3919ebe63ea5ba98d305 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 19 Feb 2026 09:43:37 -0500 Subject: [PATCH 55/70] MLE-27078 Fixes for bitemp tests Lot of small general improvements too. --- .../client/functionaltest/TestBiTemporal.java | 102 +++++++----------- 1 file changed, 40 insertions(+), 62 deletions(-) diff --git a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java index 6bb0539d5..1fdfe67d6 100644 --- a/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java +++ b/marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestBiTemporal.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.functionaltest; @@ -37,7 +37,7 @@ import static org.junit.jupiter.api.Assertions.*; -public class TestBiTemporal extends BasicJavaClientREST { +class TestBiTemporal extends BasicJavaClientREST { private static String dbName = "TestBiTemporalJava"; private static String[] fNames = { "TestBiTemporalJava-1" }; @@ -251,11 +251,10 @@ private void validateLSQTQueryData(DatabaseClient client) throws Exception { System.out.println("validStartDate = " + validStartDate); System.out.println("validEndDate = " + validEndDate); - assertTrue( - (validStartDate.equals("2001-01-01T00:00:00") && - validEndDate.equals("2011-12-31T23:59:59") && - systemStartDate.equals("2005-01-01T00:00:01-08:00") && - systemEndDate.equals("2010-01-01T00:00:01-08:00"))); + assertEquals("2001-01-01T00:00:00", validStartDate); + assertEquals("2011-12-31T23:59:59", validEndDate); + assertTrue(systemStartDate.startsWith("2005-01-01T00:00:01"), "Unexpected system start date: " + systemStartDate); + assertTrue(systemEndDate.startsWith("2010-01-01T00:00:01"), "Unexpected system end date: " + systemEndDate); } } @@ -591,8 +590,7 @@ public void deleteJSONSingleDocument(String temporalCollection, String docId, } private JacksonDatabindHandle getJSONDocumentHandle( - String startValidTime, String endValidTime, String address, String uri) - throws Exception { + String startValidTime, String endValidTime, String address, String uri) { // Setup for JSON document /** @@ -1421,11 +1419,9 @@ public void testJSONConsolidated() throws Exception { @Test public void testAdvancingLSQT() throws Exception { try { - System.out.println("Inside testAdvancingLSQT"); ConnectedRESTQA.disableAutomationOnTemporalCollection(dbName, temporalLsqtCollectionName, true); String docId = "javaSingleJSONDoc.json"; - String afterLSQTAdvance = null; Calendar firstInsertTime = DatatypeConverter.parseDateTime("2010-01-01T00:00:01"); JSONDocumentManager docMgr = writerClient.newJSONDocumentManager(); @@ -1455,65 +1451,59 @@ public void testAdvancingLSQT() throws Exception { long startLSQT = 1; JSONDocumentManager docMgrQy = adminClient.newJSONDocumentManager(); - String WithoutAdvaceSetExceptMsg = "Timestamp 2007-01-01T00:00:01-08:00 provided is greater than LSQT 1601-01-01T00:00:00Z"; String actualNoAdvanceMsg = null; - DocumentPage termQueryResultsLSQT = null; try { - termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT); - } - catch(Exception ex) { + docMgrQy.search(periodQueryLSQT, startLSQT); + } catch(Exception ex) { actualNoAdvanceMsg = ex.getMessage(); - System.out.println("Exception message for LSQT without advance set is " + actualNoAdvanceMsg); } - assertTrue(actualNoAdvanceMsg.contains(WithoutAdvaceSetExceptMsg)); + assertTrue(actualNoAdvanceMsg.contains("please provide a timestamp before LSQT"), + "Unexpected message: " + actualNoAdvanceMsg); // Set the Advance manually. docMgr.advanceLsqt(temporalLsqtCollectionName); - termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT); + DocumentPage termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT); - assertTrue(termQueryResultsLSQT.getTotalPages() == 0); - assertTrue(termQueryResultsLSQT.size() == 0); + assertEquals(0, termQueryResultsLSQT.getTotalPages()); + assertEquals(0, termQueryResultsLSQT.size()); // After Advance of the LSQT, query again with new query time greater than LSQT - - afterLSQTAdvance = desc.getTemporalSystemTime(); + String afterLSQTAdvance = desc.getTemporalSystemTime(); Calendar queryTimeLSQT2 = DatatypeConverter.parseDateTime(afterLSQTAdvance); queryTimeLSQT2.add(Calendar.YEAR, 10); docMgrQy = adminClient.newJSONDocumentManager(); docMgrQy.setMetadataCategories(Metadata.ALL); // Get all meta-data - StructuredQueryDefinition periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT2, 0, new String[] {}); + StructuredQueryDefinition periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT2, 0); - String excepMsgGrtr = "Timestamp 2020-01-01T00:00:01-08:00 provided is greater than LSQT 2010-01-01T08:00:01Z"; String actGrMsg = null; - DocumentPage termQueryResultsLSQT2 = null; try { - termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT); + docMgrQy.search(periodQueryLSQT2, startLSQT); } catch(Exception ex) { actGrMsg = ex.getMessage(); } - assertTrue(actGrMsg.contains(excepMsgGrtr)); + assertTrue(actGrMsg.contains("provided is greater than LSQT")); // Query again with query time less than LSQT. 10 minutes less than the LSQT Calendar lessTime = DatatypeConverter.parseDateTime("2009-01-01T00:00:01"); - periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, lessTime, 0, new String[] {}); - termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT); + periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, lessTime, 0); + DocumentPage termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT); System.out.println("LSQT Query results (Total Pages) after advance " + termQueryResultsLSQT2.getTotalPages()); System.out.println("LSQT Query results (Size) after advance " + termQueryResultsLSQT2.size()); - assertTrue(termQueryResultsLSQT2.getTotalPages() == 0); - assertTrue(termQueryResultsLSQT2.size() == 0); + assertEquals(0, termQueryResultsLSQT2.getTotalPages()); + assertEquals(0, termQueryResultsLSQT2.size()); // Query again with query time equal to LSQT. queryTimeLSQT2 = DatatypeConverter.parseDateTime(afterLSQTAdvance); - periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT2, 0, new String[] {}); + periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT2, 0); termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT); System.out.println("LSQT Query results (Total Pages) after advance " + termQueryResultsLSQT2.getTotalPages()); System.out.println("LSQT Query results (Size) after advance " + termQueryResultsLSQT2.size()); - assertTrue(termQueryResultsLSQT2.getTotalPages() == 1); - assertTrue(termQueryResultsLSQT2.size() == 1); + assertEquals(1, termQueryResultsLSQT2.getTotalPages()); + assertEquals(1, termQueryResultsLSQT2.size()); while (termQueryResultsLSQT2.hasNext()) { DocumentRecord record = termQueryResultsLSQT2.next(); @@ -1621,7 +1611,8 @@ public void testAdvancingLSQT() throws Exception { docMgr.advanceLsqt(temporalLsqtCollectionName); afterLSQTAdvance = desc.getTemporalSystemTime(); System.out.println("LSQT on collection after update and manual advance is " + afterLSQTAdvance); - assertTrue(desc.getTemporalSystemTime().trim().contains("2010-01-06T00:00:01-08:00")); + assertTrue(desc.getTemporalSystemTime().trim().contains("2010-01-06T00:00:01"), + "Unexpected time: " + desc.getTemporalSystemTime().trim()); // Verify that the document was updated // Make sure there are 1 documents in latest collection @@ -1678,13 +1669,10 @@ record = termQueryResults.next(); ObjectNode.class); recordContains.getContent(recordContainsHandle); String docContents = recordContainsHandle.toString(); - System.out.println("Content = " + docContents); - assertTrue(docContents.contains("\"javaValidStartERI\":\"2001-01-01T00:00:00\",\"javaValidEndERI\":\"2011-12-31T23:59:59\"")); + assertTrue(docContents.contains("\"javaValidStartERI\":\"2001-01-01T00:00:00\",\"javaValidEndERI\":\"2011-12-31T23:59:59\""), + "Unexpected docContents: " + docContents); } } - catch (Exception ex) { - System.out.println("Exception thrown from testAdvacingLSQT method " + ex.getMessage() ); - } finally { ConnectedRESTQA.updateTemporalCollectionForLSQT(dbName, temporalLsqtCollectionName, true); } @@ -3605,13 +3593,12 @@ public void testBulkWRTransactionCtsQueryBldr() throws Exception { Similar to testAdvancingLSQT - using CtsQueryBuilder. */ @Test - public void testAdvancingLSQTWithCtsQueryBuilder() throws Exception { + public void testAdvancingLSQTWithCtsQueryBuilder() { try { System.out.println("Inside testAdvancingLSQTWithCtsQueryBuilder"); ConnectedRESTQA.disableAutomationOnTemporalCollection(dbName, temporalLsqtCollectionName, true); String docId = "javaSingleJSONDoc.json"; - String afterLSQTAdvance = null; Calendar firstInsertTime = DatatypeConverter.parseDateTime("2010-01-01T00:00:01"); JSONDocumentManager docMgr = writerClient.newJSONDocumentManager(); @@ -3637,38 +3624,33 @@ public void testAdvancingLSQTWithCtsQueryBuilder() throws Exception { CtsQueryBuilder sqbLSQT = queryMgrLSQT.newCtsSearchBuilder(); XsStringVal collName = sqbLSQT.xs.string(temporalLsqtCollectionName); - XsStringSeqVal options = sqbLSQT.xs.stringSeq("", ""); Calendar calTimeLSQT = DatatypeConverter.parseDateTime("2007-01-01T00:00:01"); XsDateTimeVal queryTimeLSQT = sqbLSQT.xs.dateTime(calTimeLSQT); - XsDoubleVal weight = sqbLSQT.xs.doubleVal(0.0); CtsQueryExpr ctsQueryExpr = sqbLSQT.cts.lsqtQuery(collName, queryTimeLSQT); CtsQueryDefinition periodQueryLSQT = sqbLSQT.newCtsQueryDefinition(ctsQueryExpr); long startLSQT = 1; JSONDocumentManager docMgrQy = adminClient.newJSONDocumentManager(); - String WithoutAdvaceSetExceptMsg = "Timestamp 2007-01-01T00:00:01-08:00 provided is greater than LSQT 1601-01-01T00:00:00Z"; String actualNoAdvanceMsg = null; - DocumentPage termQueryResultsLSQT = null; try { - termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT); + docMgrQy.search(periodQueryLSQT, startLSQT); } catch (Exception ex) { actualNoAdvanceMsg = ex.getMessage(); - System.out.println("Exception message for LSQT without advance set is " + actualNoAdvanceMsg); } - assertTrue(actualNoAdvanceMsg.contains(WithoutAdvaceSetExceptMsg)); + assertTrue(actualNoAdvanceMsg.contains("please provide a timestamp before LSQT"), + "Unexpected message: " + actualNoAdvanceMsg); // Set the Advance manually. docMgr.advanceLsqt(temporalLsqtCollectionName); - termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT); + DocumentPage termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT); assertTrue(termQueryResultsLSQT.getTotalPages() == 0); assertTrue(termQueryResultsLSQT.size() == 0); // After Advance of the LSQT, query again with new query time greater than LSQT - - afterLSQTAdvance = desc.getTemporalSystemTime(); + String afterLSQTAdvance = desc.getTemporalSystemTime(); Calendar calTimeLSQT2 = DatatypeConverter.parseDateTime(afterLSQTAdvance); calTimeLSQT2.add(Calendar.YEAR, 10); XsDateTimeVal queryTimeLSQT2 = sqbLSQT.xs.dateTime(calTimeLSQT2); @@ -3678,15 +3660,13 @@ public void testAdvancingLSQTWithCtsQueryBuilder() throws Exception { CtsQueryExpr ctsQueryExpr2 = sqbLSQT.cts.lsqtQuery(collName, queryTimeLSQT2); CtsQueryDefinition periodQueryLSQT2 = sqbLSQT.newCtsQueryDefinition(ctsQueryExpr2); - String excepMsgGrtr = "Timestamp 2020-01-01T00:00:01-08:00 provided is greater than LSQT 2010-01-01T08:00:01Z"; String actGrMsg = null; - DocumentPage termQueryResultsLSQT2 = null; try { - termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT); + docMgrQy.search(periodQueryLSQT2, startLSQT); } catch (Exception ex) { actGrMsg = ex.getMessage(); } - assertTrue(actGrMsg.contains(excepMsgGrtr)); + assertTrue(actGrMsg.contains("provided is greater than LSQT"), "Unexpected message: " + actGrMsg); // Query again with query time less than LSQT. 10 minutes less than the LSQT Calendar callessTime = DatatypeConverter.parseDateTime("2009-01-01T00:00:01"); @@ -3694,14 +3674,12 @@ public void testAdvancingLSQTWithCtsQueryBuilder() throws Exception { CtsQueryExpr ctsQueryExprLess = sqbLSQT.cts.lsqtQuery(collName, lessTime/*, options, weight*/); CtsQueryDefinition lessDef = sqbLSQT.newCtsQueryDefinition(ctsQueryExprLess); - termQueryResultsLSQT2 = docMgrQy.search(lessDef, startLSQT); + DocumentPage termQueryResultsLSQT2 = docMgrQy.search(lessDef, startLSQT); System.out.println("LSQT Query results (Total Pages) after advance " + termQueryResultsLSQT2.getTotalPages()); System.out.println("LSQT Query results (Size) after advance " + termQueryResultsLSQT2.size()); - assertTrue(termQueryResultsLSQT2.getTotalPages() == 0); - assertTrue(termQueryResultsLSQT2.size() == 0); - } catch (Exception ex) { - System.out.println("Exception thrown from testAdvacingLSQT method " + ex.getMessage()); + assertEquals(0, termQueryResultsLSQT2.getTotalPages()); + assertEquals(0, termQueryResultsLSQT2.size()); } finally { ConnectedRESTQA.updateTemporalCollectionForLSQT(dbName, temporalLsqtCollectionName, true); } From 2e84d2d1bf0ca9be46cbd8752be7867203702ceb Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 19 Feb 2026 13:03:50 -0500 Subject: [PATCH 56/70] MLE-27078 Incremental write now uses an unsignedLong Not sure eval will be kept, but all 3 approaches are now properly using an unsignedLong. Also fixed a performance issue in the view filter where "op.in" was being used instead of a "where" + documentQuery. --- Jenkinsfile | 4 +- .../filter/IncrementalWriteEvalFilter.java | 9 +++- .../filter/IncrementalWriteFilter.java | 15 +++--- .../filter/IncrementalWriteOpticFilter.java | 6 +-- .../filter/IncrementalWriteViewFilter.java | 25 ++++++---- .../filter/IncrementalWriteFilterTest.java | 6 +-- .../filter/IncrementalWriteTest.java | 48 ++++++++++++++++++- .../ml-config/databases/content-database.json | 6 +-- .../ml-schemas/tde/incrementalWriteHash.json | 4 +- 9 files changed, 87 insertions(+), 36 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3243eb7b8..19cf02d3b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -150,8 +150,8 @@ pipeline { } } steps { - setupDockerMarkLogic("progressofficial/marklogic-db:latest") - + setupDockerMarkLogic("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") + sh label: 'run marklogic-client-api tests', script: '''#!/bin/bash export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java index 838087203..df2dd2c42 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -19,11 +19,16 @@ */ class IncrementalWriteEvalFilter extends IncrementalWriteFilter { + // The hash value is cast to a String based on this analysis from Copilot: + // "The hash field index is xs:unsignedLong, which JavaScript represents as an + // IEEE 754 double. To avoid loss of precision for large integers (e.g., above + // 2^53−1), the value is converted to a String in JavaScript and then parsed + // back to an unsigned long when it is read from the JSON response." private static final String EVAL_SCRIPT = """ const tuples = cts.valueTuples([cts.uriReference(), cts.fieldReference(hashKeyName)], null, cts.documentQuery(uris)); const response = {}; for (var tuple of tuples) { - response[tuple[0]] = tuple[1]; + response[tuple[0]] = String(tuple[1]); } response """; @@ -49,7 +54,7 @@ public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { return filterDocuments(context, uri -> { if (response.has(uri)) { - return response.get(uri).asText(); + return Long.parseUnsignedLong(response.get(uri).asText()); } return null; }); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 61df914a9..bc2fb86b0 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -227,7 +227,7 @@ public IncrementalWriteConfig getConfig() { return config; } - protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { + protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { final DocumentWriteSet newWriteSet = context.getDatabaseClient().newDocumentManager().newWriteSet(); final List skippedDocuments = new ArrayList<>(); final String timestamp = Instant.now().toString(); @@ -246,8 +246,8 @@ protected final DocumentWriteSet filterDocuments(Context context, Function existingHashes = rowTemplate.query(op -> + Map existingHashes = rowTemplate.query(op -> op.fromLexicons(Map.of( "uri", op.cts.uriReference(), "hash", op.cts.fieldReference(getConfig().getHashKeyName()) @@ -43,10 +43,10 @@ public DocumentWriteSet apply(Context context) { ), rows -> { - Map map = new HashMap<>(); + Map map = new HashMap<>(); rows.forEach(row -> { String uri = row.getString("uri"); - String existingHash = row.getString("hash"); + long existingHash = Long.parseUnsignedLong(row.getString("hash")); map.put(uri, existingHash); }); return map; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java index 994512f54..72c376127 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java @@ -13,7 +13,10 @@ /** * Uses an Optic query with fromView to get the existing hash values for a set of URIs from a TDE view. - * This implementation requires a TDE template to be deployed that extracts the URI and hash metadata. + * This implementation requires a TDE template to be deployed that contains at minimum a "uri" column + * and a column matching the configured hash key name, plus any other columns desired. + * The query uses a {@code where} with a {@code cts.documentQuery} to filter rows by URI, which is + * significantly faster than filtering via {@code op.in}. * * @since 8.1.0 */ @@ -33,20 +36,22 @@ public DocumentWriteSet apply(Context context) { RowTemplate rowTemplate = new RowTemplate(context.getDatabaseClient()); try { - Map existingHashes = rowTemplate.query(op -> - op.fromView(getConfig().getSchemaName(), getConfig().getViewName()) - .where(op.in(op.col("uri"), op.xs.stringSeq(uris))), - + Map existingHashes = rowTemplate.query(op -> + op.fromView(getConfig().getSchemaName(), getConfig().getViewName(), "") + .where(op.cts.documentQuery(op.xs.stringSeq(uris))) + , rows -> { - Map map = new HashMap<>(); + Map map = new HashMap<>(); rows.forEach(row -> { String uri = row.getString("uri"); - String existingHash = row.getString("hash"); - map.put(uri, existingHash); + String hashString = row.getString(getConfig().getHashKeyName()); + if (hashString != null && !hashString.isEmpty()) { + long existingHash = Long.parseUnsignedLong(hashString); + map.put(uri, existingHash); + } }); return map; - } - ); + }); if (logger.isDebugEnabled()) { logger.debug("Retrieved {} existing hashes for batch of size {}", existingHashes.size(), uris.length); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java index 8c0cded96..bdaab89b4 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.filter; @@ -35,7 +35,7 @@ void addHashToMetadata() { DocumentWriteOperation doc2 = new DocumentWriteOperationImpl("/2.xml", metadata, new StringHandle("")); final String timestamp = Instant.now().toString(); - doc2 = IncrementalWriteFilter.addHashToMetadata(doc2, "theField", "abc123", "theTimestamp", timestamp); + doc2 = IncrementalWriteFilter.addHashToMetadata(doc2, "theField", 12345, "theTimestamp", timestamp); assertEquals(metadata, doc1.getMetadata(), "doc1 should still have the original metadata object"); @@ -46,7 +46,7 @@ void addHashToMetadata() { assertEquals("value1", metadata2.getProperties().get("prop1"), "property should be preserved"); assertEquals("value1", metadata2.getMetadataValues().get("meta1"), "metadata value should be preserved"); - assertEquals("abc123", metadata2.getMetadataValues().get("theField"), "hash field should be added"); + assertEquals("12345", metadata2.getMetadataValues().get("theField"), "hash field should be added"); assertEquals(timestamp, metadata2.getMetadataValues().get("theTimestamp"), "timestamp should be added"); } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index 90258ec78..bebfa18f3 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -251,6 +251,50 @@ void fromView() { verifyIncrementalWriteWorks(); } + @Test + void loadWithEvalFilterThenVerifyOpticFilterSkipsAll() { + filter = IncrementalWriteFilter.newBuilder() + .useEvalQuery(true) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + writeTenDocuments(); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Switch to the Optic fromLexicons filter. + writtenCount.set(0); + skippedCount.set(0); + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + writeTenDocuments(); + assertEquals(0, writtenCount.get(), "No documents should be written since the hashes stored by the eval " + + "filter should be recognized as unchanged by the Optic filter."); + assertEquals(10, skippedCount.get()); + } + + @Test + void loadWithOpticFilterThenVerifyEvalFilterSkipsAll() { + writeTenDocuments(); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Switch to the eval filter. + writtenCount.set(0); + skippedCount.set(0); + filter = IncrementalWriteFilter.newBuilder() + .useEvalQuery(true) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + writeTenDocuments(); + assertEquals(0, writtenCount.get(), "No documents should be written since the hashes stored by the Optic " + + "filter should be recognized as unchanged by the eval filter."); + assertEquals(10, skippedCount.get()); + } + @Test void emptyValuesForFromView() { filter = IncrementalWriteFilter.newBuilder() @@ -332,8 +376,8 @@ private void verifyDocumentsHasHashInMetadataKey() { String hash = metadata.getMetadataValues().get("incrementalWriteHash"); try { - // Can use Java's support for parsing unsigned longs in base 16 to verify the hash is valid. - Long.parseUnsignedLong(hash, 16); + // Verify the hash is a valid unsigned long in base 10 decimal. + Long.parseUnsignedLong(hash); } catch (NumberFormatException e) { fail("Document " + doc.getUri() + " has an invalid incrementalWriteHash value: " + hash); } diff --git a/test-app/src/main/ml-config/databases/content-database.json b/test-app/src/main/ml-config/databases/content-database.json index 53803022d..965bb45d3 100644 --- a/test-app/src/main/ml-config/databases/content-database.json +++ b/test-app/src/main/ml-config/databases/content-database.json @@ -224,15 +224,13 @@ "invalid-values": "reject" }, { - "scalar-type": "string", - "collation": "http://marklogic.com/collation/", + "scalar-type": "unsignedLong", "field-name": "incrementalWriteHash", "range-value-positions": false, "invalid-values": "reject" }, { - "scalar-type": "string", - "collation": "http://marklogic.com/collation/", + "scalar-type": "unsignedLong", "field-name": "myWriteHash", "range-value-positions": false, "invalid-values": "reject" diff --git a/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json b/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json index d5044414c..d3d950b15 100644 --- a/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json +++ b/test-app/src/main/ml-schemas/tde/incrementalWriteHash.json @@ -13,8 +13,8 @@ "val": "xdmp:node-uri(.)" }, { - "name": "hash", - "scalarType": "string", + "name": "incrementalWriteHash", + "scalarType": "unsignedLong", "val": "xdmp:node-metadata-value(., 'incrementalWriteHash')", "nullable": true } From 1d19be9f9b752c19d793289ee4aeb5d485c623d7 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 23 Feb 2026 13:20:52 -0500 Subject: [PATCH 57/70] MLE-26918 Dropping eval implementation for incremental write Also renamed the other two classes to make it clear what their distinction is - fromLexicons and fromView --- .../datamovement/filter/FilterException.java | 2 + .../filter/IncrementalWriteConfig.java | 47 ++----------- .../filter/IncrementalWriteEvalFilter.java | 66 ------------------- .../filter/IncrementalWriteFilter.java | 42 +++++------- ...> IncrementalWriteFromLexiconsFilter.java} | 20 ++---- ...va => IncrementalWriteFromViewFilter.java} | 22 ++----- .../filter/IncrementalWriteTest.java | 36 ++-------- 7 files changed, 45 insertions(+), 190 deletions(-) delete mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java rename marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/{IncrementalWriteOpticFilter.java => IncrementalWriteFromLexiconsFilter.java} (62%) rename marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/{IncrementalWriteViewFilter.java => IncrementalWriteFromViewFilter.java} (61%) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java index 712f52cdf..f10b4814e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/FilterException.java @@ -9,6 +9,8 @@ * Any exception thrown by execution of a {@code DocumentWriteSetFilter} will be wrapped in this exception and * rethrown by the {@code WriteBatcher}, allowing failure listeners to distinguish filter exceptions from other * exceptions that may occur during batch processing. + * + * @since 8.1.0 */ public class FilterException extends DataMovementException { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java index d350f71f1..358cc0878 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java @@ -14,17 +14,10 @@ * * @since 8.1.0 */ -public class IncrementalWriteConfig { - - private final String hashKeyName; - private final String timestampKeyName; - private final boolean canonicalizeJson; - private final Consumer skippedDocumentsConsumer; - private final String[] jsonExclusions; - private final String[] xmlExclusions; - private final Map xmlNamespaces; - private final String schemaName; - private final String viewName; +public record IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, + Consumer skippedDocumentsConsumer, + String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces, + String schemaName, String viewName) { public IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, @@ -41,39 +34,11 @@ public IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boole this.viewName = viewName; } - public String getHashKeyName() { - return hashKeyName; - } - - public String getTimestampKeyName() { - return timestampKeyName; - } - - public boolean isCanonicalizeJson() { - return canonicalizeJson; - } - - public Consumer getSkippedDocumentsConsumer() { - return skippedDocumentsConsumer; - } - - public String[] getJsonExclusions() { - return jsonExclusions; - } - public String[] getXmlExclusions() { - return xmlExclusions; - } - - public Map getXmlNamespaces() { + @Override + public Map xmlNamespaces() { return xmlNamespaces != null ? xmlNamespaces : Collections.emptyMap(); } - public String getSchemaName() { - return schemaName; - } - public String getViewName() { - return viewName; - } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java deleted file mode 100644 index df2dd2c42..000000000 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. - */ -package com.marklogic.client.datamovement.filter; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.marklogic.client.FailedRequestException; -import com.marklogic.client.datamovement.DocumentWriteSetFilter; -import com.marklogic.client.document.DocumentWriteOperation; -import com.marklogic.client.document.DocumentWriteSet; -import com.marklogic.client.io.JacksonHandle; - -/** - * Uses server-side JavaScript code to get the existing hash values for a set of URIs. - * - * @since 8.1.0 - */ -class IncrementalWriteEvalFilter extends IncrementalWriteFilter { - - // The hash value is cast to a String based on this analysis from Copilot: - // "The hash field index is xs:unsignedLong, which JavaScript represents as an - // IEEE 754 double. To avoid loss of precision for large integers (e.g., above - // 2^53−1), the value is converted to a String in JavaScript and then parsed - // back to an unsigned long when it is read from the JSON response." - private static final String EVAL_SCRIPT = """ - const tuples = cts.valueTuples([cts.uriReference(), cts.fieldReference(hashKeyName)], null, cts.documentQuery(uris)); - const response = {}; - for (var tuple of tuples) { - response[tuple[0]] = String(tuple[1]); - } - response - """; - - IncrementalWriteEvalFilter(IncrementalWriteConfig config) { - super(config); - } - - @Override - public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { - ArrayNode uris = new ObjectMapper().createArrayNode(); - for (DocumentWriteOperation doc : context.getDocumentWriteSet()) { - if (DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(doc.getOperationType())) { - uris.add(doc.getUri()); - } - } - - try { - JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) - .addVariable("hashKeyName", getConfig().getHashKeyName()) - .addVariable("uris", new JacksonHandle(uris)) - .evalAs(JsonNode.class); - - return filterDocuments(context, uri -> { - if (response.has(uri)) { - return Long.parseUnsignedLong(response.get(uri).asText()); - } - return null; - }); - } catch (FailedRequestException e) { - String message = "Unable to query for existing incremental write hashes; cause: " + e.getMessage(); - throw new FailedRequestException(message, e.getFailedRequest()); - } - } -} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index bc2fb86b0..50021871f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -48,7 +48,6 @@ public static class Builder { private String hashKeyName = "incrementalWriteHash"; private String timestampKeyName = "incrementalWriteTimestamp"; private boolean canonicalizeJson = true; - private boolean useEvalQuery = false; private Consumer skippedDocumentsConsumer; private String[] jsonExclusions; private String[] xmlExclusions; @@ -87,14 +86,6 @@ public Builder canonicalizeJson(boolean canonicalizeJson) { return this; } - /** - * @param useEvalQuery if true, evaluate server-side JavaScript instead of an Optic query for retrieving hash values; defaults to false. - */ - public Builder useEvalQuery(boolean useEvalQuery) { - this.useEvalQuery = useEvalQuery; - return this; - } - /** * @param skippedDocumentsConsumer a consumer that will be called with any documents in a batch that were skipped because their content had not changed. */ @@ -161,12 +152,9 @@ public IncrementalWriteFilter build() { skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces, schemaName, viewName); if (schemaName != null && viewName != null) { - return new IncrementalWriteViewFilter(config); - } - if (useEvalQuery) { - return new IncrementalWriteEvalFilter(config); + return new IncrementalWriteFromViewFilter(config); } - return new IncrementalWriteOpticFilter(config); + return new IncrementalWriteFromLexiconsFilter(config); } private void validateJsonExclusions() { @@ -254,19 +242,19 @@ protected final DocumentWriteSet filterDocuments(Context context, Function 0) { - content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, config.getJsonExclusions()); + if (config.jsonExclusions() != null && config.jsonExclusions().length > 0) { + content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, config.jsonExclusions()); } jc = new JsonCanonicalizer(content); return jc.getEncodedString(); @@ -298,9 +286,9 @@ private String serializeContent(DocumentWriteOperation doc) { logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); } - } else if (config.getXmlExclusions() != null && config.getXmlExclusions().length > 0) { + } else if (config.xmlExclusions() != null && config.xmlExclusions().length > 0) { try { - content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, config.getXmlNamespaces(), config.getXmlExclusions()); + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, config.xmlNamespaces(), config.xmlExclusions()); } catch (Exception e) { logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); @@ -341,4 +329,10 @@ protected static DocumentWriteOperation addHashToMetadata(DocumentWriteOperation } + protected static String[] getUrisInBatch(DocumentWriteSet writeSet) { + return writeSet.stream() + .filter(op -> DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(op.getOperationType())) + .map(DocumentWriteOperation::getUri) + .toArray(String[]::new); + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java similarity index 62% rename from marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java rename to marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java index 152017b61..55b7b9872 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java @@ -12,32 +12,26 @@ import java.util.Map; /** - * Uses an Optic query to get the existing hash values for a set of URIs. + * Uses an Optic fromLexicons query that depends on a field range index to retrieve URIs and + * hash values. * * @since 8.1.0 */ -class IncrementalWriteOpticFilter extends IncrementalWriteFilter { +class IncrementalWriteFromLexiconsFilter extends IncrementalWriteFilter { - IncrementalWriteOpticFilter(IncrementalWriteConfig config) { + IncrementalWriteFromLexiconsFilter(IncrementalWriteConfig config) { super(config); } @Override public DocumentWriteSet apply(Context context) { - final String[] uris = context.getDocumentWriteSet().stream() - .filter(op -> DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(op.getOperationType())) - .map(DocumentWriteOperation::getUri) - .toArray(String[]::new); - - // It doesn't seem possible yet to use a DSL query and bind an array of strings to a "uris" param, so using - // a serialized query instead. That doesn't allow a user to override the query though. - RowTemplate rowTemplate = new RowTemplate(context.getDatabaseClient()); + final String[] uris = getUrisInBatch(context.getDocumentWriteSet()); try { - Map existingHashes = rowTemplate.query(op -> + Map existingHashes = new RowTemplate(context.getDatabaseClient()).query(op -> op.fromLexicons(Map.of( "uri", op.cts.uriReference(), - "hash", op.cts.fieldReference(getConfig().getHashKeyName()) + "hash", op.cts.fieldReference(getConfig().hashKeyName()) )).where( op.cts.documentQuery(op.xs.stringSeq(uris)) ), diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java similarity index 61% rename from marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java rename to marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java index 72c376127..7d4e7e465 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteViewFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java @@ -4,7 +4,6 @@ package com.marklogic.client.datamovement.filter; import com.marklogic.client.FailedRequestException; -import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.row.RowTemplate; @@ -15,36 +14,29 @@ * Uses an Optic query with fromView to get the existing hash values for a set of URIs from a TDE view. * This implementation requires a TDE template to be deployed that contains at minimum a "uri" column * and a column matching the configured hash key name, plus any other columns desired. - * The query uses a {@code where} with a {@code cts.documentQuery} to filter rows by URI, which is - * significantly faster than filtering via {@code op.in}. * * @since 8.1.0 */ -class IncrementalWriteViewFilter extends IncrementalWriteFilter { +class IncrementalWriteFromViewFilter extends IncrementalWriteFilter { - IncrementalWriteViewFilter(IncrementalWriteConfig config) { + IncrementalWriteFromViewFilter(IncrementalWriteConfig config) { super(config); } @Override public DocumentWriteSet apply(Context context) { - final String[] uris = context.getDocumentWriteSet().stream() - .filter(op -> DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(op.getOperationType())) - .map(DocumentWriteOperation::getUri) - .toArray(String[]::new); - - RowTemplate rowTemplate = new RowTemplate(context.getDatabaseClient()); + final String[] uris = getUrisInBatch(context.getDocumentWriteSet()); try { - Map existingHashes = rowTemplate.query(op -> - op.fromView(getConfig().getSchemaName(), getConfig().getViewName(), "") + Map existingHashes = new RowTemplate(context.getDatabaseClient()).query(op -> + op.fromView(getConfig().schemaName(), getConfig().viewName(), "") .where(op.cts.documentQuery(op.xs.stringSeq(uris))) , rows -> { Map map = new HashMap<>(); rows.forEach(row -> { String uri = row.getString("uri"); - String hashString = row.getString(getConfig().getHashKeyName()); + String hashString = row.getString(getConfig().hashKeyName()); if (hashString != null && !hashString.isEmpty()) { long existingHash = Long.parseUnsignedLong(hashString); map.put(uri, existingHash); @@ -59,7 +51,7 @@ public DocumentWriteSet apply(Context context) { return filterDocuments(context, uri -> existingHashes.get(uri)); } catch (FailedRequestException e) { - String message = "Unable to query for existing incremental write hashes from view " + getConfig().getSchemaName() + "." + getConfig().getViewName() + "; cause: " + e.getMessage(); + String message = "Unable to query for existing incremental write hashes from view " + getConfig().schemaName() + "." + getConfig().viewName() + "; cause: " + e.getMessage(); throw new FailedRequestException(message, e.getFailedRequest()); } } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index bebfa18f3..e00fd9915 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -23,16 +23,6 @@ void opticFilter() { verifyIncrementalWriteWorks(); } - @Test - void evalFilter() { - filter = IncrementalWriteFilter.newBuilder() - .useEvalQuery(true) - .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) - .build(); - - verifyIncrementalWriteWorks(); - } - @Test void jsonKeysOutOfOrder() { for (int i = 1; i <= 10; i++) { @@ -147,22 +137,6 @@ void noRangeIndexForField() { "fail with a helpful error message. Actual message: " + causeMessage); } - @Test - void noRangeIndexForFieldWithEval() { - filter = IncrementalWriteFilter.newBuilder() - .hashKeyName("non-existent-field") - .useEvalQuery(true) - .build(); - - writeTenDocuments(); - - assertNotNull(batchFailure.get()); - String message = batchFailure.get().getMessage(); - assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("XDMP-FIELDRIDXNOTFOUND"), - "When the user tries to use the incremental write feature without the required range index, we should " + - "fail with a helpful error message. Actual message: " + message); - } - @Test void customTimestampKeyName() { filter = IncrementalWriteFilter.newBuilder() @@ -252,9 +226,9 @@ void fromView() { } @Test - void loadWithEvalFilterThenVerifyOpticFilterSkipsAll() { + void loadWithViewFilterThenVerifyLexiconFilterSkipsAll() { filter = IncrementalWriteFilter.newBuilder() - .useEvalQuery(true) + .fromView("javaClient", "incrementalWriteHash") .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) .build(); @@ -276,16 +250,16 @@ void loadWithEvalFilterThenVerifyOpticFilterSkipsAll() { } @Test - void loadWithOpticFilterThenVerifyEvalFilterSkipsAll() { + void loadWithLexiconsFilterThenVerifyViewFilterSkipsAll() { writeTenDocuments(); assertEquals(10, writtenCount.get()); assertEquals(0, skippedCount.get()); - // Switch to the eval filter. + // Switch to the fromView filter. writtenCount.set(0); skippedCount.set(0); filter = IncrementalWriteFilter.newBuilder() - .useEvalQuery(true) + .fromView("javaClient", "incrementalWriteHash") .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) .build(); From 5f25dcc23862672963090c8796d52107222d074a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 23 Feb 2026 14:47:27 -0500 Subject: [PATCH 58/70] MLE-26918 Refactor: Changed config class back to a non-record class Had a discussion with Copilot, agreed that a record isn't a fit here as it implies immutability but that's not currently the case. --- .../filter/IncrementalWriteConfig.java | 47 ++++++++++++++++--- .../filter/IncrementalWriteFilter.java | 20 ++++---- .../IncrementalWriteFromLexiconsFilter.java | 2 +- .../IncrementalWriteFromViewFilter.java | 7 +-- 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java index 358cc0878..d350f71f1 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteConfig.java @@ -14,10 +14,17 @@ * * @since 8.1.0 */ -public record IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, - Consumer skippedDocumentsConsumer, - String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces, - String schemaName, String viewName) { +public class IncrementalWriteConfig { + + private final String hashKeyName; + private final String timestampKeyName; + private final boolean canonicalizeJson; + private final Consumer skippedDocumentsConsumer; + private final String[] jsonExclusions; + private final String[] xmlExclusions; + private final Map xmlNamespaces; + private final String schemaName; + private final String viewName; public IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, @@ -34,11 +41,39 @@ public IncrementalWriteConfig(String hashKeyName, String timestampKeyName, boole this.viewName = viewName; } + public String getHashKeyName() { + return hashKeyName; + } + + public String getTimestampKeyName() { + return timestampKeyName; + } + + public boolean isCanonicalizeJson() { + return canonicalizeJson; + } + + public Consumer getSkippedDocumentsConsumer() { + return skippedDocumentsConsumer; + } + + public String[] getJsonExclusions() { + return jsonExclusions; + } - @Override - public Map xmlNamespaces() { + public String[] getXmlExclusions() { + return xmlExclusions; + } + + public Map getXmlNamespaces() { return xmlNamespaces != null ? xmlNamespaces : Collections.emptyMap(); } + public String getSchemaName() { + return schemaName; + } + public String getViewName() { + return viewName; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 50021871f..253adb51c 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -242,19 +242,19 @@ protected final DocumentWriteSet filterDocuments(Context context, Function 0) { - content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, config.jsonExclusions()); + if (config.getJsonExclusions() != null && config.getJsonExclusions().length > 0) { + content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, config.getJsonExclusions()); } jc = new JsonCanonicalizer(content); return jc.getEncodedString(); @@ -286,9 +286,9 @@ private String serializeContent(DocumentWriteOperation doc) { logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); } - } else if (config.xmlExclusions() != null && config.xmlExclusions().length > 0) { + } else if (config.getXmlExclusions() != null && config.getXmlExclusions().length > 0) { try { - content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, config.xmlNamespaces(), config.xmlExclusions()); + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, config.getXmlNamespaces(), config.getXmlExclusions()); } catch (Exception e) { logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", doc.getUri(), e.getMessage()); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java index 55b7b9872..7eb23d1a0 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromLexiconsFilter.java @@ -31,7 +31,7 @@ public DocumentWriteSet apply(Context context) { Map existingHashes = new RowTemplate(context.getDatabaseClient()).query(op -> op.fromLexicons(Map.of( "uri", op.cts.uriReference(), - "hash", op.cts.fieldReference(getConfig().hashKeyName()) + "hash", op.cts.fieldReference(getConfig().getHashKeyName()) )).where( op.cts.documentQuery(op.xs.stringSeq(uris)) ), diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java index 7d4e7e465..754944b5b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFromViewFilter.java @@ -29,14 +29,14 @@ public DocumentWriteSet apply(Context context) { try { Map existingHashes = new RowTemplate(context.getDatabaseClient()).query(op -> - op.fromView(getConfig().schemaName(), getConfig().viewName(), "") + op.fromView(getConfig().getSchemaName(), getConfig().getViewName(), "") .where(op.cts.documentQuery(op.xs.stringSeq(uris))) , rows -> { Map map = new HashMap<>(); rows.forEach(row -> { String uri = row.getString("uri"); - String hashString = row.getString(getConfig().hashKeyName()); + String hashString = row.getString(getConfig().getHashKeyName()); if (hashString != null && !hashString.isEmpty()) { long existingHash = Long.parseUnsignedLong(hashString); map.put(uri, existingHash); @@ -51,7 +51,8 @@ public DocumentWriteSet apply(Context context) { return filterDocuments(context, uri -> existingHashes.get(uri)); } catch (FailedRequestException e) { - String message = "Unable to query for existing incremental write hashes from view " + getConfig().schemaName() + "." + getConfig().viewName() + "; cause: " + e.getMessage(); + String message = "Unable to query for existing incremental write hashes from view " + + getConfig().getSchemaName() + "." + getConfig().getViewName() + "; cause: " + e.getMessage(); throw new FailedRequestException(message, e.getFailedRequest()); } } From 7f87ad83f955a8890dd47fc41e5bd000e26f60fb Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 24 Feb 2026 10:11:26 -0500 Subject: [PATCH 59/70] MLE-26918 Fixing binary test on ARM, and bumping Gradle to 9.3.1 The Gradle bump should "just work". Also hoping this triggers publication of the library internally, which doesn't seem to have happened for the last PR. --- .copyrightconfig | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 45457 -> 46175 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- .../client/test/BinaryDocumentTest.java | 181 +++++++++--------- 4 files changed, 93 insertions(+), 92 deletions(-) diff --git a/.copyrightconfig b/.copyrightconfig index 024c7ca7f..f26370e80 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS, *.txt, *.xqy +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, *.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.json, *.xml, CODEOWNERS, *.txt, *.xqy diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 8bdaf60c75ab801e22807dde59e12a8735a34077..61285a659d17295f1de7c53e24fdf13ad755c379 100644 GIT binary patch delta 36855 zcmXVWV|bkZ_jDR#W81c!G`4NqPNQwKF*ml&#gVUume%9b>_7z=BQY- z8${v$0On2okcB}A24;WZJvgKf2sMc4OW5no*K!=QkJ2UC_?9&TcjuMeJ*%&gwJOJ^ zBOmlRj!F(IlPc*L>x7BjWPSq0!t44;Sx(hDrP`K(m#6@kk3L15y8lPUffe(orgSCj zlG71p_(RTjUQnJdW+4C+PNUg*y5M3C5PE6_V7Vp8!1wW->mwAij4$W-rwY;c<}8<8 z6)8pacYaCB((&sk8alX_sFQJy+<2&aj`Vm_bK|l%C31^phDVTF5x?rKn(r3qzmg4L5XD9sAcpJWv^~@--?e#b~a}GQzalb39YEk9z z)BGZ7JL%7@fcb$ny7*fS8;<_d!+aeg8tOTqtpk-c0Ec&Q1COv-iDAdi?Y^r49&N9X zo*e^DyTz7dXN8NpuUaRWhep4MNe)|W_jj$mAEBHyj;b?jqtq){0PI939MsIK3`! zFihdKVb2?J)7a;VrBkydVeqZ2YRw&WB6zc{rMB2<40y4WBLz*pIR zCdaU7k85@e2%+tm$Cx@@w*gS4e~sYbEXY+HmWL)Rvw5Z@lLO!rzzdaKB~~jD*hM$E zhy^kLkFZibj7Mz{X&KL8Or}2}ZKjixR!lJ@$UJ$Z6>?kOO#&&89dN?Ch3(pXODZA^ zB#*l1lcx&qQ1wqa$Pv9W3t}kW*M5X?+ube!4LrPK3aF%jbCnzY!?{kOi1I07SRZH_ zkMeep`V{8&HqT%cIIh&2;#msNxp9#_eqVHQut@rT(3fb)-J~;_njzC&ks35D@>El%6Jlf!K~fXt~C69L#$Y5s9tkQVovk)hvpb z7zLPdriviW?VcMC_l}KgliJZq^auVo3G6g!Y~WY%X@Ou$3Lb}EC0|>+0y|q@-yg4q zyS{*JQsV$dG=1^$Q-jq zIY}4Zt;i@M5aA;Xqlre0KMhYj7fqcOVz>rS48I7bVmUSi zFSKkcoXcM>aukdb9D2l?hf&@tfyrpBd0T>8fPsGkbu%YefO% zhxxLcTlo?2280lv!sFIK;H4CMlW@%RR9Eo1kT3ppSLdc&;jX72BG~Z9D=O>^-w3!` zCR)^>e-0nQIBE}eg=%*U9FDbzO3j)GOYG^CgK3j!jJGH;8MR$$M0$zc5D8TvVoKN( zqE4`lZ?#zVp|PJ^bj9NYq$nTPG+SAhW%N^i;NG~U{!tQDkF_S|!TG)Oqyq6==#WRU zq@fS7tjH0T47hN)CD0r2_Ox{%rOiG+9spg5YBpr@rq^N}A^K(XTRqG%%F*8;UU;O| zVTT|#5B$fmPj_MrM$k}D?XX}>A`^8bCV(PZ49Pr%i zWe-XX^QYBJXRtR|ueTccRlrb<^KG@y4A(gpC=epwghdrdKr22ZGUi=cqBd6LB~z6H zzU!FB#AJt8892mo)7fS`ccPs3U3v{l^}3 z;PTHehwapHCIx7vh8;kz6BURi;<33FF3uN>`^SP{;C7qw6uPF7NPPSRXjO5vfFzmj zCPH_K4eJ-7CViY8@1nQtI21f#s>imxz{KKFMBtYvaT!$tc&Z7NeGaeJELq(|z(TbbR zmIlJTvkU0B)Rwn9e|aMO^gJYONXOr9)BOALOdQmgU_5w%LkrSlHxpZV39^?|QT5V7 z@rgMu9Ll-7i@UpRWLGlAV_dz$Ytbr47}sLxD*ZjTrYiE)U&|Z?M6jmIN*s8x z*CNqWuC4|Cd`5WKGLiW?RcC=Ql&x7hVLvmM=IZHsWgAo5L(YrCv`$IO9fuDy}Ut-0@nJWL5qJeUTmU!*t!&1s1LIj6=4<1 zrZLS4xA;K1hk2j*N{I|^Ij-YP)a_P()YTH-1h?1Ek9kkv0{XhEd*}%o_}rFn5=?f# z16$_0R=CD7?8Vl&=t@5chb1?GEdmJ#Xs&ImoPQAJhS`sj5xy4nP4s+5F7*fB;}JwMrhHxUIK>+s;`Z*0%kNQ*q2fy(5V)tc?_64PH=((*CjI-CA#>l z%vNSTJDdUsrZ(wez|gDJV-ErzTk@C6+%B%Mv!{84k@jb5qI}Ekk@AU zTe{?{4C-?ITS6^~=rxH;?T|t&QgfNk^y`StWlyv43**-~!qd;wa^XRqxt z${eXKuZuc%$TbXU0eUt-UE;OGL>;t2bkUW~QRA*L6jD-My`m^O-fOVwp*d5FE>jq+ z+dup`WSMx}E!iX(XJWBDDn=^%_%(*fNL_*1aS+U)H zPdDdwexkm9Ucl9GVeevQaM6--1byzBTpu0+pqBYeV6kbeX=D&4rsWb{DR32xJW0$# zT4=su&L2AW2Iab#fvF*0+7^5RS!*t29kT+WQr1Bd_J2kC<>d%-g|+SavF!Q+sf`DJ za`jW~{APO5prqjHXR~Jo`lndT)F3u1`5UA~SG*9Wl~$Z$MZ#oQU@&=n#E`wy_3K2r z9c=S}Nk0VHj&M1xi63?ijfeUVpMTJN#9bi(=t18=wJX{t4w!P;Cwv1}oIj{*ICoFl zFAdZ7O~#0x>hhQtr;hXIR`pf&5C2!<@^ikMF{L~HHy4bgLqD9r z+{$`FYsVmywN-rb)-CeO?a2G1* zJ8v}9)$miL-OiYRhD%bRx-W;>2Ok$H|6a_$rvUNLgl*36=XdbSE2}*R(&PyM`#mvx zG8VH4j7RtGp>JJG2LDV#qeaHfk*?@s@pM6Awt|!&$U7x& zyi9)*7EOEQuHa!2KaIuA(`ctAeH!*5qDSr=j~+hAa_xY8oI=JZ&6QUaK@y+hkA@#5FSM64;!Kl-zc-Pd`-T;a zEoD&>%$hTz)N!AW93M!~zmYrgdeQN^IjGZ$3;fLYey^f!Hu8t7E0_Ir-VZ|Gyl(Uc zHzJZl)fxbX1bdGw*l4egx19jri1w$Cej(ej7#}-RmLVKr9`;Gtn^IL2M=!L#Z`hYr zH3)xhitHFDH2Whwv=H|~#rT()QM@+?VzJ6*g`raipMWJqdychhqF)hfY04ZF-9m1` z4x~Mv{#M0OqTrwJJ2>FJ55y?jl@mB#%0tI^cyvr1+;S9=o$h9=O~ST`5AcfZUMWKE zl;6#c%vv|P=kn;6+!a*o`$?oTY7}uO^kR!9e6Myril7}uZpkkuOTA`;EiBI--cnGn zxKKR(7y3WpWz&_SFh_ub5<-W9Qdfcj;}PNivw)gFnC?tfb0N(AMdYn!A1zS#*CmFf zQP`pAp~;S(AImyh+vPX%@hRkoZujAILfGPOFz*`UE6apDN*G(m1%YaXXM&YydB2}F zUdf>{%sGO-?*o?tD%-lcn9IspolX;VSCsd!gxHcu%!mrykRB0+aYb;Rn@6NZt`WWX zufG%n+j~cL)mK{^T>N{T1e!rMti1Sc{7A&D@bCErSE60aApm^J@2=ZKsqn}7O6ZAM1cj#DX~{s zNmhGBwltgQSxMz%$xGnz#^rj8tmxe<(MHy=%-$PhWsN7??w)vG9+Xz%(qJ+;!&>hg zS$Jw|6qH8pefOGQxZpxeajoYJ%|#Xo9m`}**8B^rQ>zlaQlC5)rPgW!Mt=|vw{;wf z8OP?tnqz&EQ=#UZ;A~}+XH9mT#hTdjr?Rj}@nn7B$mHeP@L7E-@vvF9|0uS4LUFA8 zZ^6j?Z^4iN&(LrJF{f2NYa2@#ZIv6B8%Jn|fnb$jq{D!d-hTRcz8%)SaTwI-hum*c zsF~8(_WdY=7MX_9aWbwB9tspZjrcd-PKn7BKptLN!0T9PTT2$nwS06fUbtJ zMQjEl&S;7KX912j&Q^i@oEE9F__Xs(F5^I8juUnDjAig4^vA%K2Ob-)xQ;nxW5884)o!IS{j+*?J|zN@-2D zv!h&4BIupCzRI{YK=&o>_e!T)8GRoYh2KSVr2O{FvW1q*fZ zBRrShUVP~DBVZ~4n+G^CIZjB&A7bg~m8^W0_3=xsI;!>ZTwJ+drEF1k$_H~ zYaXTE@bi)d?qPnFGL1W^cTxu3KlSoIiEsn|ii&+2B&*}@?CBGA`}+f4q*ns>3xcV@ zb9*m`3KwM;ZuKLWNAi$Y@fco##N#N68sMf}KyYV1Sw6(d9`^x^u!Nskr9MDiqH8(??+0*_f#i_ z>p#g`VklyIuh2l4E>l+28c;72T4rC~#m|jgo1takXEuUnVWxCQ@=zN%TX7k<8OV`o zuleC4WuLymKQ#f>BayWUafropGIbcb{6uw*OK%wee3KKhZCbJavFCiG14EiasiU1k zU&RuBB8&0y@M54 zJyJCTxbCx}d%%A%6{frWC;pGS5pY3E3sv$R5#1L+C9xu5(Yy8EIc(TkIKP+T-4hVE0!9uFLGKLBmX zCJ7hY+lQ9-h%DadkML|5p_8>U=(e*z2Et5GQeAs}h>=FxZ5CiG|6s99iU3sOJ>0ad z8yfMCrTkz+8Be}?gK>ziY^NshHZxRF@?61G8EMWn>c8?_*r6*wGX_br-NAb-^Q9?11xxGy-~`l3f*G6E+#$~s#K6t@T1 zt-bP)>rs%QvxF+{!oherK6VE&o#0!F__TRe{W_fwNBNb3@ki}T9D)h8)Nu3+Ly(PJ zFsKWue$BKtYeEuj!y~--@$Lg0dXpu|)cI2Qd<@s&r9Vo_6^ig_4hUx;&gdO4guk!d z9C4z5*br+lJ7ymly42$cxM0me1O?HT6bFunXZ8{UT`B*J=Z9|Na#^U;jNs&~FiIZN z6kA?f4Rc&-ln1A)qR1~nBfornaO3EzX*lPVyc5c6R{i->6v{(cWFjuntW9`$8aN9a zI}G_ysrH1wfjyzTUxCI2H-uoZ`P*$^`v-yJ7wM9XNALirECeUQ*+shyhV%EEd6SbN7NT zM@7-s2xAH1fmH;FEC`BRF>m!392`F1(?j7qU3PEZKri$l9Sf&_kiS6VzMvSot9s;o zJ9cYc4fc5_HH})Nu4|oCS+7Klhy>QQh(|adJih|JYTgZACXy~NU-fvr_Xv3gab#|5 zTsOf{<--YB%S=m~%ICMiD27!5DlFCo<=9SZ&5kgHFZ14csCnXksrUjI|sGyywz101pB17$XF?vIyj$ zG7v`b`l$JNhc*_X3@lW21+8OF2K}?M2#A(^`nDC|V74$|4fkfXJwq_Fo)iGIp{@S22ckH*lV8Mhl8*op;7Ft2iu^>7Eq@ZkF zb+`t!eKTv5vr|6ViL4Mx;6%+gEYZagXC&o{C&_G(Yp`Qu9uwEz;V@k0NXYH$3^f8s z*%`}IAn;q_P_`5RADz>Y#PXeZanxk^Wc?=O zTfDR0qY|?p9Cp2Nd%6;CqWg^#vuNa%%SFS=#nixgV3~ky5-uMAwulGMCf5-uG?_^s^F0=m84io4D#8AuNtlK?cek?# zQmV@y5U}gjn>MrOzpabXUa)2QS|MICP+ER>`>2ChaGG~L-}^MneJ64^@#kTI-3{v5 zBE{n86gcLO+yi05ERs|L`e^#5zn?KDvyRGHqAif9I<^}5n~@~`4qmJNsjngauBnOxTU5=Dmjc0UCH-2U)n8|ojVbfYFPA(jks8YYN|rf>6dzN4 zzI>Sr{-iOT9wkYc10Z1rEjS(~D{$e?hUMNjT&Zl1FV`lzR_3u9Yn`v4JcK|!7rtLTJ)gPhU}2c@)8 z%YZ!L#HM(OEsxag(TiAfvlVBtQk3-2T^j*wUD2zx^01@5uT9(juI@b|=3nK(YD!jtPItUnZUt>0px?A^Xg~$TB70VWnVsk69DkKFy3P&H* zz>7+R_QE@!*TOkkdqZsyERX!L2@9qwFB;iMVQo@8sXJfi_SyO{e|>Ba)ulYwsT-BT zywza3byMQ$6$KJLe^N7b=QaE@7O;V1v?Gyt-ACgvV4P`dqrNmncq(ecXnPIG^%cVQf82(X~;>2D2((VmU*Oa_FBWYVTxDSDaGOB7n+~H)y803Im>dTImYiHLT6RV zDpoDk_}XQq$wsc?>}TT-+*PGvk|sd5B`oRLpjDref9{)%k)aPmuos3x^#VwQ0|2Zs z^SgeDMbIN-*{k1bc*9;c7!Nw=m+01{^j#Cc@Ix%YQ3Pn3{Mx5gZK~ZLd4^TQ<;L|~ z)GP~ObeED?!KC5QT`EWdBwoU_201%)%5EUP-Y;KeS9>oEN;i>!ZfMfrF&$~M(sF@s zvD8zAi@#uY%n52b=|4nUQi~N-k51jDK5ebOHg*jvc@Kh;lI`U_H_|- zdiUN42W4Lih1gQOVbwPuAd4A}&*^gn*us6x$p4KlbLwpho8{&S!8Tvm_!E@i%=yM# zb|_DQ$08k-+Q`uhj9uFI{q7a`eWK$UJa(wR!MtNK_K|tO%#h&w0mz3CCek`++q{tm z3wpAw^+BJ_KeY{X572MCS`-xMQG6u8kFPIYnM7OP!WbHDEXD?75z#0^IoZn&=GtS? z($8%x==)o-EQ%{T@2B#piK4=g1s<6Fr^kw)5}7XljYjY{J}sb;9=0CNA2cLY0_Yru zSVP5ls$>*Ke23!_xL6s5K_cda<(XtbBp7hEx9#xqnvR~{Uzrb>{zF5YRs)ydODVm- zpUHo7|5~j0eH^9$UgJae`f3vG0xMB)SeziH$o_QhL#hJGQ6F6lnn33mbEB6}ZxQ(rv-yBUrN%?x9MweAf4G5W9V{qN3I!}`qwURYwHpFV?lGVAtjMLBlpYz z$mSp!hglh)MYf=R`l^UujbSvzDvtpEtlweD64F@C5t(R|cSo5AI)@Y7%gGnMLGq74r9@28QEI}c?`7^fBmgg$JIG(o#H~ZB8ceC} zrZQG~u_!8m<}nQCb37lh%JY{%{g6BeuOv{b@K>phYcT%u(mD6|?6H3~%21owGy2lG#&57Aep(+!ZDzQX5GuX)r?~IYB@- zPPJj!5hvwzj(k-z$gb3E>X2cpqxv6KoO*-Vg--NE;2x>q=!Z!mFqky2#nkYOv$6Jyp73=+8h!d zCj(zFu86hk{z_3sjFmEVpB7{SQGuj~r5{!-nQ(aI##k8!aLR-*>HJ7f>9698YWogs zj{de6T)W=m3`3vbO)afnp!Ck=Nz9WIk7LLw%K0~pi_?VjF5}DIFV2;MzL_!3MyWj* zur4CQT^4!qNN-3}a5rAa>H%f#><3^AeB(H38{DT0QBwY5`i2O>@51PU3UPXotYsM4 zs$d!h{&#~8i@V2&04E8X0!KdMh*~+VtpBQ7tB+%m9wTT)E{~;v*HqpEA(BK-QK z?C^J(;OBQ8KH-zWY!gLoUpDpcn|-y#vE@u8tFT0Wk4fvan_~@8ZM2#6rA>;yLtabf zMP@!xZ)Ih~F+&D$fPsm+5M#m64$b${z=)!p=2eV}GlqKM$Dzrp=C`ET-i?KJ;Y z@9O?ny(_{-2P4*U0)2F5{IGQh{_a+Atnfr)?P*#dLQ~F3F(he{A%Eh@!K3+%stxh= z1OxJCBeE5gWCr>B-PQE2zcUqp!$f11lDa+4G#4O^i=_`e;Px~u{4TrO*SFw--+hly z1N$FP=KQYQ%DbH&-G2?O)2^(BnWW*V0ywdf&V9=rvv#}BP@_X5jSa?v{CrVDO@aS0(ksdGMeS8 z9*sfc5Ui+Npo4;{0P+@LH~W)E0aurfVm?V9i?`a4rsUi)8=4&`F`s0fv*`&ynTa0> zOL&Nj+{&60e3yD$p8xM)qJ5NgkB;&lhOyBNO%_u!{U;X>W6X2 zkg7MwO02oe9Dz=_ob?HKOioo*jjNHA@DXUniR#jU1Xq*X<%fa+;a46LUQwuxvrXN& z7LS)hPs>i!UlOVo&x`xBEh?>Lais~mjWtftUY+H}X-2#e+0C60+RceY$1rukS7~70 zCGTz-G|6bosCY~({eZZw>TI63u9jxPkAi7mAvtbq(I_Hz5oatXPWBMNX_~1 zEH7~ihnbA*R-Y_Dq$j&#to_Mg-co&>yyOgcFKM9>YlmqwrUpU%S+nv;@y%tbEVd=h z?UI&g6$4b>kK(3TtRPk2xV-b*1ZP;j#c0p+`!lStRZhw1T;7DJ$9oN1jVCih=VX&F z=Ugf82+QQq5#Vax&?-aJ!r>dwR!|bEJ<~IV><8Jw8WWt|c})9xodHKc04D8)M#cnC z9V4^M?U*o3E>E+bipA}d9GJ2zxVFcH+j#DqXo8dHEbr~Pjg_&z^|i8Nd#O&f2TvRk zh|3bQvE)WuAl(HR$Jf6|byI(-GlrY3L?0b{TuZh(w+Sio6_MMu>`@@p92;G6vR;0X zmife*r0RI2m1u&S$WJ)T>ka>awwDiV`9aL#bY(pMfLilxuw?_ewj|Z~f_~?BKqp2a zy)eIpJ*==o2mE$|XV?)mBlo7?e}ZlKuY3m=IIQ!)$m^G*E6j@Fh#KsithaEl(N&&l z=g@+_##B1|K-(Ib`MFq}1oYzhX<3grU&1@}*b8faToYFQmtS2XuGZU;nC>1>f&tIe zWa1a&J06N#mDLJPNNgnkZ}KGDCfY#q?Wd@2xm!zvI^i`fxw`|=y7LRw-LZQ8*J$?h zxrxe@no{<{{Goz(^E;`<>4I@cl)!Xy`4uz3uSrbRtdaXYIvCv}wK#F-zHj`~d1gVxVmEjhhVEPtlmL|B=*D+4R%&Xyf|^cV}ARkbVD|WhV!!MwG_H zi9txZvq0yt32OPwF~gJCN<9tvQ9ZUU8j=0M~{`e)#v2rOwzj#?3ofu zcZ=kQwegA@L=WH0pRa#LxB0$zgcuTze9fJ5FVvdG%EvUwr;TTu6uKRI^LK0IN3$}7 zE!3lc4LJK^gWIUYu~b&3z+%b~Q}uJ}fLYm{yKZHd4T&BIt4x(Y ztc?*@?Awkd;_(1)PKkQW_r309H|~oC!#AO+9UMU*u19?vDUeUJDIJhVlg86X2XQS! z!*se3@q}m(sT^z7pzq)=@*fN$lL#L>+vke;T5~g&1MJ*pxPm42eZW6&3i(|X9yNDp zt+o_*3V$)9(ngsmxtL-INmHk~@QxDNei@D=*`EhzMPe{?EQcaplrPgP%Bzr!%F5=1 zh3W{Y$Iyc1Nqf1uW+r>pyLfB38Ho0|1GrTLxV1>qVDVm0)Zq>3Jnp+&)QWvLsGG1< zZ>U4=H`D^7mv{|)e)krNT`UHd#~0+;4wiu;7v1c>6jBwDYX<=au^*8J zrqF%@0f|@@{;#SW8?J8R0^^aG8U5xLu<1WOwh;9aJ%S^)<=RS{f3W?`wZjl4JWHyc7R}S58J2$UB}aK#63GkLG*)!XzNC8|I0KRY^g^;_=y7 zsO9EGpr@rayIGLpkupe%t?TP(DEdJuh6@6$ym7_an$8+8_dYP_1O5O59aqf6w4W>_ zdO^XgY3{I8Kk3M{uJ#)r9Guo>SsfSNj>pik0SNmsrl6_+zGZnr&WcIo^-xF+s91yD zazFWPWC&ad1L=miefaC*K))f;A}oKKOR(`BKMkAT{IW^=)+Bz84vs&2n1D}{^wOTo zJH@Xsk90kG~sX-Q#e@cxfh4T#EJqkyxMb6dA(K1Y)ThTkyTBjWD z9=^rm)#<)()n@CJ!7eUt`ZKE!#8$=8724%zy1u4FupBmmO}K>zE#+a>(@q8MN1Q2 z_@99+=dF)Q`iBnL|IiVYGX#E9Ai8;PK`7&9v*vDD3`oXBaRl3{O~uC70B~ z739IsgSWkdIu^gwGGnsGEc^(V$te`}@=ffNczOBbd)*BiL24?dq6OJZq?bPM=PM4z zjKGPzzr5aYZgY<5_wd7rmvH*!VIIqz$V}iEVy<`n)u^G$8LoE&TjJ!~z)6=_N1_>? z3t7+7)z6JkHv}pGu_}-NNez~18L)}EcZQo8=xbvWER~fK`w8@Eqr(N0Lz!<6t0;wK zodBNIF?5bv@pyPUnmh-I)TatRy8mbbGC(3B6MzX*t0i_ux)38~C7|oCCOO6n(k||- zup%%(Ws>y)P=Q0kI7x|qD6v;cO(Z{~aUR#{E#+V)9lBRb1p_hg!Zr0x*&0xdohg43 zfc9H2h?0bu5;d#X23V_K0-77BlahEyM$$$})&3nu&nEt^9NN;H=y-%l+<5WhF}o<1 zsI&9^S*X5XD5`>BrxBnHwAkWuuad_2r~RlNF7T?#Dq?gU8|IdAar>%-l-t-77B*W_ zQXH>^Q`YoMwhknm%CRC)B1kl=ZVdXejU)P#LOah!;L-NV@mfYz@7h3of}@$g<0^k} zUvWO9dcBgUMxUnM4#C58!h&Q;o?=%6^)cx8b2gm(!Q}#uFYc28RM zCsMkmDkV^)^7`-?P1UXvKTTX?o>4e!K;jc{d~t1G-6fr@^oLx?<&YFW;a4ciXdh*G zTf?77Wkyx;w(tdVjeM{`C>~xJLyTw?bBxKjqh&6SVh?&I!v#{=Ur_Ia^4KtnwKl`W z^Q7117(OhT)T!pze5xo-lK+zQl$Hc_2hrjAlTpybm<{!#r z`t@@s0WsNRlW=R?_O4J5T+F}DS9p?G?GhF*r zqq9nOC^oB9$jRxc^J!QA>>T}Y+q1>4@e+df3Uh{Y6hiwMU0ea}BwHkPC;OzqLC7)- z;!}(n+pk~1dt|>L&Z5l6DJpi_8;&R&6S7`!o8%_07W@HCUlOi2xPo4JBgm3f zmv0S77h0^}>@>rxm6x%ce#J2mgbSUemotvL$Z?d3WUHOgc95i)X7*}haRpBp`H?0W z1#>S4wsRVyArGTEgAod}7bh3nrZ@bza3#9uH)b(_PyFwnaTrs%;M(sAh3GmSP6mCK zwR*kti@%Ke-WOE`3*_Jr0bW0R)C_gXt5raz z)M7uQHufd71!W0s$5y`G$K1cm#?YB-BdqhR>V@|#; zAg{kbqhp2uLwDE-$I0bvw%OEn%QfpM$v12P!c?KLBAq~-JPUhN1iHRKZqX>nqG{8} zZcqRlV!9+!z*42;gaPO_;7)(jmqmg;&@?iXPpC_svSBQl=SSsMVK9F|EX^W$1p&p+ za>e30ydcU_c}{5Khf2~dyQ=7lG;v^DkxFmq0{P-W)Uwdx_*Z5- zMkB>$c7+^VrWn=A*G7VQmEaNe5;7i5gq*j;)A?NNLJwg%+FJm|?OyIE9$D|Et(e3D z8PEGL78P;=-TM*#)*#L$o3wQM@7Q-F`2Um!+N*jMtfA@zWXqy+cIl2}iBMhUYSlNh zb){>}|KjF#z zt{>`5{AnEzkiLJe#n?|k1?@tWhj&z~Ctq^D+KqIz1_Fv~+OcpIZpD8vvShYmUE(2c7?6dtLZqn+%ReMe zV6oG;+|2eLUtcjGBX52QD%gH_S&z&QrC<~dVCdR=UsZv{+1N1j@Ud1fj_y+_|(g4Efqce!?jrHre*$Ipf3xM|Lb zb}R&%mA3TV6P$aE7FJ4r()r1;1o{=G9O({~S9x>f<$Y6rjLpp|FRE^q99fEKKf%48 z>*hWNGB)OpX=6}j=GkR|0zXNq96i=cw-I2qmRW`_!+_{<>nmxHK z+u%HyGGH;r9RT#oxti`mQ$%bXJ{we-Yz1`N8i!zLRQAHC%;Ew{od?Y{^CG5Xh%5{LV<#GV_m zr`#7s#UlT|8HwKprfH;_Qgs(6I5Mfyi!Rek)m?BOt|s_26?Wz_j^IeRB6GZ9D(z*a!R3+Y6+`TGL>Kl0G?*g@P=%c+^K_V>m zjOv{rxWjsaAyWk1zLKFB={A}R-HGN#7a1Vbr|K8K`fO~jF9!3PuC3Q&lXDDhD1SL= zf0x|6Oeu=6^Le`RDbQ!4Sc##Y|M159=Pcl;J1j*djt6K5#UJZbqKlXf`(U4*wZN7^ z`vrMM#lJ({gZm`V>UT4KzZ^@i&U@Vt@U`YP@O?*z>9#Ul8&~^Yux55RC3R}?-b|e~ zbIBL|uo#MLa3sy#`UIX^rjKsuH;QTg-Kn?p@VSgmt4m>dz2?9iLd)@BXJsf~0BlmM zm*}|r^sEvm zn~L~%!p)rJmTKs}kWK~xCvF<$7dcRHmSceo!iP(sgO9!LGB=yZh4q)I6kXcOU zJzSmJep7HaKQxDlpfN0`TI@%m09Dvu=$EA%SW_kfq|Ci=?~#|7E5lxc^ZIqINmO?u zPj$KcEJ-za@?Z^i1v8(LN-(ugnrGR3;<#as$!DCAf#_}|UK$u0Ds26%Ym$j685YrK zZi4O{G)ut|7kw#m4Nw%O8eI=M&=RgR$J2E|qRv79Hr?lL3~4Tv%GZx(%Xl<$`-iF+ z)~1{zp=rrJK@b?xA~7ryx$UFXLt4UEVw-03@TpUJiQGL)e|7(;BsJ6yJTyFLlX9F$ zdZfSY4Rtez&}0Ffo0_k3)G6QzWuk;$R2i-GD)pWl$10yz=0>O3c`}Rye5Q{CQL$ZF zo5(odv00wM$pBh^#U9lEp#m8?QLD@;jr2E(8w)4y{uZJ)rS|2zOunrx%WDf0`Hm(SA-d!L0Hr`$zl>C!MPnV_ zg134bu%~HNEsgV~%C||C52G#FBk6aP=~mxkXb97fHh~+H(VgRXR}r@?Xf5KlF!|_~ zB5v>53$>|;_xD5(!MC5jE(?8NXAvL19YNC79{OkzpA3DvQ^n^vXaOJQetUYVDID;&5@?4XM7RQ?e{|Zb1$TGaW$@|d*m7N z)Gczn6?e$-PTVcW{p1%hK7UH}dk&3HQPmaWGk8{x&*6DFda>w(`T{n*C;;6?cwZ!7 ztKc4q>PcLV8VT_mi6uZl*spJikRj!ObUkbjvX{R>p# zWm3PRwYDboe8Ly{d%_=0)P{WtCG1bAk;H=9ro;lJIT1P>uDuU0?vwdQEtAA+&A$Oq zO9u!PF%D7L2><}*5|a@c9FyRF7n8eKGJin)Top>fcB93lL1KbZ;GqW7S`(j|%kFf$ zbiZR(<8_5Wq)M2 zSquRnZL=UmnJ#qXz{$3Q%g#shXKaNK}Mxq-vzh*ZqI7;n_-wT5BSNPnk62oyVE zsw~=ZJrY<6m18Nv7YL&qHNg!PO$>Fwc#%Wdyc>@n77Znz_Uxu4O`(cv7==wptB0*h|8*R91nx>OY&` z!tIQCrwk2+0;X_RVDcG1Ht&84dH!I6t8;8@X(*z^_kH$OFu@kE^aV4oKVh~~ImN;W zu2*jIaRU7#?tK8pv>cxk$o&9NO9u$5i{lpAlfhbAe+PV9)w%z^t8uTdl_kYVlqsBq zII(4Ck)e=)R}v?(lg1e+gG83DZ4pV=Xe4n51=_T zp{2W&*F9bzrF78+asTIB$+m1cq`#M+;of`B_kHKv^v;bd3cj*c6QNJb?mlRbfbrWsWSf+PFw8NtMcrF)sCjI1`s z!|Ak2I+Lf%$m~p+84v-BO{PVovTCVCBW*;osaU4BZY<0O7rAJXPUSS2Y5t{QRhr5) ze+dUQLRpr?OmoK_F|rHdZu00fjixirng~jz8BFCM8#E)*m{3fCXwt~k?b#Isp;_eB zX(r8Pa*f_mX)co^WA542G7hZ;X!B`-PV>lDjMk!3B~uyBY=@5|Ajb3p>S%4dXb~;e zX(3$+t8~J+8dVip&4N>D8I#kvF$*7Kf2ybojy3CsrTbk}Lw=pAsTQ`fIEk5cf@a;$ zaHbnZT+UdGddg5AA6aK>rDG5HH5d+5e8GARY-Z`23|e{8BTJXZOxJbK&e=HCzW*G$EL~qvVB;7V%m(mHMqcp10TcNxW3R}bJZiuVW z+fWiLtEL-zEmq+u!D7hPa1V}qJH10V$vejp!nR8P1p%Z&;8L@yMswR}#^Y8c0FgWC z-8!A3_b_>@O2b$_`#zoSpwps|1;=rn2YJ6vx6|EBYhEcB7Bv{1e`d-G=k{zzeqW^z zGHt24gwtBs8^%J6Q*NH059{m zkxGLi04`VOkLdITdK5DH{Rgh!c&J*V$MBH|XHc2bF8Y$-rkcKt(vZ$}r1S1wQPom1 zTYr_#3+Ts@dCg>zwEHi!1iYfC7Qs=L!?9nZuM3s^H`B`he;i+>Zy=lH*%elPN@DbM*&x&1LcE4ck16bQ+!U{><_Q)I72s0*T;!=0L9X%T-> z7yaBSalb&Sf6in04+(@{6`D)QPkjM1-=+LUr{9XwSspQy8FaDf?MAPQekZ!IQ}lmK zGslY3kd4KoqW=CK#RmcK2c4c5t%*}K?@1I`e@XEtAOlJNM1K|}{(}6GF|AD(y(k)) z=jm@S7J3Av#e#ZW^bfjUXy%_%>ri7)+{mDJc*!#Ff6L$`j=?0;Ewcd(IRrqeX3Qbw zX0px9_XRGt2@T)NV$hIu3g&1|MqTU_J;lAO7WcEVbgEpI?_7qPs<8!OWM_km%h{!~ z&Xa^fq3EkG$2-PlgOT=vr=lwGG^Q&r4@YGW5<+lHLCzQ0JGr8ar}KUM}L` z4qhShL%KQ9lj(KwD)=9J8Iy%Q9ecImV$2d^e_`#oygOWIR`PlQfpKENsM3jsper1g z0pENgV&tub(PECpst;w&m&nF5F}S$TYCUQ--lX$J5pWCgP*KxJ`;uk`;KvMKIN57~ z0HoLeP zf2|!i@#f*ixmGmJwX$*Mt=52=_l#b+=4GV-D194m7qJlp*|BG8+y)zitdTtC;++;C zW|wLC^GA&|+|IPHs(6N*VD#WU7%+G*Q&kDYjJUQSu@zwyN225Ftos8i`bP)-f-z?< z9pi^C-p>bg4)H-WgC))jnq6Jufa^ukf7x&GcSPsI92Nuf2}B@VFe1`jJtMJJmLQS8 zGig3yM6#kC;!b$INHa@H>SJtnvd)a@+{HKGOgMgL4Ar$LAB{PxQNm*)Y09sgkg%L!7VP%aJG!ojJanfe|S9x zDaKo+x@rPhOZEJGf_rtokufo?tSTk7Wupxxa9b?py;h*Vj%juY<6!c{H|TsTW1Kp4Nro?BjFOv0yyQ=Mlg>Buo6&+qW1_X} z$XdZ57}NX- zZgYl7-^uS53dT4!DPz{RH@39oTLgZeyg*@$P`1{lt2BN;Jh1o@t<^}U!(B#GtjiF^ z>;qPsl1532%efU3r>W93z|V*H!#aPEF$FpH?B48Or?D7(K(?VbBfM`$e<_*=8eIHw z{)A8him5Z(6GhGkg{lJ$qE>y1?-evZU8u9@?z`(6VqGoCj3E=mXMhxy9EeOI$vwcI z6*!;6PF0H}1ABd5=ll7L=$_7tx14C9kPD`cHeW+Hjhgk4$meN(7`E8CYsa?c#@!l! zVGN|ar{YH~$a8>vb*#t2fBvGi_9bi0g8PcK_EkiJaUv4WrenwCjcRzS8BE2VRw^X& z)JpD^%!ZZ~zM=C4e$w&^d4+@eQ8a+&?{)Z_{4JeSei}xtjYp1ZfBYR-GjTMEG2X@B zv+_RXkMbD0{1iF~Glll!ht@iVj@cs=cV&|qQB=`i`_2&t?qEvOkrViu^O3pA~(FmJBCNk(FhGz0JkH zR=dr@n0d?;}B{dV4CFM;hW3ZSvd@UR44kwdFJT1-AXnm;s z`+|VuK!RXcF@jxou6k69~=H3eys9KXk_G_Lu1@b8?O@AdGX$nf9!$N<%SsTWIKD2hje~f zp`v+YcQ?!$RTTxPBpo-59+4fk0bH>w4qdS+&O%>b05^}z%Nj)kWCX75QgnI{?y8hS z3;AD%T*@R2Km39+83vBWIy7Y}I)V}*&|sPwWQ%Z*_{Bz!=a^zwsES)xJRAViFk>X9bJ}$gaa(+^8LKPd6?&tu63!g;J?2K4qbcTCBIlLY4!?Kfg?XEwh2L zL|0}jRVZI5C?fhSqm8|jvQ}~6GNoErt_Fgn#ZO7_f213P^z*KNivo^W*$WXT3=$2ocL}v^etJO zUQ(+mn3tT$u_)+ccrBry61*1XBxS48g62WlhGLNKhsC|UrUb<=j3q9oM%}I`ZRi4& z9ZYpTxET13`i_TV834)bKU}MQVVR+P8B-R6e*mas+H#75FWxa?mHT38U)K6@MN{?^ z<(84kqwE7uBkIEp+YKdQR`*%Alh8_tY1yUk8;4U>K8P?yJ*!}fs>zpK-^lc5l`f(0 zkx5w2PB;jI)uu)yaV$kKNTw38q~VJQKkPwelk(@2nQvP-mQ8dRDY=3a z?;ur{Qp6W&_>Yw?qDe>aR!*cUr?zH+$^#EP<5Fvho zedOLZNcExC>Krxo)7F~cvg*S3cKp}of8Ocdm7~4=6w1*->n}J+*M|-sZ0o16{VW-d zN2od!vbnq3?e186juP(bvy?8ZX0du)tnMqU^kU^TVkP8$9RS_0KTB^IptlUt?V*5u zknRZi&(OPa^xl5DtDinFNFNFX9Dc98pYC~xKFJhtdYuo^XPHj(d9OpfpJ93of20Fy zjs{Ni$GxiiVId|>8>BA)SD>Ej8@hn?FXregr^yR670P+Ss~*nLg&aK{aP$q`hyCx! z{aUdRrv02$CAf3;W3(Q~J1x}YWA3%pJB=V=GZ1XP)XdV|+7NY977 zWry7_^wS@6^w%8yUF=rdYCw}@wIZ?>GZzB@@oE7O=o>l*I~m2yUKFQ9Cgdv*&>%2!>=5s3f4p`u#o7Q* zZX2Xi;JlxwxO;Q#KEpF}JbT32)KX+?56{o>6`?iS-84gVo$K6-}D)ob-;MW}Pf9IP9`Q}h7B5#my z1xZJBKcDpX^KF0+wVmO&3HsCohCTfD9KS2HM!j1&_GGWK!qU00org~q_H@Xk_R%D- z(^jEM%lJbeGr;f7@m&GU!*>txJ)uCE7q1`7@h5Y9-yq))KeDgUa{OS02A0T;62jE=dbo%RIf6L7Rs<5ASh6h0is+D;__c{V)eQ*=3JR(+&6O{ad z&>4Pgm=>H<5`#_!wX!q(f^L0zdFNl=iRh*ke>_5 z`1(@~IQVmp|0W&jU!k_gX+9#|KAQ zQTvBUud%Ic?IQ=b)|{vIL1lL6U=R>`olmWP5i_r`XQvJT5vV?o8jvUbK-!@iu-{5hdEf4RKfRt>N%%LbI~LSy52=eBbN z6~i_jrB&MIcR6LJN7*HeTvnvXXWK_5u5O`MhBN zk$5_%e>>+mPY^kmIakQ%T4z8$H#s-U=VoV%vm4K#bBBEHc3v-^9nNm~yv2D^t;h4E z^PLj@l=D5}sn)AO`P`xIlF!|0r+miLTf~zT1=u!&Ru6$aMWu}@EhJXynjxB;{|4D1 z`Ut7khy1%X5gB>2(P`*SnZ1ZTQ%}29ri^*$SO0#WiXpXIs=Gu1BJX<%-f43!R zf$fdtv)x8l*uG7bwijuk-A0S-DlN88p)2ifT4JxFDtiqrwXddS_O(=PucsROb>z1n zqFQ@|>g*?Jx&33b!rn(K?GMl@`;Ta~{YARU{x4eNU|Q=~MC%-WTJKm+0Y?jMaO|L~ z9SPd#I7XWsy>yM^eRQqk0jhUSIHv~ZT55E@hnk#sQM2=hv{~IuThzDFR`n@rQJYt%27H$Y#+5QbsO9u#{)Cb{z761TUEt3%%9FxOwF@FhoTvZkR?@W^SGMR1(X*;E~ zA#EW|Gf5X3$^eCuwh#>go0c%N5MO6rl2>Ntg_$>PaY02bZYYae5f|KwiVB!cB9S6u zTR{bJ2T^fBLB$185s~PkEG}W$f%Qe?*S@-(- zomT8hJAW0gkJQI{>znFhZgRj$Sf1mi!bvx7b3JV*Y%61Pv){^uWBqpQ%1kzysgLwp ziHzM;KhPIWS_5H6c*NtUty#Tx4QbQsisyT?i3Ari{Z@DtQ9IS=q-;Cwr24qJ+fHXF zi|gx}*EFvS$L-zqZ#1D40$px49kVw(30q;In}66Z3X#9n2lTH1Kb+L^Eom^`@K zN-RydF)MMIGmw`yvqK+q+!n#lRHzb~xRdcVI%$QPB9?Y`X2nz6(ure-QnuH!ZA&{3 z&3_RxO6_&}vT5y6h2pdA( zRBt+#uUNCjq zysVRm+i3%h0jv=52HAC5NqeFOd2%ufqgj}>(9`0BR9qq4a6IAhXA7dpVii`4v^6xo z*}c-lS_RW{^Hf2cE&^6yox+kyBREcqc3ngilDu>>%t$)QO<%1Ydsz@?W4-L2Lw|Lh zjBp8JLw@Nzg;_Lq!_JJG$a?n0me(J|#=Lc#6c$XK5(duag|uQZJHw1z$(-zKm^Op{ zpB2*_URr={QfTPAcDyQp3-D@%Q(xgB0~b=;JmCdyk`A~?60#E)k1G>hS7$ssX6{+B7Q3#pAgGJ1(PfJ4u8B;=-$Ny8n2*%_b`}_XEO#aGjQ%W6WR;wRPMcaUlp#$ z4Ycz3eFHZ!qu87~?Y&+Q@5lNo+>8&fvZnOHhphWNg|S zvj_5b?yLF!lP|?1d4D^;#Zfiy#mf}L*Kxma`3AjF)atx! zZ?B!U<6CS?x4v&OYQ??w)IhdSnTp#-ifyxCPzi~FZ%q<5-IE>);6Z#_p?urc&Ea(> zzN^qUMp(jQ%C7cE07vmXDQU-!^ZitQJ$^@t=*`*xH|V_vA;xpVKLAZZ;9GOSxWMuT-u&-l_gNRx;-NFL`Mu z$@F5X8Tb_=m9cv5ZD|(LMGX^b+{7sT2EPs9*LZ5eEKw{P)6NpVmz(#rf@(JL2fBk! z%DAZrmHd<>oyVT!f4L#)*4KtMdcRU|p zAN)tL=I6_p+z7hwUkbi$UB^0N$sSMs8!uMk1^kDiJ-5T%!`{Oe#hB<)>Pbca7cU2J z6-H^u9w!xd_hd}PH-gFW+OwP#OZthWR8hgqs*LAVIsLQKNfm-< zDnnuZ*eSY12AtxAs469^`uU16RTT@_>1)@TY6gv$=4++gltX>>%~iAX5T#~I1>ZhJ zdaLSy3aA?L3mscS;|zWuzB?AU4^qI zNto?ZCh>U2l-!_}lecQ*Eg3u0p5kUYJK)*zvCFEON=B&mi%K?{MZ0z5h7Vq4mIXtt zVuL6=^zus+2mAag6g>ICEbB?JsN>B^ zIvIJmW~4mu>Zyo`C1cO-wD;(-Tb-pR7ZkG~{zlLq6{S_()%a6ZkCOOstXTD+m`gMtAH8 zl^w*~6$dfD=^z$_4`N}c{2&$$;pDp@e{)ceCHZsaa>^uk{|${JSQhPQ9K`$_mXBaX zw6SLhO&VR9!)ev6{FlQSLpW;?3vxJjKY!M)$f0dNnt5g}e+!~HY#v5O^uj^BCfa!f z6$kvYR@{wlGTEMkl|#I{F&f=LYEsPa9K^y%8IMKE2eBv`sc6cfzk3kLh~aNFD_SeV zn!8zR?nj_094gBp8!FFX?=7er#x)W10NMq=HX1RHQr76RA#()#qLIK5t~=CP<$rGt z)&^^GY;T4-L;h! zx8aeHaTE_VX{u<%(CiFxa1Qs1cYp6Ia(p0Sj%cYNGZY9HLJ`hWt}LNs9O#e{9FFdg z6Gx*Xc#s+n;XBn258=@v{4j@~M9dr>51A3;06N8Cl_6QUuPIuz$mpqlk`@i)cR4&$ z{l{Zw75B}a>SwjZe?7LPB1T!OSzGCQZM3!WW9rOW^Ol#piz&e0Le1>Xl7B={Rk9t8 zlu3ZApBu(M@5W0xCa?14RKaS73uCf|6 zv#Y$dBB$omR`hfYsS|Q)KGP(nNuqm%qKbzU^agXgwaZV%nc8#)|{g8edXn5(bTvir`C7t3lt}KO=tMd5p`} z)TTmkxsTPjk?(~|aSuMq$y?wZ9H#{iazqvQ4II_*Av~<%;~aXn3m{*6?fUQ4Jqe zKU7Zv>c{FajX$OSDA0Gk?*smsszt+q3j1#LeL~{`1;5Sr8I21R{C3|#jSCcf*f*ka zfr20M-LG+hfQ62eJ^NSpy2=US7=-y zaeuXcp5Pi1hfkf)vU?rs{)Hqn&gb>HNVWjJ)^j8 zN+Op;ROpFOlub=z;IO9pg%~ys)e~BybEem5luh&hL}Qce<_S9UWU?BEQL=itaF_!P z_fBDmihIcKP+=8rGgQk<@RLfMIYz^+0u>~%r5i0{8)c;%;`02)3pja{9lDHtx#@WCKTt~t z2$O>I1(UvhXn)s3>*2<`Rq)hx(N-ub-Uyn$&&o_jw6Ay7F_yC$FmrAc5ZHIW|~8EW!xjm$DK{!wCc zsrBM_-Y+gz#-PB|wd_e>%OvtoudXS`%NST3)$u;9#PHGA132V008is5+=%tf;01 z2KV`uQ01n~KQq7;Q(RRGhO^*sFwW~Nck?K50F$eimoJ!Fdq%DPjF~5(kCyrtrB6^x z2MB7q>hqI~fKq>W9MyF`*ZY{&YWyuPB(#a01}G^NZD<;|W@X}lp-q9%k~Dp%rKBxM9|>tv+O#z- zZ2xm-R@#*%NLzpY_RhKY+;h)8=Rc3D*WUl)3q*94xJQ4`>HF>*+=k>I%SvnTSG%J=I)04-nLdI(99?{a4-rkfOjb*f4 z%wQR*)K!}{Zr%jm{MPdRkwQ9+32RJ?Z2+lfM~$qm=Z)+rW{>N63uj?|YsaRJt+AAT zyy@Nm2|<6sA+wNA>Ngr`UC?D_ezbEmucgv@=XhSr<@9`Kf7Y_KbXp;=pe1)`$Fb1m-G?6Drp(lf(pY;QXt$kW<(AViC3Nstt(6SVFBp|?T}L0U?6AqvsL8uHPy z5Cy1)zgC1ONVWWR8QiJKU2E5`UoU8M&I`H@-4>V5G|Wyu%%!Ajhipd8wzd!0yw)B2 z7^Z*h+fm)_OKX-TsG+s3LYAD|7NRR?HCsUy6skN{p(Z#)KVew5B@K2cL~E%fNX>L* z72F)16lxXJC}#_{k?!m>(`ld($hH)U2&&ODIeQ`wX@cs@dPq*5gBtA=3sRIiz?#Mk ztAKOsTH6j+TO&m4X#;DqQPAR9YYGCJ8fJe)_vG`MJX4`9LF!^p*BaJ#BM;5Y{6vVZ zb}rP73u-B#zp*twJC3&T#jl}jc|VZ3s9JG_ZV;px)(*a1hk(O}Z6TQ5b);W_SdDOZSYqML)%M*V`;{fMwqXh2YN>xaTr z#@MbP#c8)7uVvh&OC=)xLrs4zw+o65*;*c{V(kWnJ`$wc7+r1EHpyxk&KEXk zojG89JD;Qp+WFyF;p4SDUv(Na>Kwap-=v^rs42$CL^&t+xdltm<~dOE;Z6j=hi>kP zZQQ&iK%*!nlEu=Kg}h-;bnZeuZW(_t%`r*` z=?;R%%PY0(&*lm?MCe*ZA(N9swek+$?hI0nP>dEF?p4Sx=L7ImZ9fh`;tMhFYdze{ zkUT*XK^mvK5LJa-O0%K6f~Ed`7JCof%NpHR7AJ4BZ!B+)Yr~u??}waZ+O+f#{Ww>Z zar!9aq~wKg60%rth#sV$U?G1$S-w_oi*r-SqmBdKnNqaOuH)|sDyZEf>r z{e-jU5=c)+^v}`9g7mZW^V|TS+pxz%^l`d{gZvYiVk~8G@l~yTm+p2IfskLIUu0cA zDJV9-3+LH+ig%ty@v+Uau1j0zRP~qWGtB!K*P1&E=%+&T1Si`z`elDMAUZD_HvO!V zr+)=0AK>a4w#;DF$RD|mUpXFqen45o1I^1<+GnIP{)vyrVc}`uZ2`S#9YI&&U#xV>gk8`)HBY87} zG+^fo7N;Ij33sQjnhKtf%N%o9m;Xa8;MJBU{MmdFgg2cY$7H z{+FucRj?@Zy9hPBc6OP0eMef)Kq?~h_qe_JGQEsEl+{nz?!xTYp02E~(pQ-MHB_Wa zwB7+VClvYvXpD&7jY1isuW}^6PG3XDnYSP3nBSMz_|<=;In4-X#;>D!wX_kL5m=U> zD}SV%1ttHO{v=3$M1RUBqYw0Fj-h+NUH`&KIp08@EIXmMFfzi4U{ArWs3d8%KZHg~9)ctJ*)(clhU{ybDDuD51zHA|0YH(s@Sc_baRDo{B*F7rX@e%mTuc zn-sZI4bZ=GMn?pBIr;<_P>RdJolO5ZQ#&RDeoQ z47PC_g%DoHfXV}60jALc! z5e>5n?!2vgMZpz~Fu7PJXzh@mM{KBh-7e&_Na{E5+qV~#l|yPp{xwrN%qdosAA9cU ziokyZ)n}Xg2jdkcaTeoHZI!q@C{~Iqs<*`zfmh=q6fv%eS?9Tj)H`ec%o-#$iRPeK zBi14(;KkLeSw^y_fYN{z?G&Y%Cc12y`Gg^K#Fb(l+YE2ddH^snq;~qWD;cex9Q(P^&Sq9$ki=;AI%H;@&Yn`R* z$~BJfa5HN0tb5$x^h$%S>-*sOkmyAhD0)O6H-B@qj+Kbo!HBvMhEow>Dugp`@KAqz zha-FS^vOdeGH_Y_nam*YKwREBZ;leX_zHMrEg6+2u;G@t)2WKRtmArdOVuA4&}!=( zQA3DeJQ_HDovGQy$C(At_KO2Su}>Vt2E*bOI-f7((B_0h0+}5vhkV5UmJs12G!LVQ z5{uvTdiIjPWz7!lwcGU(t&q0M^xl72+j0JF;wZcM_Ub<_{ci;+Vi+~L+yUanX&0=% zFlg@Jo+q!+n=SCXQc0IXcb-VY!a_xiyvccG*YBB2aB}s zGzHI5=fEUgA1%_R#K0@$pDcgTiskqn!iiDV~Un$Q13Fq)&ni(DpuF z$+rIzwp_&X93>Xei`0zy=0qYMsXshLN1*H}YK_YC*F?|LZLHu?(8GU;_{+37`hxcf zs)>3wPLfx=Qh7w`k|P#MA|gOUzW@_05O?ACZdzD^sXc7zHlh+N<-upOOVyG`q0QyB zh2#UJP#0(vY6_R&4`%Oj83eN{4tvX$CF)_i*`aU1*F{=gf^QaJCJv9>4Fw`_xE~w$ zVs;F|j>B6YgV@I~I|+Zjw*ZWHu%`sR8q#UR^(wD3biUgD3VF}ekDa6J?(>vObbF&y zWYsO9F9o`NVK2Mv?!*@VV^kjt`#g>QBK_DT+)bZo^e=dv{r+Yw>@CvCBK;KhKZN~L z^sr}(uhAEK;YsRx=ZMgAjj@sZSp~=>sCZ-p+C! zPwXzzlc%6kG*P6dfVQXO3VS7Sq}%I>40)qNzV;!1Y^eU%!(PWl&m`cW-@G8TxBv?@ z(q%9?0m033Y{T#7X@`BBg1+}@nL>9clvAisq7@|b9Y_)iJy2rv!s3n-r}^Nnixen9 zy1!zvpr95IIx~MJT?f015cm-LbP;(gHb^&jR-Y*SxCom2;KxUB{A>8G;YW}gw)-Va zh_*x|kYgC0W&E!7T)Qd?E3ab6N204X0`ceQxzptHJacjZZ2B#5KJ^TC3VHX1D!JKa<7(`R_&9HBZ{;o)W-|emQ&VtsbX>l1^ z-<>RdCzf#W^fvxamhK;j;H0-Nv=`}nXZYs=vRM#U>6J!XD#zud%CU)BD8kZ*cfq1QQz!PLaynVx+^p>aTn=v`V-!mYeg0OqBPcgerK42~e=r2vkH zG1k#Pm%T=JVgu*%TcYjf=m%#KyCaFS6y`9D9<6`>w<-D|!Uq9X{~!2icfT*<2XR)U z_g&W5Q8d=fD?HOn7jPH>5`N?k2Rq0yH!7g&gje{So z6-IwncYvV zdWvFu;@UcT^$AX9UFYeY1Z%bGD5DjoxT!((L+!BQC^sc`UKj`iwv+UIcPxf|iK5uk z?XPPEpm!C;;1+L9S4~Yc{S1Yc)i!z`eDpLek1gX#y{qP|&@i>EwkSr!fp$+LV&%Imp{us5X0r4I>)Rj< zRx26~g#EFz;*JK9h7HT1gSr=pvb`hliFxo0s`ci9@-^X{261QkehtFHXH1$1_bz`p z?utx_6BjBazf1v6*C)e1{(?epM=sDBM1v_NMO==b1yJ&IucUco`d+%9z6YAeZ-YKb zpOZ9?On;t=lID@=m*`bV^T_m9@XIuhO#cv1V5WIw`aQ8g(ma9|9^q$Nj!Z8SOC`-C z(<{XqN%P2bm)It09+@5#`z6gI)3<+$5lQpN^qt}!N%P3`)8ZTQo-#KGgO-$_I$ANI zh$T)L7+1s^j%p-;X4#}+g+qOu%CTO;SLK4c0V3jZNoI^@xvYrwRT5WFt+UF-2yOMu zcCVM$yJxM7s`Z8vHOzIg(=JpTJFPBOR?`A^MfN+2yWL4>?tMIHJ1VCk*S$Xu+H8!o z*C1&%_4)XjDP)i30Nr98+ z5)l9Z(;$-(8XS{okuHC0PZL2DhToyULMfC|KoC$|72CqH+%yd}CWgd>q+VhTM!rpV zhwZ}N*bBxO`62uYHHwKJ`~m(V^@H&YEkdxBo$O}zo%6isOwP>r-(SB0Si=&-(c$6q z@>{jejXmL+>bh#|s0*s$yMgWS*!Dfm^-V!~C>+5fL5mF@X-0oT7<*CTM(X+wcOtQ% z2A1aXK(nQXX|AoV++C&EuzbqzB#uKu)Zp4O{R+>rJt0Edvq(JyQV52=%IOSt3->%`m*n54i4d&(fdPcZCrv z=w%o^Qy&uHnY@4Q2nBr%J^P6*Vg|O&(0kwZ?DzpgcVWA@#gHl#w=&3JC=CVK4AL0r zM-Ote;`kQBSfP66TZ`pBDv~=_L+woz3s=DyF@8dM#+r!j>(3}YCQ5L(lED>B^kW>8 z4EZ(z3Z@v8KnjkR!3;*zn8r0KnV)QjVd;D=8x0=T#D0GwJVR89)pnb&FzA;d{}ee3 zRp!0Pu+ov;-0YZe`EsaD$~l?x4mG~8(b0M0?^+v!k~Em7u5QpNWEiZ)bkHC3;Dv-4 zvQ;{SPZGB-!V?8K>ahZbetL7V>2yzeXr93!eKqubTl#B+kt3u(;_Cb{uFF7%#ir#?3=uPn8!cw7#pIgIK$E%kvh<$Bu2bq4ve0ST zOO%wQ!|j&CJ#^Cm@=b?9=`cNWjKT$vA4Hvbq)t4FovWBS#=?KApbJDUayj(}P)i30 zC$0jCTLJ(8S_G338XS|*HV>2Ueh8B=lqG*n{~4>;Cs6j(O4FdN5UL0wcoGza-nW}+ zvh8LQcGGwe`yjrBC&7ac;6sTM(Sx%vAK!1knPoq}-ai0f0(%TzHkxp^5pF z%LN~DnP-L4qExvFvOGrO7BvcobeRel$Q0$utux1`3!xnjd65K}C<0aQh~vrlb zSsVS$FVt$js6?oRNy6Lt5@p$j7K7HgGOD~_aL~W`38*}* zx1RlXgBb&_KbUo)1HKP!*kRDPqAw~y51M4_VstvNO?{VKkJKY=9=$>L^*2z1E%3ep zP)i30?%vMzBmn>bYLk(G9g~`sO@F3AF@%KJLN!7!wnzmVpb{*mUT`uwNd_h}FeAo#wKHWDVB`scGWRVO&GS7s@g?Pa+jN2^EfFhnwQcmTz_Bp{Hhn5 zENQ04lQE~9s%lQkkQl|{#Q5ba<7De*_WVn}X_COJXsJtdAXi-k3=3x5Tj3}?z*pfYqC-cDNW@saxxM9`%ojFSv~P2XwTG$}IG<|*iA2=S^Twg{2obo_9T2zqcv z#cA|1^fpz^JQbX!uvZPs5Z8mS_aZol0Tul?&(PnR;hg38A}3s~Reu${A)_5CFmQcS z#UL&)beOhQWH{F}YVi+jFCr$x3^AP0P21xUye$I{VwkX-xz1`{g-THnS1}@!>Rjhr zIW7*DOCldf4Y>+zY?9XAeL?d)9gwnsHHt_RYY9hrrywK6SNHlu=4eX%7Z)q=xl8T+TlQ9zj#$B3H4RGU+9utJ8}C$~s#a-H7>0TD-I=p<|?jeg)QO1m~E&*i2qRJ#WG_@s+gP*EiaPTy1=RitZSA(wv#V zR;U?4_-ltji}srm!=p*PD>VrCQ~Rdqu;55{J~YAL2(Qyo87bc_TQ98rV#}utghX#a zRXk0>NropmfSXPmM;=(KUl|NG8tdJqyzTu^614zprYARC;H=f`v29SH4wJQZvRDdD zD?Tjr;lsjeF967tM7KB7g_?13{TioVh~Vr=7d0`lC>QQJt6y(}wDgpyO~ozBwFHDe zdyvT-4)rXGg<7l2FIgdNPMHt7$y=0U+VWvk1pwM9S7+Mpn8`Kk~A(_q74L!c*vaqwxYBDD_zN8 zdQ$L}F|a*cVy?Dtw30uFrnwl87&tZ9xiA4Bt?@g(CfX0719HiL+Atf#3f_{tvj5(% zmCFUAGfVzX)Jc!g8Amy%e2Q1WuTIfm#)^+Z)f&rCPcJf(rr?Ta*neL4Vz{V29A8;& zoU(8`4m2XFDASL{2_fUWszgTREg$ef=IzZDT5UvndCiuYWN2CtYb~bZopC#(U%H2g z4$g4&j_=PwYk-&~h=vF-3;kSWIu?pp)ew+zlDqc9Vu9QjjuK62v589n4c&*djXPAB zX~vf>ESV0|1d^4P%buraUEX8W<&i#?m8mL*-WyMp!-aluvpHzZvQBK$c5R08mZ$j$ zZEC-3UrN|?=_zZ<3s9VnyYWLeUJ#GA-%e^{l5yWPoQ~uFj#g*@2YCrhzuqA=6KRR5 zSVPi%k4sJEe62QU*6JsdPG7hbg71tEi}Lv7Ah@HH|h{RdCL} z$3C9L;Ytua_bc<*_R$?N9T?<%7=s^yAFHf75ph*n^N56)blx1(x^N?6J&rvm63@kV zn;4Zts)_3fu%4Nb*8CQ8#XhKsr>B;bC#Ff5PJBaT%7Hgg% zK>dAb6J74w1wnHH|B~fG(_M3@89=>BBdvt3Qa+WGB(ngY>5+tr*BM4!B*`yEs%HN^qW~OpY=Vf4Ey6k$}K#RJD3+E4axBJ%CrhIW6 zzQi|n(eP#i(MtCX+^gqC!}F3emHJkNjJHq;QwX79-`KX-xgM&SymJ_%29JhH*V~3y z)LRTs8s2we-YsxY=nu-u^TkAa&q7pnyPvAdy0=GCD??aoXNJPn?eTtS-LN$0NvZ(L z^G_>MO_q!V}bB>F)4Ebm`d{f3Uyf+NYxwMPB$!>HWhGL zJ0l4+dJLK{c0%pB?;RBjg*cwf7pd}h)#1AONh-g@ggj0s#YU(=(W)!4$mHYJLsB}o z-xa1Sc#CWZ65gaxuP~Htz;|9-SOL}rMz84AYIRv4|R{M-rl{X6+WeP z*?{P+*s-@}TRyPjvYR&v&Tjkg$!ci=0&x(jpq0fZ z!q-lHNJtqbV)$PYx_o&(t%Ton^`=|jF26hlP3tybYy(@d|$eP&cnH2S&B(v@r|J`B=d=cOYbo%&B=2Xec58}o zPphF-AGe0~_*mw)VhIY?7EX_r2?hRfev8pHy@%_GTMzB<9b_p*ii^@;;zkW>V!z3ViX!MSBcf zS#Q0RY__kzo4=J3e>?X)MCsk;Sj~HbyB=_(pJ@dvxtfpYT(&io9w}{N_=J2MuM+!u zU4;uxEQZ>scmjA5$I7lh7*X-%GubZv%fEA*r@*qYSmQ+CDb3KY9|iz3ESVbVd7 z`SWcoS77M3C%Z}F2HEDV&R5jKfNuZT-PU%2oE|BUn)5E<5e6}Q@N><5jpYwf50ZZg zwKz<3qWo!#k@r{1=T#hHM*KgtjhHCp)Efs@8r{=es4)n48a7&I_+rMC^=*Ko{;@IL z&s;{ZLTTJ?gnsTVCCvtmwt`SHps0-8>44bvn#zI9?|n{qp&a5J$8q_XMx*MN^}?aJ)V$cm7fDuN8KZ zXsu(2=*@*@;q(5PE6bw8$a0oV!{*V;%P*1T@ey9WWoJx2Q+LEGctiZ~>^#plCa-ze z%+?>TQ^d4=ZoPb^uf3?h-;k%tSwtx%h0{N5xC1*ZvA~!#KP8-6>0po~7nMTize`cm zpnGYa1-KUAXxZlko(chwLRX$$`}{BQ5?gFOSjWlUY6h?IKQUBhKIH^uWC$1=#;eg=-u#aE$u^xLTwL z`y==a0;wMY&I%p?nT3S+OXJzs%^f1UWlUzZKWsH0C@;Q5`;TBN>r*K8iWuc$7ICrt z3t@kRIsYBc-x9xpV4&ft2#lfuNuJwwlS`31{5%v98FCF<{2#6ZP{Zla?8~44AvjKI+!^2uP6(*Q ziSYa(_5XJyj=&bq40b>}PYwWaD1ap$U>nCnmH;NAk9M{Dt%z$`6bu^e>NF# z`Vhd}5MX-=1swE6_N##QA2#6J7f@;gBT7bD2?Jnfc$!k{``oDXK>{sOg=O)_MwhfD_Xc&okfivFfsW#rOD`DIwo|%B} zfeDt#i{l23mKWe zR<>b`0|-(-jNu^n$4@2@IQkX_p<%hQ1qs{Cbq0E z>L1hef!!ZqHNQQ6z1)Z}8DN>1TcoavxZzW%0LL~YbW_aN#P+GAs2 zR=Z~-!vbIPqC?W$cTU|nm2|JJ-VD$F;SzfK$uj2%)ibp94<7pYf*|dq7ftx!LfmBJ zXv34lP0VN+okYf|iD|UE^cD53M}OvlGPiN1LDGISKtfYKrb~!*5miZ%l142qqMaqW zu$KL6zcef>;NsR-kEyi~bIH10_<;;~&m!rjxXS`i4+uy$PYtHrs8kPDH;Zk}Z^r4n z<^xCJw*MOn&n=d&+EdvqjfjpH{=QF>=iOaj0;7~3pn&I zae>t!M$%P+F@RC%6pEPlu~_K&{>k~{I4J-qVf6-^AAqRH{qFMoy2|AO(Np{*j0vdwzdoh)JQ;Fchq#(r6RJu|x zHXpfK)~z3_)Qk>7o^jv*z)9zY3G@(}W_BUhuvvD9i79Pgt@EGgj?r`twl;=o#HxMo(*a( z*76K}@W72{ngiSF20&+U(5#B`y-FL8qP@2peO^0nI&0+N_W|XKC25UTWDCD4H9bm* zr)p+W<}lWO!~}sLMaH-TnXrI`Rbccjz6=vIB70%9o5$4sHcbM&&Hc#J{L>c&xulgM z{SG)Hk`&CLKR`-3guft@^79^lVo}r;`flw&#mE%b;In%-rLUV}eLoOr*(7Kon7Bk? zY%0LN)-gVgjCrfm8;8K|NWdJyp}-QYUVD8k1C|~|*fhIH50h2R@IeEeSYwMjJ(tQu zzA&>5+w^;J?3hdj-Xw3>|1S6v=f_@FXSETu>KV#W`)N4!Td+rIw`2{-$+AX1fmtwN z8ErJRLmQOWF;;{^6GdT9oVG-Gvnih4k70?Yi`guFgez8i45DJ?=gu8hvd&oyNk$}8 zk&0O-yQzd4{x+e}kIph7V*NtQ*PneKI__ThW-7@c1W`MUzs<$q?&F+DB<^%TNiqIn z968lJnX_}6@@opO5-z^<*Qa&M7V%#mP1Y9{=4eOrPoBfAOh=d7@A27WqguN#$ZrV$ zdt}S6`#n0~U|>xNf5`9?K8eW_q)N#WDljl1AjqNc-dOhyt*XP@GHRzb%L5F&@jW-!FkatbU4@qZJEKj*_MXcE z+ z`sLi%>0=yw%!Sr)E&3k9zHAe2!_y}RXrdvSPV7MU@BGjsndLN3(kvK<3wCTg*?;W__49<-WuF+mr!M6qbKpcS)7b7M0< zAW~@=JeV$XK`HR8`}JGz6RdjttSlt_iKiU_cSZ7ge1d2Cjd$8x;K|wJ2;hdY(BWD}e;VIw~nWE($m?ejf-Uy~#&XT)w1QQ!tTB05?~Tz}qd(mdCNwZH{t@ zpR?R_^T7CvZ#%rkO#x-YIwyH^wkpOyjXPA;N9Lq=Cw>kL(C2{!WH)X>(#H%MH~DQx|b`NMrB zNc1cqd&jX*6;Ta4p6Ve%CJnGu{wrAnZhIy|3nx#Ku{2R}32PIfH&fL3SYe%d!Pitd zDULXt1ivZ1=AZzC*RkF*SUC%6^E1IkC^l8?S@vl&RTYD1dd)S+=t*XAq+TIbT+LLs zxy}EL``v!!X~*KqC%E-)0ncJn(qSsp5q9c zt4*(Ts+#d<%k%zJ$AS?_1`zvkC8rJ$;&&%;#mAX;V{)-nR#!v=;DL3#kc|7Es| zpn~bfs$d+juP5>llBJ2=mh4{Cpsx-h_%b5(9)E<>?w{>Ea?nPUk3%ghU^{%vWqI+d z0}xv`s`7${$rFmz9W^R&s_EW7j>S}WG`8`tAyJYqA?K2D9U7_m{uHJdlgtTG=(**Y zwFkb6rQpF)ft;m>F8?NYzQRt8@ggE*YH%__fcAmN+Fh_Gl}va7V_`na&byDll9;1k zn8Ra!SSh;`E@vj0g@4!jk^q6Tf_ShckL5QwcrA-5t9lc_d4A1rWUx;J+0j0?5Goi_ z4%>z03W~IL@IlDAEpm+uj+=OB!;6kJec*r3kAQUX+=7HrNwZSwu`741+*7M>Y_<8& z=}82-7~MW2PMSbj{+!1O3Z7WAtT+3lse$^Zo|~6>L`hTP2Dt$V+p&AmiC7eKyI zyNMC=qUR*nVfcveWTT^&(LPtmdXUT-$KL>p4pBmUlPysyJ0?sddWZu##2GLc+C}+3&E!{mr1eXncb|9DxL$9n>h^*k)GttNDaD7Eo*GhKy6c5_B;>_z?jW zGa4#w%pyOE_ar2~zBz9d+|c4B0)J(N$bBp7zim{}p?T#I=f({Ta1&x=R=&#c`Q!@4K<5t&ZzXP9PAOSB@o=-sdx3U&2y%@F`dE{QYH2%%(edNaKsd zEf}Mw*!7!tKf(&8X3r=`z+g%Y=IeGAZ;?Shuj(G&oJC;CGi5+~ zO^+*tkEmdFOYxt0=GV|OfINmnlrqILzecz-`;lm>= z7!R!o$CUw&dUquEN10Aql$5LrN4~-IFMMFn>g3J|a#c6n4@7ayf+cZ&D5WO8Pja{7v)qpW*x7ZdEG| z0|o}3uuE(Y3{0}k0-*iao`GNMOoZFh>9Gj1FkZ+QYpCx^9ln(i!~zRR3dqV<~VZ!3a!|SlThW<;EBhJ1n8g(rZ2Z5bEo-#|aqTX_%$+Iohx=ayLFuU~r%I~YZ|<<}Z8 zY?0f{oZq>(Rmnsxy8H7_t z(_-QH87yD?)~+has8Z3txcl}Z*mOnPcrVg!%s znK=vv%47VB-zFduAlf+zZ}f0V4j5-6>>>kN45R;;9Jc)s$Rms@-=i_RZ3(cZVc1jF zF^S6E8F$rx1bVwFswrSlNF@~2p21<4wuH5E9nxEg$NT2IXlMbj`7GaYHgQ0;E1g>p zYa||{>s38m?E1Ow;}#P2g6%)nj+Fj1N*7a`Fo`xPdS<PTwU9XZ3)AaUWegKa%@{yx-ZO8p zEHpr(S~qy&LiLceT2VjPwC(>nxcOkE)`Sc)4`6Fz#_y7rfTjkSRuu+e8X=K@V6eCy zLY7VWW`uTamUgkwIG7vk_gs)nUKrAj2=;W-pN;rSih7>ymnnA!=fnSUs;oiW?%n99 z_-TIn-`6e&r_+kEqs5>y%nOM5y=+Yvw)RxiAQ?lrOL!KHmhuy*+^cr-bv5(54JR?6 zFdq|J2BAe!bzD|D&f+jd7_Ex>=S#flj~$E_4r%ROtRm7NHvGUD5n|nNF%K&bAFjf` z1G%rI-y(?{H0h*tJ2>cjnRFAy=VE!y!wT%uMho+7ohh58rO$NQguhqdKmRNiaPSPX z@!CLzx57qI%d+=aU}Zb~q6VCv()+Uj`P}8qqI0)sQ!jf)_|AEDeI7ebQn$eSGb~)a z9kPXu(fRcGSO8_?r9MJ#nP(F8AX&_1GvpkFgJyjQ4^bIf5c;y!Vk@Ua+SW7V-q_*Bw!`P%@n{Fl!M zZ;9r|;;H(~^bB1z@96v-)oS034owA0+deH@kkx^@sFkY^s3nXLmA4MKw9{i(b>&C61 zL<@6BBgqWdN*jHnWQbfvHWQ>&Z5LI~${pl1m3I3zLZ96Tx!K8`F8Bxf2N2eODO8|1 z2(o9SJ8UVX;Y@^{*o0thH7qBW+`+$T0Gqq37f5m}x=eltXSq0|9@ebunzKKP`D&{1 zr!l+h6SvoxL~Z1Z>@)!emf&((QN2i zwLDm46e408vEPz|{u#?Wt5_XdNf%Kx#LvVZn3*&WJxIqf!jU30ULp^@(|21FcN6cP z?;l`b`UYve)9vkdmvYF?j#1ojO%TkmOW<+l!KxdhIw6|bL^sgrP7!3^TM-A52OH^G zacz)V>RK?+EK_fQB#B$3TWf2`8T!=AE?-MX(uJWlA_R7dq1;J0vyxdTJi^8&jC<{8 zIm_*a0r@*_I9P#Z0*p~Tq@}UFW_EK_?7T`?aOC;@yyGqMUI#8iQDS#D4cB}EFuue1 z9Dl|PZo*?c3R>jz=bzz&tp&4$eATT^95}Y=1`otbH!ES_n^eYqwGm z?#o|?taDNaDqP54EpmKX8ibgQuqP#BTMh60+4Mdsx_Sq9`&_S0%HuhF983~#!2^mW z1q(}zml4eEVbggeB(Oxx`W)2?Ye_^6-4}nKP>_3q3Ex6y>03KxOznJn%}Y6=xP{+c z5Y2vK2`mhwgy|e@ghF7_`FDnPPTobtw&%S`b7GG z=(3ud6Ga977Le|2pLzaf7C)NC`jqW`H1YAdKh6pEq-6f=;PamM^6SFkLy}9^ReRCf zN^Vhiot2&-wJLwrP92#sn7oDYxNh24?h4@pI6}D6)wa2x%xG0+Zo%=y#=3B9V{3cr zPN-O6lYR6Aoh%l#eY8e_A8ec&jXdW9FbWZVRgHKy`_StwbsbdnO@THZY1r^z7(|^% z0i@GLfrveqQ4lv-aNDbFor~Kgq^Dc&gLT%Q`m(i!N-~543_oL$Jp1>eXIHN9;Q1th z#G}YZa|TrBA6N(V!v-VLiY=|vXlp=78N?aR4+{vx&#AmAQ&LNlCbZwG0Yo&c~%<0jbrN!vV-Qj zpJHH&7SUOPalfQu^c+|N(Kkp?TaIRdI7*91X)IAOthuK=H1w!Viajo0ubw+Fdg7w8Ab!5^2jk&2(8 zb3s*~QowNOWs7N&rdL!kVLCB>Ysh#qKD`?%y3hYClDikVRf0dAF;GT^R0w&vJW7f! zoxQ3$A*ENO&IHc~MU}UL3RWm3%Hw)zUL~#J(Iq5X%HYQbkspFW0F1zILg^;%AxucT z)T-g`SM|-sX~q}@uo^fSz@?r27e`fRYy|B>T=RZd)4#cZRGadREY1Zbh47fR>WB(i zZNG}Xi=s|pC)tDB8c?TDD$cE?Tb%e*!%CFa zRd$tsEzwJYM_e+>8aZm0IOsmnx#`%!plB)S=zHECac2mkI;stc_mo`dzV zbox|6>WlOlUV-&ddY~x576#$P0nN(bI42|AV9u*5Njf=SvY`%kNFi|sE>>@Xz4~4a zliKF)H8?si!seNFzM{y$lh(TzE5xq^>;V7>A|uif2D>=HfcbR@1wF;*^*0}? zeH-dIBB}nItX9^i%%80FPfs~H-C!Gobp0uTfhmOwUcnjO!dkC+c6Mi;oA;e%`?oVU zg}X1we_BwgcKrVwd`AK}OQd=WO(NV6+FnAFRIA(e@2m9#!?r7WHXw##wVtfU&=m00nRu_0bQ{o{ zb(Gk2i(B@*2~vKgrRq7L!?J&9T}pj*>XeJm#TcCtBZ;9nf?lGpSsgY3VMI>Y@PDHr zZD+_qSn3?Wegn^xtGm4MLR}rC{AlV$^K1j``}qkug7T;wqD>_Hlb{$Z(L>y@;MW!xR>Vrq%&m1LUZB`wXd0=~|S{VEqorX`)A;AiX7`PIUOPlk0TMJO!9yOb)kQB-?dEDQ+Ji!WJJER zQk%pTn^w&57-5BqRxEY=$I8(q=c(fF(~S>i8AfCzysfm>P*x!&nW2xh|IDLh?VE%+ z12gQ61(>+zdmOs_mGb>>t^`rt;Zn4l-$q&LbZ@FF|AC7Jr;}d{aXbT!_LcW&Rn3L% z^2L;2xmO8$uzbnu8x_X)o>^#%j&Ga9k({oP5?~-B72(LW+^I0z61Z@&fd3EAO&$ol zT>jaDVMwrl5EB9t4Gb0*7VICytS0;ZZ@S11)K*7T$J!i*q#V#7|HFo1nr;Q>A4{W$ z+%XuM1P+I?Al`7#|wOWii-ZH{nudv?y=tOY%w zT$7gd$j3a#x2wK;{QYK{(f(WJ5O{wCrNYn~!C-MK3oU?si%{KcZ_{zGLX|y>C*86D zNK-^JVh5+|pgm)hwoYA0m#EqPt9Ji`Ri|0Zy+cb{&4azL6>r{rI$Kpfi@0L3xeHiS zbY@aB&g&Xq)JJ6rLQXbI9O0&<%a|hZ)>p7s(3eJcz8NNF0y*?nXoj~`odgJRR4TWb zII3YF1V&__gutksx3KkmV(E6E?MxH%CjF% zjd~sJ1y|^KIa_oI{;H+;pSAr2j;FFuc#>XS8LoH>v9cYZh!D+s@&rI|VS$2x6rdt} zJe2f)3n3NAALsU>5Z_sY7-klI10XUuCr=M5g0T7x)mGBU7be>7ph9@q*Lg1OAPN7_ zN0M^k2w}bHjJbiD20F*}x6Ap*j>Ri)J)Dda)7x^+0lOlgC0&#F>mPv!#RyPTNm#2c zqswN0oYlifx%^}LxuTMM7ugV9eFbm)gKT(J*pD?DCR`1R z)B>c?I^@d7j)QV4vMR)x?+Sa*#Zt$GfChZKpO5&H)b{&Y?qMxcJ2t8Cr+8sXyWiD9 z24~vQGw7Ymi3pa0aznY}BVRKlg1oM zP>U5Lv@ZBnifFcFQmv$myPZYcUcxQn#F@d z!Ji-O_FhXCZW9?mh*=Xy>{(6=>hFyEino*vpI}dzeQ&rcZs#{Old~XlJOqAUsGm6c zLVM`({Lk2JGZ%rHk$6@_>rTt-5&kj~>s|foqgDCEuYuLGI0y{8379R|2;vI#xmQqg z0wgCmjC90zcv0JsE8~j-+eF%cPxw+Uyih2F+1>+qP%<;{cAdl94)blL$Tv>|??S70 z({uxf*cCJ-+wY-C|8Y%`E;=f8=Z1o@g>`q2gu32LHp!~c|9hC=`M9o5#E1wBe$T-QA)rOX|7QP% zQ%IO`Ag8f%Uet~&UaV3)4^;a4-OME-lwoo8v(Rn_7ES8bYNtnu(*3`Q4biR6b zlljDZbdb{B_4T7i4Gk-s&)19bwNRka#H}^^VfuyjCvYT(8D30H1!%K4(+@>DE@kf& zMT=1!vmTB=kDvZC)KF!Q1%ci^_;G`cDdp$mwFk#F5>-|}+V*qfPqe{c}lUd{noMyU<>nJs0Cm6M` zI;b+z;%&D2pJNgOt$zYRD+WH;s%r(CD$# zt(IYEs#RPBDQB8vQKfI}wRI}7+H=Y{X{V-j7@Xgn*RFyAZ8FC4(iqW4GSC9${341< zdBe(+ug}9u+pM+?xLCJ{ZwDZtho1)^IOTgtMFLWK=RqocKuhev>$Y8Cd&V}$rpfRC zDK5(Wj+*#c%8D(ge%*wvqBicHgq;LOC69Y!!4+L2hf)qs^mQhvy)cTA=CfB%PH{&u z6CC5~CSVFvFVPR{rIYZoMLrkOP(?y;SXQowvR4;UHQ3ejxVx^U6dhJm)aC+jP1QT!*)wLK|gEtJJiVht5ln&R4ZA9ax0~l zr0lV)wxo%=a`xP3CPZzvzbbY*m}fw>P@|x*2bsVgI6ar$k{whQMoN5J!CJ$Nd5IEY zlh5p0>01TkyiX9{_qPjIMk6`QlMROTAuj7KN{d+k%xBT1QiRHd!AG{o8KkLLLb}WY z2E4zb|0#Zt9AiRf5Ye~jz%;V~} z@a;wLD8WVf(C3oMa}|k~pi4YGsmf_!D8pzlcZ|<&zkdwqXghi5#*!>-igqZA>{We- zH4oS}EQ!1m?h?FpNZ?f-5+i<->hcM_@wi3@Qebxtg2T-NsvR36S5^05uLz8?0ZaYt zfn)am4x2`XQ=Y$bou_7qF*tM7{eVJPSZrR=JxW3V-5{B~!T^EY1ST1y;Ni%$YfqT7}XryhqVELkq%RK4Fo`bN$dU@0e| zga$#;qSHvR%Gzhqf^*oiZXxkos?X&7tJN5Xy~WQJ1Qto)uCqUX{IMp9OKuT2f` z@Bty1(fDdlEh=MPSK3m8OI)}YB&cZuVD0Fr;QtX@XCAqXH0n`VAyI*WZU^Fk0Ky%K zux2QCJBFhq=LDZGAD?erJ#9VXcXx7UvsaSrf7otgd=?+9yT;;kIaro8amX}9E3{e2 z@M~8S7!>)GA=JsH1Wu**OAe9tiQ@VNV}Lx|lV&0p_)berpHuD##u^TM*lelFIA@6? zr)*}fqDwanOWF?~UbLYW2Lh8qu!;8deKc(|Ajl>b+>qL&xLieSd_9>nD(>lc(ITac zw<^#+t#>Mh#>sdbAWC-sx@?{@$vYK26<dxSdiB<`Z!S=&!8gc@W6tKq3fh*r($WT26Bh zpgWOZURFw8imxDrO(DV1frAG=wv<8f{oqv&6#Y4_OF+wfb>FYm_tr>tWWh*J4_Q2| zSXI$oe)WAX6>M?85$YAq601;ngbEQ(7gJe@vF~gvPrcf8pzW?0Zb;G66aqU`Mu`r~ zDh=?Vcq>o$PPHgp#QWHec5Cmr*-q;@)xR&!ATJ4F3i?2L;eImC27Ll2A4PsxKML2 zM5rbUe|3wu4LevAo2i=a8t6S>tMM^0k1`#4Qgj(G1=20@N~8BLyfsDY zfk$YNh-b0aBNF49)L*cA3V4UGmgYq}zvzq?c#%r;tb$l-$98}=#d=}JKW?A%05D;a z4{Q5^_s-V1T+=&~vgO!3%5LeF8SAagHVlD!4GcEY$r)}G7yO!oQ=CCfXWpP2I||e) z?5PWfrj4mKpdjh;2kC63xMjgKun(tO)e%lW&EZ=L_84ND9Kc&Eo#EVdt9yL? zWpHJ=VBEzxwgDJShs^THfHgu0f=j(OZjsiXh9k`XqIVDS6@jdBH3$+B*>JC%ju^CE zesu~W6^RrWjI{RU3t$~6v(BRG5nJZgB&xEW#28^-CMhFTV6vR-?&)*R#5Wv#tqQ$x z%K3;wcpCfaX<4Rgr9ay%#ngC1P}fTcVAjch;u@PDE+)rHZ^$OIJrJzB!Lp>^{XPbZ zP@#H|@_!&Bdd{U8_1`i~%YRE*22%g4nC}VlRm=>gN^%NlLdd?8mn`RXq;ynUN07K6 zII0XpEL4$uGUo7JX9fEUn|eONlk8Q&&;)-GLSXiOX-`T<2Gaf+QL>f%Dj6620^16r zYBt8&i1LwU>LIGpMm+IJ=xCZtZhBmLJB@4atSXK~yHH$EDV?yc_dxz^7dMwlCH_}s z&aWY+9W$1Vqr^H&_^SPnQA4K22a+K?7>CY$?O`nQ{a!~?y38eeAYnb<4 zcN^e zOb0$xvHe^Op!@#Dg5VmL-`UgzEF<7GtE$X7Zig_&8u4b)`5POrK`Hu+k~v?b?&q>m zN7t=GA%U9(mQN}OnTm7XU+rEP7p0__Bie|T2bAYOPcaupnjwkWSF-^NFU zvt4UOD#yjD!kY+r=>AtxT%5x3zN}Mb%rg~5L)mzay0qux^srX2SV^f9So@;AjZJheBoXEjwP;b_e~KVtE3_X+=@ zV;7M=SLuB{J5q$sPD)2w+?;OZzwgS2E{Yu!>hgVi#!)0?yksZ5r$A;Edb-kUtR%*| zMK9J4Hj(4m3(71bT7M>4ePU~*iJ0J#j2DgYt&fe-UmK541P-+WMk3X+vFQ?jZvz%V zzs4a*`pnKYNG7C(yKgdZt+)1(GDNMc7}lokfO^O3V8Ptji}j$mOb$SBENL<|>>-h5M+=T>ItcqF&PJ`k7;qI*3s z*Gg2jFp!cuMT?7Si|nidetvJ?cc=-deA+sE*1JPP3Z_@0_wqGj_T?+a^H_dDsjL-H zNzk?IXO7kbC=XEi41EvHgJWWb61^`6)RKW{s^n(7%)GqkC&SZ0XRqB4`pEK(R`SS9lRP30xj1{r*n zl6zq+^%80W1sQ6*7b0rPYS;^Z7XDnN^JUGf`~G7I{V})Yt(Gz3q{D*X<-V*m?69lBZfo=(0WFSFE zyCz5*)Zrkc{h=9Q2+8k7Ld@`Ws$tRY!)J0!?U#OBl4`B@L%Ea;_cH|E;#R^7(zXhf z3&1(x4C;Lrl(d%8g0AJ3d$y<9R*d=Klexe<xGa42Jgls6wmw*%}fE7wt8 zZ~qpdi^Wg7^ESvJK0Up>(rcSxR1yc?jG0_m;eVrM+nWueN}I6Un?Wk9z@^*$9yHE> zxOd^Br)uTfKis?e{n>yhV2mBO?(A!q)k4w6>z1#6x9)EbLjuSE%45MPK+?3PTq4C|n~+68r1o8%YQTcco*1uS9gvg*9zj8(6J#;(0FH~4=B6^%R=FBs%Pk8 z*ob>nOFu?|DtNq|ftF2yBBl5fyi#O2iBcV8U;aN{{Tc7lB(`_v=bpIkao05Itta^= z1n!HO+aJ_Uw+JHnCd}EN^s05$2=c`{(lT%9X$muiKV9Tk`%iS~OpnUP7^#jDbu+tV zzIFGnOcE+tr)1}SI{RJD8b?aK_Kpv9(grr#e}>DdQVTRkAmu1#)pLb$lqn|;=nX^- zR2lKr(&H5J{Vq?u2h2?qUoopF`GO+h$+NPb6TEJI`0p>&8mk`i)^d&(uV>uTJp6}e zQuL%FWdFGfl>ZxMlgh~f=`mE1I~YsWg|(G>os@b>^m+urqN0$@R=!`1xiwGO3H|T3 zU?+dtqCr$HQX&00f1zGSR$I0~%H4eGW$Nel&%4Rb=a&)$hM;xh|ipb^%w)NR60FDcN@+rU=g{Vp>9OLgCW>$4k8I+!QaV&oSlIto@idJ)``*p@b}?%{BJ|zRC%jdu&Wy~w zBv#)flD?~ERi_aHPi;NAsFcY}Z|QfL@zOVYQ+&teH;)RLL5GgxweM#DeDN&C$5>*M z0Rk(<+)_p4S!8h;gmj0{Tpq@E&r8p?xM?VseWXT_6fgUVcKH7GE?77lZ>%GwP7r`q}4K( z{FjJ0yxok6FQ>|mld^PI?~KI&fjOJ(UQxJ`iA6jRdwPGOs>;4JwWk5(<^l=h9@59^ zl3RKrmeYm3aD}4Z1oe_Jg2<#bi~mOrqEfA4?hZ(B!EL!MXwRO#DX#qmBf3rU+z6BQ z#8f>Siri`15isS>bpuU%V*_PQQ!xA#>IaIu0MvAJf*FEUNQ_TGmYgoYz`J&==sW94YoEc*-?)pVbp0Fafx`{ z+Yr>=j$5PMHv8Pkwqn+!RtR_et9hbuQIeikPk$4=I8qsXp(a7iAP zu;I%&a`WkAH!`E=tLz!BF#x9UR}9y;rG@FU)I=2PKs5pyHE9!eIY!;plxD~&*Cn~^ zwXrjJI0HEjCnHe*B&#$l99K!tV1Mx=(VP-Zy!w;)HHez^G} zv`qOr(75ep<49pd@u*wkbAe$0_|dLui8<_Kwt7vZ5b?powUWA)uP3tkUWKQHAZg%q z;(5YY8!fV6I;jQ98896iL;mig4slNu-sm6(;Lh`+fovYp=$I+O(f+Fs{W-IAc|U$D zNdx_NfU_linEhfs6u|&3FBJWJ&v6))1^N6P$pC#+Jz~~msaN`)LsuyJpqaAyGI7$14`WItP2tZE#<-DaKKnLY_(;Y3K-4=Q`_M^ur5_dd zQbrRl53`8*)Y+JJzhX_f9!dh}p~3~D)h99%MR8rrhAbx=8#?TGbV&JVZUJ$|gyr@l zP2%_dbd1<0Fb4kcYwyI%lf$~P zWeWQ^_vF6mgm5MZ^JVE3^N3q{F)-2P_`gY)m2a}hIenuib2VvjAJI68&4#ce`aA#9 z*|tAJp2+p<#;wiA4CJWWJ5nT;-p%?OcaU9)Hb^$Wqd|JY9CJLLH~23_hjvLz+HGg; z#|pwWDaIC!-_%0_qS`vp=YaGolurjx0(seX6nW(Cm|con|j(Y8S{3WaX^^#({!#fpkLtgHRjFFU++R&^Qu z*Ry@O1Rm!7w^lzdwJYCr1*_~PfyxbOnXu|tlcJx~Fnwc^F~A&unVrhv)A*h2xA(OQ zh;~d&t3W9!(zyZnqaUAJTpTyr?BDek+O22X#t9DT7FH*7LqxNiiuh-wjoitqs=uCq z4q9d-HcHVeX+OU#`*dF5XWfF&dN+fqf(@5@;*0hvIvu6i>CdXYCrD+IB8snL zxl=bQ$-x{kbz%k9yuIpsNG|Qc?c*W~hdspTrInqA(eA%M^coOY-Gqb14gAx=KY9Sn7@bLqb8JD9BELKKjU#1T8f6X=bJCHur(EruPonivI;^jO%0Z$u#G?gn~1;N4nh6+B9wGB3#O ztQG74BljPkcD)>@ek$A2A7;7*_3=n;f7_!cRWqN}2x;D_+=4#0)KZh#_bI)#RsP#= zA@eBtif`Js_=JVicb_k}Gi2g&#I>b1K#!mm(LyiF-&Pp9?)@liOSAXEH!E^|QCvMP zOt_*s5MnP8>(Us)uQfm^l5>f6tDF0=|7;TqeD`8Xvv~-n2!HlDj$lXH5&RLN^^wbn z@Jh>{m%yG|O&*H<$eR8u+#|k4nnkOAjqug&$F}ytw$!{rd~n0Zn607yaftBasAm}p$C^3UXkoJvFPT#??1P~+W)rXyps$` z089O*b8o22+Khalr^jx7@-j*=V)!cR_kb{={;b1ZML^o1Z;&eSg^eqb68n+XLc+qn zv0p3H9t=Ym?WK{1DvU*<2SuJ6N!#7%i*z<5$Qgy~8k5N?eM0?8GH{^GO82+{b@R6Pu&}qmV&Yso`r!5&-r0% zHH^7IA8_pP8j90IREE|4ymX3SK+|G4pII$#f|sJ zOxV<70V=Axs8PYKBl^Wchrvnvhv3Ls%lTN@xfib&z9qG{vk&~c7SP8+WlMjCcvl)) z&0`Kn>e}FOHk+DAYkgXOn%4iu+f&!mzQ6zLBWf#QDHdFNd!e$ct4KSSur*tDgNf;V zhGGcalgxN;M{o$lPGpT$?DziwZ9tO0tZ(+E7t@*^IfgHPBg4&V^forut$PnSSJi*< z`>E$K5D%oW&MpOx8xScsa2Oqtin$l#_1gVf-LPq-zttcFitR8}7VWwo39N+5F&!ms zkIdIfMiD73Ob@vaRH3TZn-;8#WDSITKZCRchxXELITy^!O!2I7yPg2*jN##)%nBE%a zmT?ZHB4%)2ppde(+JLobxbTOn_EDUWpT|Xkl0(UJ74kgEk-&2GtdiBUP#KqSxrjN; zbK`6ofh}PC7x$-TE^!4{Gogz-saZQ+t%mFDCE;mYs=!?JpUszWGh^N=Lqnl}DclyA ze9Di_yPrayKbYX^92;92iYW1Ym%SBCumvoB!OW|Vn5|SkVc`VVDmVBzf&Kw^-UDM= zj?W3?egIHQ2MDXU9opIf006QAlc6LWlLK8je+hh4)w%z^JIlSf8MX-l1`x&o5=bTi z!~lb!*?_<#P{QJ{2se|PWMpP;oCQc1tG2YZ)-DgbVC`m?wAN~CVG>OhyP@r)+S3lBd^M5~n>nC`mirk!hFQ_*2W ze~y@m&Wd0~q^qL3B4WjRqcI~LwGx52)oEfqX~s+=Wn#0(NChH2X5>gJ6HiqHyNp=M ztgh(o4#bV#KvdA^sHuo9nUqC1)}&15vujn$)OGKI6S zzP9GdnzeyW^JvBEG-4*b-O3~*=B8-Oe`H#0CA(|8lSXIEtUZ=AdV9@e?PmG8*ZyiX zq6w9pOw(^LjvBQwBhg*Ez2gQml2*yhsz@8brWilV#JWs9a{#NSTpLGMetI9S^hKLmrxZsVG@Ir! zdB*OjG@r?pws!AqnSj;;v<0+Kr_0D+h}NP~1yc#mY=@7;A;!!+>R4@iXfZ9(X%Srk zt8~G*8dVlp&4yEHIg{JGF#{iCe=4sGjW_H1W&1o-O#z*%s0OyOIf+`ef@bXwBi#cd zu3&P2A^1;ap%8hQ#=?WORdl6JD`_>8cjCTEbzmuN*&aEf7l4QrV6UZhrL=~E;HHS1 zsdRPT8{~4EB|WXl?Al~y5}nP-q?J@@V_vB_vMOE6qzXp_2Oes$b=L?+f3A)uqUnv} zbTi`89%`mdI@Qx=+a^1Vq?t&2s6`N{r>!>8HY09&C}gj-g6M&o8;s;)jkd#kYI>6v zA}bv=QyRSrd?n4^m?0uEnSx5!7CE;FC&fIVopuSc?PgkfX+)$rdj*r%+0kN)BNXJJ zeY8&O>}T?i$r6!R6!8#`e;bL;5b_NWQYQ3!5FSx!(>tWo^>i4YbOE;E~MM*G!qed{8f9u9f)J$u16e~>{ z9fyfieW|n=4+ukR^lGN5l1wHYjn#&tDWuNVLa25#?Y9B_IgjY`TV4KikLlmKr`2C+ z)^ykS15NQhvAZGOchrbw%w;ti-Gmc5%~T{A&FRNm%o%Q`TLhoC=97Rty*`;V`Vhcx zgm#UT;Du>Pfp+s*e;`!IG6=qj-mKFJx^1E^r4w|H(Wpvqh4MxzY%x+j5LczQp(NN= zO*Qn{tin-3g^;aAFOGXVy+b(3J0}prwo3m60i;6UQgbTDa@%OdVs<3}kvr+#I-R8V zF!?Hr!`MFiKArBMQ=*WCCUBhtdB0A#)7?yUuM`Z68_X^%e`$wvd!{3|uhIvZHdkK6 zX>IKF;~^#}H^yT?f?9IxP|wHd6Q%Sq z>SwBcMXBsZd)i2Y{-^Ti7En~_)5w45X4=f-X_*iZ?4P0gOX)s(0A(p5mkY~R&fh%r zIeJjQeIEWAe>eI%Oq`TVZ_jyn(PRwbXDF-Fy)?k21Ogg8#1wc%LF&7}ZZ03GG$aDx zQg!}_PG6u$A!8u0|N0FFt2BBHA8{j%%AE4hmjpLe^ktNWRHh@9bMNxXmZI7Et8`94 zKaR|6B?_e7cZnt76-BiPjR(`ZiP=O>~;ax1%yRp}ZCkeV4u`boG7V%Po_s^M?ZD zN9b^^M13xeGc^?Rod1;DAJemf+y6m++gr{_g$;ug(&0tGfuRQyTEK+-?ap9P7(Ab+GSd(%TNibm#n`WuXe z9sy}FuU-%RgYFla`KQ!6)Yuy{)94*uvd#N4e>jO@FiH2wYyd+J1CXj1b4aO`XtQ#C zfrlMJ!}qcnG$ft8Ihqrl9(IeK;$Bt@`&n5!RW8YOE+b9V_<}IHv);p{?9o~0DMF!8 z^wpQ*9TT#_XnVoaQ5ARw(-oJ7qjDJ%LTFq;&K1}@xx9pD@~nKSO4$ zykjeLd3xxyi(+c zRCByH-RI#e;eYI7Ocu^m^wp+^F-wSre>D^G?nt3o#p?tF#)*YvN+%kEZX+fGzWI2> z%vlSg#XOr;Kgyavo{6QSaB;ugdemsVQRfXJ;1=efIxREhPgrSyA2t0(qR$2eWIej_ zNvG}I$OBu@_l7L%NTye13?g%ynm5(&4(&R$d1rl7sQJ+D_U4_3wrp>0_HZ*=e>-mC zO(TtSjcA-}WaG?R>;X0B8GUfgOG*Jy`c~d1Vj~2y=^P(OPz7>}GN5 zxN9VS3%^yE<#rgUR^vO6e-1FYrd#Ze%ERxlivZ>-MpnWcrKXH7b9XYzv|y6koDtG@ z^1FqCF-}cMTlMXYEiJhgf!`-DP#7bWqqXTOjo%LsEWAW(HB%|0+iZ$nw{r3{Rh$O+`4E3t=MOTbAlL3)n*wV!7K0DSHuR;1_suFse{+9>hd<7r5K2HX zb!U1zk@G>Ja({!URiEN}1nytap4x_JcS|B|$^`KlAazO(M5d7B9^lUkoX=sW zvPF`Cy*{t={d`(bkHj*m=uvs&TOWx)g{?*cT0~fH80&jc z2$)P5G5cmNW<`!bUA4`VqC@|W^Aja-%C9lapFH3euT&Y+M)IP;ROo5NLLx`4=w8um zXj|bMI-ln!ZLg45IH(^518DAEhrh|+(n;l~Vbq#f;Xad-!{H-pBk= z8bz0%L?>Y-(SH2UUdPZeca-AJOd^duIi`*HF=nJjD--LKtwAJd!sGnC@~+L_nWyIO zvXXwGcE2!yUt^3L)4+9oN6Lz2(xz?MpUO)`{+Z6tioQcj7zs;cW!YeF_3$tGSE4rm z+C}2uw1#UPf5hK;EI)NX-8)f9t+;JTc@xSQEG`?lmW}iniG&$TNwYNCA1ePoFW>}_ z5ExeZ4;a9c$29(<&d-U0t_yA3U`&@+j=2^tMjzV$3;$K1z6d8yC;J3Zk&Y(A6Z=5= zJO4xH=NZGt`u~R?tNaog8F}Z>7_(C5tHgC)tZy`Xf8cbvAx1md&R*bQonKa{U>@1k z1G9Fjih@*u&gw)h0!a1v616Brr4FL;?UA`;j$}ISsGCMzf=6=hNQ4>P>g8merxF`1Ke%lCnl=qr+jW=Gu9O$bhV3%p#eROpIdS>&M>`)!GkWq;w%FOy))Y@jUFmAOhK z$`=ZXh(6nB&Nm8*mv>NE^=^7n{VGu>lBplgc|*gt{5 zSdvMzOWcXp+7v*0of6ckR9RneV^IjDDjSd_qlu%|5hS2>MFz>qua*mjGUXcOT3y+w ze_%**MMSK5lt%bHjMc={JeoRV;%7Hg-jUnd^XIkc-&()ZA5G+!$Cgh2(j}>-HJXBX z$&DO~fDlnFMi)RlB#k+gemG-1yfXYuI&0pr$4)N34M=F!g6-P zK^UwyHX?~*sS|@_G9FEs{)q6yUQ{+Ie=eE%w;D-*SJI06BUY!`0ip9IJe+SUe{-Ee zdtV}L93LZZhq(Q@2=ASOclfGP{HBXMfJ_-Vf&pTefI+<#S2b;!c!!ykD@gG!Qe`Pc ze36F#Sm`F3au{!=L|VDmm8EG}D$mlqEL|QBWofB*S(a)~sn1jm(p3);;wRKk-n~Oq zA8xJ6Qqur!sSYi#%71Uee{J3!f8L#0+A~1mEFG}_LPKSWr%JM2c1K7M>uer z-j${I4$xf#^noGzP&nuc_?!cD&qMS{rl8yBeuzHHbc)aUT;lyS(_k&J#ZMP?pYT>FJ=WfA~J^e@E`ui2dms zvh;&G0ay;uXKc`Nm-DcEdm>9e5lF{?^fQU%7f8-gP@n1^1>5l;{qioF1K?jvV0S;2 z4$*JJ1N6UV13&|0P=nMye=SSTouZk7mUz$eHa(D|9V`)0B@*flKGzUEANG|T^1d)Y zf6UTfv-EedcOF7#>0hU)EH9|d#)Yr>@NpsNa@A?&norHLa?gb`K3BQsJS-$F*QBUH zO_J3L$lAitsI{r3z}98FAj_AB>$JORhM-r*i?Y0QZ~ySqJ}HV%b(CvD8r69? zXKK0qd7m>J5Jy&dyM>_NeC=8LwL!c-$eZ_;amygL;;eI2EOTe`Y~d*iSpnLm&jz$~>U^T)~olxCvGs5i81_Rl$;gPxF-sN&!LWG(R>% z3(hHtL8pRR$!Y#_IH>2TmH1pCA*G%tc15+Xq-qSIbA^O*ukI0=r}^tcd_ElVK~kTy z8Y+D%%ioq+INU1Y z+Oev&pIqEpeU93Pl)2#pAwbN_DhpbjkI-ddM|Jz4vN)?;F`z6PR0248WtnniR#}7H z(s0P(-Oyg9ti|%xSWvOByq)pYus5qTe@>`Pe=cuxQ~_-B@bclf=lcOJrbnou!#*7^Z5aN`&UoVydKToDVq9s81@_IR~BR=_91t zAM)>dm2Ow*UX|`6dWq^(s#>`Eied7Ke+Fq7jgnRr7GMH=F`mP;sR+=Md7xpmRV9BE_lA&I4Q}#Oe+L~f2Re*v_~jI zA10k#@tDJ)NC8QAYpQOJ;Gg;`OIE>`-WlCty7o|$4e~gGb z0ZxKQLv9oY7XVRSXZ4z^Nz(kM;QKam2t7%p`8H)fFTcgV+(x-=Cb^;Vb1FaYRQZMc zZUZ`H0n5*e|2+r4Qc8x&U4Zj~jq_X{M4D-NjNTa+D=M- zcednUESgQS3}zW!9}%Ytwo*z)e>a5nN@?WZh}Y;7mq<{)?gDuvF>$hBVv)^++>9tu zJZos1oFdj?e+NX{M@}*!a};{%1IFvY@w;I1dvFLESNaqv-Urh@fOve0rqRuu+!to+4ayn?SQ>7)&X>^6tOG}-VROzgyWzN;K+_{FTob^=gyp96SgH+>; zP_6R>t#E#fRyzA>mGc3*()lA=?R=50a{i0zTuf_Ri)pPZK=2Pz^UisA!xyaW`6iVE1Jh4K(}o1mg7 z_(a7Az7R!3MMUcVd`Z@{w1xhD>AC0o&UfD5Ip=%qwfi3eaI9)qxc<^hH?4g~eXkX} z$WDL7>m&8CzWS#6n427Q5|-zMz zGKIO@tsPcN!bC0`4I2+LCnHz`8qU+IhZS7hbj0Qykl|r)Hf* z+)f*43}A(bH^{EjO4^e($di*<7|p`0g`O54q~Z$UhSw9m{%k=MS**fpk#-D?Z+0&- zu|~o4+&onf$BBRySgUa4lo6aDMY}E{3Q1l%8D=CM<)$yujy*q!ldw*9Po{smPDZ!{ zu|B_as=^!^yS_K$CbFJ=w&e{3u_15WX$p&`PYDBW;f1tfF+0PIT*;j5Z z4lgahHYqgpT|3?y!09+c;pjJc$iSJ@HcxoEo1_EIl7#HU*%Qh{*CiRxP8!%m&)I3- z>)L~ApG_@2>S|j_YOonwD$#$1b9u-6EGLmo+h@`bRzFjwda8su4^feJJ}bo(3=M2! z(hbT&f)$~5s#Ic-FGNoO7vOCSW1I!pqZPgRFvgfX3}aiu%48^FLelC*s$io}Zdd=* zPMhj78*r#hX;teQuvV{W?aC&DxJWG8jzsY~7OIGW)I^VJ^$iTt{e6F~6mQ#$4JaHw zWm*?Ykyx8XMuP0oT6-6D$ON$?Z|zQMHD1Kq+(d%uPVF)VnDUi&a?rb^gC`h^q9-(^ ztkDtgz&itYJKjao1Xn~noi?vw`PRubH>D?O-j2SH&ikj zH`3}2l6wqlUA$Ol>P*}$HK<2w)-4L5X*n6Vjh>;%AU-GLpT&Re3`0Jfbt9cODKErV zdvK>@!snT4rO6n?7p0YK$6agyp1Z!Qt-ZZiKff#`%*9veKaLYl-z6K|ovDOt#oG$A zio%*HZrPhDwfEp&(dMg6=xplk&R~bk3DYI?K{I%8FLH8lm}PZ5U}Vt3A>*`NF?%q7 z=kCk*pL{7E&D($R0N0u``tq50h)CLI!QR1YQ$Ky%DPE=^zJ^DH%h&0RqE@G7`}*v( z9p7YIy7hgNQ7i7Xrv|fy%2eFmUu>HNgGxvYd~1rZ>7Mjh0FUC^3gufiZw#+B@m+<+ zal#TF({{D*1#kf0my&kySYD;V{tp7!had97kW0LSLu7vtPl?O+;YSo3OSl=X{6yx8 zefVkd#%eJo9{>4-jm-mTcV~VS`~{uT=4KP|x|HkH^-1Nbky-jZe^UD7bA#!ZgWZ}GbTeuHNx%@W0;G2<-p2f2BFR8Y+({!Dk!Nf|d4 zp^|@*zGr`Xh4vK0U&TGY#NVizn`usQ$}#bGjt!D>X_xwYtf5D}sbPka|AChR?1TR- z*8F@KlN&+z{aeAerR!ivEZO79|KOEMyo~=+wC8rXJK1~qq8JxlN?#_&<_(m`}UVE04Vo5)=)QYwNE8S&ZoV9;bF=PfjXnPr5~^sRiLD1XZn?FO&;-(O$Q0sF1k8a=eYwFF5hF2i2i!aX>9n9Ian^0vn*w*qu4z9^sd5*QzXpR zX_I&&V@hsN%gI|c@|KLBX-{!8ogMV-`1oa2O(i2#`&lI$&7$4f3Bw1kGRuOYRmxTx;P^hj&dE@pI=(EOcpck`-fK411_r8)&uuEv zdW8?Ra!!V{8Rc{5$)gP*3>F|CY#Q>prXinq0DPpc!6AH>ZzR^p^A&_k8l&5`h069~ z{))X=*t8dm!h5keRK6EWhH=C_kiU7T$C3GS=5op;cmK7GqgWR0XdJ@A9F~t_MYOSJ z7)=^onZvQwt^Ak6@xwTA2#az!WjBA;tjM8lH=227K7Wg%Icyw3NA%1goD=QbkBUA1 zIVRTR6b_Z;kPVgRuU`P}jp&5Jd+wR)Rid*r$ zkZ}NyHEF77#L(;vac~X~ig$k>E^_=v#2nR9LuM!tE`%bSr(9V=$vDsA4kj_eikw##vXKv!zx3v@NiSKXpzxV{R}M{!S8eUQ}uHP z%_{DjJ=M=^i(fdnr6NXIt65v=dt0=%@@92Ht$F=x-Nh8(Z?R@}cS(ODs4CfxM#?0> z)h~|VU-#nG9Ftf1a;joCV~3}-&E?@5WzsO!IjREDiU)CVG#V=JiTZ0)u&b;_&F(61 zt;nf)wG};G!|ITnTFA7?sU^FS5l3{28zM%COZC-{_t0lggbX@jR4paluv$iU{+I;& z(GaSrQAbD2vIk*ABb9&tkkLhVSLW0T2J`98J($biB4M;7sqLVLmW{BejNuid<>6k_%jYf0%d=M5%@0+SLG=utRu`+QG`w0}qv5sc1`TgiBN{%Sp3v|K^`v?h zP(M;X)%dgOIf1@weAoGBs}>CdD(t(_cZ`1^Q^1ZBafr9_nU!ie<#QoL& z1%hix96t3Hmfb5+_dlF#V3~o=S1@~wb6>zfxn4M3|9AEO?FNS%1&pzZPfNfWjtavV zV~wAd#=zyIdJS_8T%pwBG4_h8>G_dJWcp{~XK1y|nMi*=u1SucS@ZJ^+&_jZrzLVp zM1`InL)r8+2KH&HUy5NfP(7_RI(cS|#@IC9AR4F1Zl0hsPbRBz7$vLw3Wqt+aPKIF zsJMsx4i#46Hbb?%3O}jDnd3CvDo{ZJTe{IQzEM`XAui8vyo@8p*rChVrwfD}DdoE} zpGpTe6!l}~+k27t7-w)C=qBA(?q5hhUdCbI3etUyirv8$|0)7%J*w0O1XVv~sU&9m z)?tosGv@j(z&u|J)xLhz_%6jE{w~z|FT{L*91Hvo7Wxwi`3JQezaBgM{|8V@2MF_% zQ9_g6fM|bLYwO|0yH)Vib@511@kS5@MNkmDOt;f*GJ^)Bl7ZbJsQ6gS;nH)y#vH%OvXX_=`c_M)UotQ*oK zE%9bsS}$l*aBDk}b$44*TdKKf=tVO1RPNE(*;#)NHny2H^`H4xM{5>rTYBrW#l(buXk=59e`jQxlJQSsn@Oz~ zzVl&zu_6WqCU0a{`dY@Jf8MyEAS+^+{l3PJlZgE$PWy~X{M>(!g_cyhW9W>ml_3+= z(_d(p%PhYwQ^WfzR@s5T{L){8|M2paKw)Y5%7KH45{f807{TZ$hEQ=(!dPBS2@D?c zE1|+ok$+}@E2g-r%7KKkxO9u$1 zr-P4^tb$U1d|T&LKc6M}$~VfxcI?D?G`Du#$dYB}vDm57m+hpjW98{QrX)>zEnnL= zk#tqvt0Zn&x3ZK+3yf`rEh%eDp>u(*ERf3Xvcv;MJIcB--jCA3ItFY7Mqxk;tM)(N zm2CNy4#+P*efN8v?|kR{&;Ojyue|%YYee)uVGFu{_~3&Fwmr}|peIfn>A}WmV`8YW zwJ~9(GGUz%E@d}HhxDXvv^HjjBPl%-FTV&8U)A#{D z2|;Rqzm>}-j62PwA!wDA9c~}a>Vrw6{cKjxWQ=TkZ`yYBWKtoopk=4@GkSYcPY<{6 z9XMqq9EBBxkNH&n`fk6 zU5SKY+q?C&E>F3&e6yK$jBHv@whv)pd(ujOoW_OQcP_Xc!Ygkv)24HqpnHPX(f7I< z&NsPFcSgEw+ei&0vAyN6AWyL6aDbN3GL;mn7PS5Up|?V{DlMn#00n4q75S(>Kz^#? zuayB(X%T;|f;)A&YyHNJ8wCx|d%>bZx5uP2O{<*`EB2&o`yEEj_Ll2xUSDi`7^duh z+hN1$N$NHLUmI*GlO+eY2j~V`$5zk;1+@lkGiLG6@s{*|tIlYmL@U6D&XAe zV9T+Y)(Fr>+QeFH7PNHMoPxln+G){$UD>QI&s3;GrB3$rBGcYsW}%st9SzXU?uDYb zpgsun*9Bv<<7hiy{1&>E_XC+rW-6}G9fB0o-pRKMP&YL%qAuzYbnji#JK7)?WzB&c zTSD8=Y;Vv8EyLE*mZK%Cw4yqYxpN=vjpl{1O#^|;z2WsknncYyV-_f(6iuIcmx<{oGjINfMHc9I#<_m{eXC4^e z%O~lAcD*-N_;@|bSDiwQHqS2HHzBAVImH|rEpcK`F<}YXIuAlXtm)J%kmo=Ty_TAt#(BKYp*x+z55n?d6L`ymWe{Y)S%%UIWmjTm%oTj8orwAIa zDA%qxoyj>6VdyD^EGCDU%DZ^GPo)eY8C4wXR>&#w0oKgeeg=TV7h>KQJl4&SJV&D{ zou&H`Rk_Td?m%}1Q@y<`_DARgtkHudaq>0?N3zygeSo?0Ly(h5TDB3OALXoamOczQ zgYrT+2`ttfpoi(lSjdlmm#$T2lJ18_r08K1TaF$UlxD#!?y=UlZ(^ySu0eg!~-+JnQlaL6L=B zxWLW}yz?TGk7Jc|T^^iQ)nA}b@!BUi*W8ywJr$s*m~30<7ukS+sJtB5^p{+o{$)@; zz|}QiTgjYba9$74r&&T1jfslN!;E_~A&WQ78k#Rcv>_c(8N9JM-JFi2zM6MUN*~om z^fQJwU>Ir5(Nl+!5#7O$p=~JN+&`itQu=eL4O%8^VWTsu zAzVlKESF6pMK)=FE6#(>G_Ex?(?)b>nYxe@26>C7XQ5g#j$tr)TyeWLl(kZz0VkWY znFeiHEw=H+c9dV{P&OIWnr)00HIX|!#iOOdHY&NNIo*|T;E=LmtvGSmv`t4F zah!}DZ7)(}8?$AxP@XQ4+nKRkHj=7OO|W;YA^6I~3FYR01F`oGxz-wBKxsJ}=FznT zE{W@wFKyLq!;ntVOvh$xpD_VIaNw_?PMyZufn3@#QwAzHBg6X?`n6e^en!6fj7rbZ z^C&}H@S$3mhiQ%?sFSjoshg@$W&-;+=ruM)Z)OCo zn>VLU^V^Jn5(_)pkD3{`So^$6SDEz`Bkgd06x1-I%G#OErHrg}JCvKGFYx-`njx=j zi9)}FP{V6yx0N+^CXE!NA~JuM%bPFKOW>ijan31D%#Q7;%=#tzJzo9_GSVEacS6lk zg}w}p5z%{)Cv4|xgIS$lO}bluj4(5P4aKXi4@pK~S%Pl*p*Ral z{t^ALN`FXy!Y88+tW2Fo^?%K%b*4jL?56&!T(F0_k7z66mpV2vaUnJ)m1>o03KK>x!L_}}z>WRC-26Q(IY6`&YwQ!Eq$ zcpU>O4~Ys4qXfny*>Dmg3&l^`aM}+Y=#}w*vlvqLfmPFv`>tLVY?)P5rOnK4KvatwRV)*=vl8xt zrF&Vz6?HJVs4qR?iZT_k66!nFp#!n9i@K9B9JorXRz-tYGjm%^5jOydNKKsS((WUF z4um>u|MVOrY2rpztP^-K*5e)3t=ndzD+j^{@w%yIx->4`cOhX22(ex?vnBA(tN{!Y zxg@HwL$;Ca8ivGx2*UZ8Zh`Z8G$M!nB3$B`IYJc?fhgN>4xq+BMYgY)nDOFSup*w7 z7(~0+sERhR38sPkvsU)>K_nF`2l^9#y#cXBysrv6ZAGrYImM%=R(OM4MT$n z8B!U2u(%>1w!2fepf(IH7~0}CUUNH~IV{g`aPOE~;E662c$n;-@is?=YA_IY0Qji2 z8TRhbY|eH^;mJG2U8>kA?#2ew=E^gh&1Fy>1jH^7B4+x0#Q&BN;UwhT;Vfc*lAppx zdd{DKW=F*O9mbHJOFE_gzFFIG{$8<!`f)vq@)K)5-@KAGdcFzbdYRH0r z*Dm(PA#qq02gMQ4#uRM&EhLnZ-=>L9#6ezD_0w71*341;j*ZiC zD0_i|VR` z_05IOdo&wfY zkwHU6ukt)Y=PRu*llM~1$ONVLT%k-n>J5*RUA>Gx?~nQ#yzH_E;vJPwP)(%4=c%jA z(+9`kZu)p#WyO)JO@#+1=%eu{ zHa`Y~FKX~E+nA?M9)WlaJ$~f84~Y0$E6aH@z9&ylUw}&Cc%GgC+MbOm?3MWOsMizf z_lEm@t^Jje{+eHH@VYK~E)EC%`lQri5*DbVRWLaL!A-J%ZNcx>DTjTGRNuQ)uh1!l zG79Aiw2~x+qDw-dhYD<7Slo5u)H=B9ZSof&y|QdFry$@7H4=A=4lYhY;3MqQCFCvJ zAl=+P^F-;#CD7alKYkR)zk=^7evTBw_K<`LQAbDuIfCW|#_xL1t!u)t@*0MGD7D?=ehG0u<19k@|owbQ^>n7CeQb&Mxk-B@@d^%<`_MXkKY#vuUF%H_%NU(lBYkIpg)yC_GcGpDck=qkBk+*I!4D@BUk7( zUio^QK{QTZZ}5%NH}dqYsJGfX3tErU(h{`3GgkP2b|hZJ)0_A|R`^g~2q(Qc*_x++ zy2L+|U^5k0>6S)YF54BP$+nT2WgDap+1^aI$#y60l5LFk%Ju*qm+f&n34;^q2n%jU z$dYZ29+fTsc1zHFQnoIHl8l2T9GYL0ZoSGr-Qse<)hP`5rlu8om7^Go8W}uOqpvA+ z77!wTdWTjPa4WAAfN?3~9je?>0*4BDg8;{)XshX;OJ1YS5N z^1N74QfT!a_BN7?sA4pUs8>XNa>-iI2ZJiAFsgvhuQQ-T6Y~NXi2ui#LBxi<2-S+# zlX~qWgbMyde|D&>9gZ>BU)3VPk_n)QD$Ue8+f1WPMKDXR| zktSuITkgL^UzUAtx&ICNmh5xOeY`PcpIh{W2X8R+Wy}3mRP5a6mir0unAFpM4a@J* zk^+uWWq36)H=}Un5EJVVQ(iCAbX3$9s1_*^p@!-Zvs8+=0=~+|-CdXwM-|SUepl>_ zZHVY~R8gFexxdmC;Kp`QtVa^-)F=c?sRbTHJ8KGJHZn^*R4#~EX`dV{9b6@)7mI)z zu)uzV>^tXq&zYQ=@4vr(1F(uEhNHv7=jFF*of~_?ZK&(2v7;7M!*hJg=8@&O zn&UMD>4C5X4+SkYd8iqGO=0YXu@kE6JKPRMQT0vD;l5@`kNVo$vaxcHVuSK2zZ2Uw z31O3K%k(Q;({hCfEY~FUKm;M>BE4L?TPkY}aiG2%0Aonjyf`q#Bg+;H(_UceX22V^ z&|e4K_eG#rJ<}9{f&|0pEx-Aq8F!b%mmWUYG zHbeh?%eA5h42j%!ev6?um)}Yug^?r_q*F*@Xb^qK(2DJu3=_HPnQtwU`>06nTn)81 zVI&*{6U2Bi<(X(9mZv|X_=qUMok|K5S7#iH!}QhYZxTH;fMns-7mUt)#@GkQCxdZh+c696m~`Q46UL5^{D`ZI$G9#78A>h7 zpBN!#9yi*|YMaTln4uPP>t*3Ri9M&(FQlQ0jOW@)apC{$*=G>ee9MUBECT_(bL zGC{d=W$O5BA+*CG&toqYxu>cf;dDBd#}j|b+Td?~QEE-VCBhq%MH4H7XqAbHuF*Pr zi+C_P83kU1YyR8@#-Q_%l~&@l(#YU2v#}pr5oz=vt;ln<{+%e2OXn~RHQE-`8SF2` zTKHO+*uM>zD2o;}88pw8QN;y=gZ|A=KxKZl_3XbJ%o)`BgLxO)(CI)6b{N#J=nE>) zg9h2E7@an3Q{N@mBdw7(j^3dA`WvXg7Sz50P)i30wv8}qBmn>bYLl^o9g}I8O@GFq z7(zm%7mU!0EmFY-s053t7o1E^l7Y$0cxDF5QoGs*e?FeAo#wKHWDVB`scGWRV z%`6ka_CpAaLCx8|(D`k{^O%VU6paf`3kiGiC1O zwp@=_o1P38;@QC3u+tJ|YGmi=dxn{w*PJPaNUL6f%Ft=JJ88AYNA5=uL6?d!PBQd0 zeWz{Hq{vj8tDu`9#H)_CMTiWir-a~Dn2w_fQ zO4?ne3_P1UKny)>yCWsr>$ssp!G{cCcb`*ZA>2B^z8!M~9}%5hPZOTIVt5teN&G0L zWYTSXtYQYU46nIzU;&F#ahJ|zc>}}oqvamk zfhFYRwJeh(k$@p{jDO?*gt~_nNrcZCPWcvX0;6PT1(OE@5RD(=|IvB4k1ymrd`V-w zPt3)c2Re7;NGbSwPZ302t_XWm!YlZO;e1oEhdy>V&jEgp`*RpA(Fk1&q4Y0z7|d2h1&2YmBKMRLH%sQ7i55~3>q zh+&CiB4v*$emA_MLdUm6_G#L`G+;T8Ry=ieS=!KbWNG~__|*azfrR$u31YJR5$zD7 zhry-87&=J<{G6!a)Ke%g(D!^B{rP;hj@P#_n4cd_<`Z>9Yk0eccegQ;zf%VpkG;fY zhWX@6e8BJolYjJajUm5K!_A)Q8s?rf{!Gz#cesZ6{A5QBpZ?VNBQel1O483rQA2*^ zS>w0F3w-rF`wXEZp}*ROp5F$~CsupPb*$B3)nJd-Azo3Ey+qpYv5EmigLf1|ctoiW zVK_KH!jHkb4IW8vqBGo}71XX^L_xnoGmpP;qk#@Eg)kO5{jE08F7;vR%%Bu#QrkuX z(h-DD&q*>NV+zrR$Mj7@L((?1{{v7<2MCx5wak;=oN5{t5Tq({h<$)kd8!o<&Dqz zrONcWMS$!O7&K{6RVh)6ZM7xZH&l!s5$Ii2G{ssY(49yg2rvZ0LGZ&(+{%mm`qu9D z$$nuwfAVtg)pnD?o*{n}APf=i-4`GVgWP*SV8CS7)@7Y}G?e=v$0z%iZ5Z23_BBhk zxXch4OJyQuH!g0L7F!~k8cDXX#ACbJC1lVHQe@3m$TA8MAt59#BBky#)9-NR{^p$L zpYM5nKcDY1&pFTcbDrn@en3IvWl9=S*U~A@4Ijmdzo-EnL ztqaR~Uo{$5I{JRQPl#+jLxpHRZhla$olX1-wybsyBX6rP`C+LDuA^M@AWEMxG@ zKMBVeQZNw;LDS;-I@%MFJ~{@BKJtf*dW04%ZrmOfl1Flis3GTOcu;ErLU}gpon0=t zNvLbknyepgu%kM6JCb1dPp7;yaz6HS{kDf6j?$6g*1?#!vm9wR{ZbGaH4(6;eQohp zm&*^4^6S$Wk2UPf6VCeIF3wi?ezh-vdcHmWTL|==z)vh_mCbh2n5bN+&Qaq-A(^WX zIL{88a8hzpM|b%Alfs?T+y1vjk5v9lMBH}{rIiSMg;R%1uO-f}-u{)`9JBRG#*$Ke zsQ=~5tnZV%EB{Pvtz5_NUilUR37O)F8Zt{-Srwp$%qtjH- z_>q}VRGZJ_lv0KzhR{|ea++s-BgUaOD|yqwK|56W%!`ioK*>rrvEw;8L<(^vrE>6G zrx&5@1ayuBcSTE$1jcpN6?p$~5AiX|qRG37=wWGQhio&GHpBhN^|J`8MZpmZ1EXSm zXZim)-dBoL_k3}OL5KhPyMUepxvGx!c*)kDBlu>dlKfeR9@$!+&+|SfjiG-l3G4Yd za_EToE?f-w@b#_(Y2(6k!p?o~%#Iv? zf4-FKRmJ5-!5tvIbTtZ+?p6mWmpGQ_#8}m;-SaWzrihiNO!!fgJ0F6hD$BAFi2|=? zqU4AldlCKJR@O3@pRsIXCyS&cF=#yI*^Gc}RN7U76(T@UF2O0*ML$fTn8|i$@YXr5 z^;1}nWXVxSqRL_|OGu1T`2!}+bKO~=xW4(zw?#Nt2$FeQ^o(pYhc$a)T6OW_GKU;X zqIEq69{2jSyF`SJg>m+NQWbGr%!IMOBUnnMzOvyLzFXlYzLJ$2ZqN27F{=f?JJeXW zgsObQ`qJ<5n^{sr*k!M(%&me9w`qYPfOufz+1DqU<=-_NgMLoH z+Sx)0pRFqwPwAKjD}`1Hsm#ZG#WUS)BMn#!H&me9-K9-UJZrjc&jsDJDx_Q(U@mJR zWh#G3KPM~ggzHvhZ{eM97AbBy7#Bt2L%ZQp{paF!?Xnu+Ek*|nO~7Z52o+XJ!&LA? z`Io1_&k3xn6?)4y?{krT8%sT(-}AkLX648j!#<_lZ*w@_A8n_sX&1D;ia%?)6hc&) z7P)ZAVd6!P?XG3@laR%$*1sfBg!v!3iSr_bzZy4#xy5!ektR1fE>Fq-(2Wuj@hxBZ z1LauY&6nO2%C2S?W?m;-H(>WeH`eYUarl|6OataGPIqr{yQy>Oh{Jv}Du$CS;~({2 zmziJ7C{V9hbnj_WQ_b4)FDu6X1$;8!4zG7Du6<{KYbP~ec^=|d4Pa#4bJ9aso}}&z z$D4LsS&liXS-B;3`Le&QMJnF!Ml-WrbI z|2$}~rJXn+s8w=i_k(v`BwkF-ZBSO39I&K_JCxLEGlY*MSWX3tRopo0zEL-3>QzdJ zJV?HAbcH;K7&E3Y%KA8!CKcTpHAqS*N|-2!ws_~{h*RFswfK~h`k4IQbg~nAR9Vd9 zD0g0m9%%H={K@>)2_0S!T$3=d$fp11_*h0b6vIKJh6-?p%6wsnW3 z7cw5&w=K0xNH#*|ISu&dw6m}$@S|)qBc<^kl4N1Ub#zo?y9#NQ15>;+<}n!10IOaD zE4f16$Qx-Jp1@ZINMssuw4ft^2;w|1?Eh7W8h8D%9U{=97 z?wuL)l}q}39463&mBa#KanXd)m{hq{kj27sEjvzZLjD-^>FR_G=Gwi_%W@+&WbUtC z&&SR$Ke8bwL|zUg@uZQMQ^b54P7VxWE)@-5e^lU;yoFoV78UiF&&V~E&N^{Y{4!H? z+UQwZ$0>HK{B3MWQ5U)p57_sGDWbdKwxUztXsW0#+an?1d8mxJ$K+{=G%!`-fne?C zz$SvH-f>VAQVs$tA?^LLcp*RmR1COYDvKD|`OxAFQnjFygeDyo1*K9b;BF}|Q)+fS z2f+Oh#&`-wZIl8m9%>=>yqI|)5ENZksOWF%w5Tk#JA!JidkRdF0VYAxk*_2nzyT>! zDkZjWQ~KX{@c_Qe0lsgG34!R+MH!X{!pjr^1W^f2bw88>Q+a~3rk~G8MH&=Olts|2 z_P|qZ9|&sDMG8>b zQ7KVSY~Kj6FAffY7|>A@#t`5T+MGH{I8CA&T#7^BQmD~UDrlgUBnW6f(q^V z&K`u!#r8jrfk2e#fEPdu2@V`1D>2btC-Wu47#;@D0%wsG;MBLCE`alz2QT`-oi}Na z&@E}&@b?Rn5Qqd_g4A#tXkR4<->V>j`yx&UVUlUkqbgbWUabU7N&naQ1<;^h>2O*~ z>aj=zjN}O#nXA%83s0kg;h8ctkoG_dZj2Sne 0); - - switch (i) { - case 0: - handle = new BytesHandle(); - buf = docMgr.read(docId, handle, 9, 10).get(); - assertEquals(10, buf.length); - assertEquals(10, handle.getByteLength()); - break; - case 1: - break; - default: - fail("unknown case: "+i); - } - - docMgr.setMetadataCategories(Metadata.PROPERTIES); - Document metadataDocument = docMgr.readMetadata(docId, new DOMHandle()).get(); - assertXpathEvaluatesTo("image/png","string(/*[local-name()='metadata']/*[local-name()='properties']/*[local-name()='content-type'])", metadataDocument); - assertXpathEvaluatesTo("text HD-HTML","string(/*[local-name()='metadata']/*[local-name()='properties']/*[local-name()='filter-capabilities'])", metadataDocument); - assertXpathEvaluatesTo("815","string(/*[local-name()='metadata']/*[local-name()='properties']/*[local-name()='size'])", metadataDocument); - - docMgr.delete(docId); - } - } - - @Test - public void test_issue_758() { - BinaryDocumentManager docMgr = Common.client.newBinaryDocumentManager(); - DocumentWriteSet writeset = docMgr.newWriteSet(); - FileHandle h1 = new FileHandle(new File( - "../marklogic-client-api-functionaltests/src/test/java/com/marklogic" + - "/client" + - "/functionaltest/data" + - "/Sega-4MB.jpg")); - String uri = "BinaryDocumentTest_" + new Random().nextInt(10000) + "/" + "Sega-4MB.jpg"; - writeset.add(uri, h1); - docMgr.write(writeset); - DocumentPage page = docMgr.read(uri); - DocumentRecord rec = page.next(); - assertNotNull(rec); - assertEquals(rec.getFormat(),Format.BINARY); - } +class BinaryDocumentTest { + + @BeforeAll + public static void beforeClass() { + Common.connect(); + } + + // a simple base64-encoded binary + final static public String ENCODED_BINARY = + "iVBORw0KGgoAAAANSUhEUgAAAA0AAAATCAYAAABLN4eXAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9oIEQEjMtAYogQAAAKvSURBVCjPlZLLbhxFAEVPVVdXVz/G8zCOn0CsKGyQkSIIKzas8xfsWbLkp/gJhCKheIlAJDaj2MYez6u7p7vrxQKUPVc6+yOdK77/4cfXQohJqlOVZdmBSpKY6jQKBM45oVMlgHvrvMuNWRljvlNKq69G2YyqLDg4mLE/2yPNYFRWlFXF/nTC2clRWbc7Fss1IcZzqTA8eWY5eu7p1Hv+WvyBVjnGZOQmI9UKISUqSXDO0bS7Tko0xfGSp18kjM7v+P3+NUMr8T5grWMYLCEErHM474khoCw1t78eU/8mEOpjXpxekJUORIZSCbkxSCnRWpPnBikTqbx31E1DjJHpeIzRhnW9xceI857H5Yr1Zku765jf3DIMtlUAIQRCiFhnabsOH1IEAmstAGWRY11ApykmM0oplTKZjNGZREpJoUueHI0ZFRV7exX7+1Nm0yn9YLm5u2fX96lUseLwxQ0vX8H04i2/XP9Et5H44OkHS920hBDo+56u77GDjcrHjvV1ya3TDO2M01mOUAEAhED+R5IkpKmCiFCOjoc/p+xuLbPpCc+P95HaEqIBIhHoB8t2W/PwsKBudl5FH7GxwUYYouJh5ci7nLbtWW02LBaPvLuef1AdrItKKolJpkivwGrG5QxTCsq8pCxLqqrk7PiIwTmW6y0xRCVTSg4vFnz+raM4+5ur1RtSUZHnOUWeMx5VVFWJTlOstfTWRuk96NIyOUgRRc188RZvgRg/3OffjoFESohxUMvmjqufP+X+MqDTU77+5EvMKKBUQpZpijxHSkluDHvjMW8uL79Rnz07bwSyzDLFqCzwDNw/PNI0O9bbhvVmQ7vb0bQdi+Wq327rl+rko8krodKnCHnofJju+r5oupBstg1KJT7Vuruev185O9zVm/WVUmouYoz83/0DxhRmafe2kasAAAAASUVORK5CYII="; + final static public byte[] BYTES_BINARY = DatatypeConverter.parseBase64Binary(ENCODED_BINARY); + + @Test + // Requires MarkLogic 11 or higher now that we're using Docker; the INSTALL_CONVERTERS flag does not work for MarkLogic 10. + @ExtendWith(RequiresML11.class) + @DisabledOnOs(value = OS.LINUX, architectures = "aarch64", disabledReason = "MarkLogic converters not available when " + + "ARM regression tests are run in Jenkins.") + void testReadWrite() throws IOException, XpathException { + String docId = "/test/binary-sample.png"; + String mimetype = "image/png"; + + for (int i : new int[]{0, 1}) { + BinaryDocumentManager docMgr = Common.client.newBinaryDocumentManager(); + docMgr.setMetadataExtraction(MetadataExtraction.PROPERTIES); + + BytesHandle handle = new BytesHandle().with(BYTES_BINARY).withMimetype(mimetype); + switch (i) { + case 0: + docMgr.write(docId, handle); + break; + case 1: + docMgr.write(docMgr.newWriteSet().add(docId, handle)); + break; + default: + fail("unknown case: " + i); + } + + DocumentDescriptor desc = docMgr.exists(docId); + assertTrue(desc.getByteLength() != DocumentDescriptor.UNKNOWN_LENGTH); + assertEquals(BYTES_BINARY.length, desc.getByteLength()); + + byte[] buf = docMgr.read(docId, new BytesHandle()).get(); + assertEquals(BYTES_BINARY.length, buf.length); + + buf = Common.streamToBytes(docMgr.read(docId, new InputStreamHandle()).get()); + assertTrue(buf.length > 0); + + switch (i) { + case 0: + handle = new BytesHandle(); + buf = docMgr.read(docId, handle, 9, 10).get(); + assertEquals(10, buf.length); + assertEquals(10, handle.getByteLength()); + break; + case 1: + break; + default: + fail("unknown case: " + i); + } + + docMgr.setMetadataCategories(Metadata.PROPERTIES); + Document metadataDocument = docMgr.readMetadata(docId, new DOMHandle()).get(); + assertXpathEvaluatesTo("image/png", "string(/*[local-name()='metadata']/*[local-name()='properties']/*[local-name()='content-type'])", metadataDocument); + assertXpathEvaluatesTo("text HD-HTML", "string(/*[local-name()='metadata']/*[local-name()='properties']/*[local-name()='filter-capabilities'])", metadataDocument); + assertXpathEvaluatesTo("815", "string(/*[local-name()='metadata']/*[local-name()='properties']/*[local-name()='size'])", metadataDocument); + + docMgr.delete(docId); + } + } + + @Test + void test_issue_758() { + BinaryDocumentManager docMgr = Common.client.newBinaryDocumentManager(); + DocumentWriteSet writeset = docMgr.newWriteSet(); + FileHandle h1 = new FileHandle(new File( + "../marklogic-client-api-functionaltests/src/test/java/com/marklogic" + + "/client" + + "/functionaltest/data" + + "/Sega-4MB.jpg")); + String uri = "BinaryDocumentTest_" + new Random().nextInt(10000) + "/" + "Sega-4MB.jpg"; + writeset.add(uri, h1); + docMgr.write(writeset); + DocumentPage page = docMgr.read(uri); + DocumentRecord rec = page.next(); + assertNotNull(rec); + assertEquals(rec.getFormat(), Format.BINARY); + } } From 84925402ef3fc1e5a52ef98f9470a28b5a769fa3 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 24 Feb 2026 17:11:10 -0500 Subject: [PATCH 60/70] MLE-27304 Timestamp key is now optional for incremental write --- .../filter/IncrementalWriteFilter.java | 14 +++++++------- .../filter/AbstractIncrementalWriteTest.java | 1 + .../datamovement/filter/IncrementalWriteTest.java | 7 +++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java index 253adb51c..25be19f06 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -46,7 +46,7 @@ public static Builder newBuilder() { public static class Builder { private String hashKeyName = "incrementalWriteHash"; - private String timestampKeyName = "incrementalWriteTimestamp"; + private String timestampKeyName; private boolean canonicalizeJson = true; private Consumer skippedDocumentsConsumer; private String[] jsonExclusions; @@ -67,13 +67,11 @@ public Builder hashKeyName(String keyName) { } /** - * @param keyName the name of the MarkLogic metadata key that will hold the timestamp value; defaults to "incrementalWriteTimestamp". + * @param keyName the name of the MarkLogic metadata key that will hold the timestamp value; + * defaults to null, which means no timestamp will be stored. */ public Builder timestampKeyName(String keyName) { - // Don't let user shoot themselves in the foot with an empty key name. - if (keyName != null && !keyName.trim().isEmpty()) { - this.timestampKeyName = keyName; - } + this.timestampKeyName = keyName; return this; } @@ -323,7 +321,9 @@ protected static DocumentWriteOperation addHashToMetadata(DocumentWriteOperation } newMetadata.getMetadataValues().put(hashKeyName, Long.toUnsignedString(hash)); - newMetadata.getMetadataValues().put(timestampKeyName, timestamp); + if (timestampKeyName != null && !timestampKeyName.trim().isEmpty()) { + newMetadata.getMetadataValues().put(timestampKeyName, timestamp); + } return new DocumentWriteOperationImpl(op.getUri(), newMetadata, op.getContent(), op.getTemporalDocumentURI()); } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java index 6c136f0e5..610d5c83e 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java @@ -37,6 +37,7 @@ void setup() { // Default filter implementation, should be suitable for most tests. filter = IncrementalWriteFilter.newBuilder() .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .timestampKeyName("incrementalWriteTimestamp") .build(); } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java index e00fd9915..4c89e3dc5 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -157,7 +157,8 @@ void customTimestampKeyName() { /** * The thought for this test is that if the user passes null in (which could happen via our Spark connector), - * they're breaking the feature. So don't let them do that - ignore null and use the default values. + * they're breaking the feature. So don't let them do that - ignore null and use the default value for the hash + * key name - but null is still valid for the timestamp key name as that's an optional value to set. */ @Test void nullIsIgnoredForKeyNames() { @@ -172,7 +173,7 @@ void nullIsIgnoredForKeyNames() { new DocumentMetadataHandle()); assertNotNull(metadata.getMetadataValues().get("incrementalWriteHash")); - assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); + assertFalse(metadata.getMetadataValues().containsKey("incrementalWriteTimestamp")); } @Test @@ -219,6 +220,7 @@ void binaryDocument() { void fromView() { filter = IncrementalWriteFilter.newBuilder() .fromView("javaClient", "incrementalWriteHash") + .timestampKeyName("incrementalWriteTimestamp") .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) .build(); @@ -275,6 +277,7 @@ void emptyValuesForFromView() { // Empty/null values are ignored, as long as both schema/view are empty/null. This makes life a little // easier for a connector in that the connector does not need to check for empty/null values. .fromView("", null) + .timestampKeyName("incrementalWriteTimestamp") .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) .build(); From b529df198539bd9cc1595038605cb62c21e73da4 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 2 Mar 2026 10:38:05 -0500 Subject: [PATCH 61/70] MLE-27304 Small cleanup for Polaris No change here, just moving InputStreamTee into RequestLoggerImpl as that's the only place where it's used. And it makes it easier to see that the potential thread safety issues of InputStreamTee won't happen in practice. --- .../marklogic/client/impl/InputStreamTee.java | 128 ----------------- .../client/impl/RequestLoggerImpl.java | 131 +++++++++++++++++- 2 files changed, 125 insertions(+), 134 deletions(-) delete mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/InputStreamTee.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/InputStreamTee.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/InputStreamTee.java deleted file mode 100644 index 944d7a7ed..000000000 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/InputStreamTee.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. - */ -package com.marklogic.client.impl; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class InputStreamTee extends InputStream { - private InputStream in; - private OutputStream tee; - private long max = 0; - private long sent = 0; - - public InputStreamTee(InputStream in, OutputStream tee, long max) { - super(); - this.in = in; - this.tee = tee; - this.max = max; - } - - @Override - public int read() throws IOException { - if (in == null) return -1; - - if (sent >= max) return in.read(); - - int b = in.read(); - if (b == -1) { - cleanupTee(); - return b; - } - - if (max == Long.MAX_VALUE) { - tee.write(b); - return b; - } - - tee.write(b); - - sent++; - if (sent == max) cleanupTee(); - - return b; - } - @Override - public int read(byte[] b) throws IOException { - if (in == null) return -1; - - if (sent >= max) return in.read(b); - - return readTee(b, 0, in.read(b)); - } - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (in == null) return -1; - - if (sent >= max) return in.read(b, off, len); - - return readTee(b, 0, in.read(b, off, len)); - } - private int readTee(byte[] b, int off, int resultLen) throws IOException { - if (resultLen < 1) { - if (resultLen == -1) cleanupTee(); - return resultLen; - } - - if (max == Long.MAX_VALUE) { - tee.write(b, off, resultLen); - return resultLen; - } - - int teeLen = ((sent + resultLen) <= max) ? resultLen : (int) (max - sent); - sent += teeLen; - tee.write(b, off, teeLen); - - if (sent >= max) cleanupTee(); - - return resultLen; - } - private void cleanupTee() throws IOException { - if (tee == null) return; - - tee.flush(); - tee = null; - } - @Override - public void close() throws IOException { - if (in == null) return; - - in.close(); - in = null; - - cleanupTee(); - } - - @Override - public int available() throws IOException { - if (in == null) return 0; - - return in.available(); - } - @Override - public boolean markSupported() { - if (in == null) return false; - - return in.markSupported(); - } - @Override - public synchronized void mark(int readlimit) { - if (in == null) return; - - in.mark(readlimit); - } - @Override - public synchronized void reset() throws IOException { - if (in == null) throw new IOException("Input Stream closed"); - - in.reset(); - } - @Override - public long skip(long n) throws IOException { - if (in == null) throw new IOException("Input Stream closed"); - - return in.skip(n); - } -} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestLoggerImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestLoggerImpl.java index e585df7c9..941dc5449 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestLoggerImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RequestLoggerImpl.java @@ -1,13 +1,9 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; -import java.io.File; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintStream; -import java.io.Reader; +import java.io.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,4 +113,127 @@ public void close() { enabled = false; } + // Moved here in the 8.2 release to make it obvious it's only used by RequestLoggerImpl. It is only used by the + // copyContent method in RequestLoggerImpl. So while Polaris rightfully complains about the thread safety issues of + // this class, it is only used in a single thread context and so the thread safety issues are not a problem. + private static class InputStreamTee extends InputStream { + private InputStream in; + private OutputStream tee; + private long max = 0; + private long sent = 0; + + public InputStreamTee(InputStream in, OutputStream tee, long max) { + super(); + this.in = in; + this.tee = tee; + this.max = max; + } + + @Override + public int read() throws IOException { + if (in == null) return -1; + + if (sent >= max) return in.read(); + + int b = in.read(); + if (b == -1) { + cleanupTee(); + return b; + } + + if (max == Long.MAX_VALUE) { + tee.write(b); + return b; + } + + tee.write(b); + + sent++; + if (sent == max) cleanupTee(); + + return b; + } + + @Override + public int read(byte[] b) throws IOException { + if (in == null) return -1; + + if (sent >= max) return in.read(b); + + return readTee(b, 0, in.read(b)); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (in == null) return -1; + + if (sent >= max) return in.read(b, off, len); + + return readTee(b, 0, in.read(b, off, len)); + } + + private int readTee(byte[] b, int off, int resultLen) throws IOException { + if (resultLen < 1) { + if (resultLen == -1) cleanupTee(); + return resultLen; + } + + if (max == Long.MAX_VALUE) { + tee.write(b, off, resultLen); + return resultLen; + } + + int teeLen = ((sent + resultLen) <= max) ? resultLen : (int) (max - sent); + sent += teeLen; + tee.write(b, off, teeLen); + + if (sent >= max) cleanupTee(); + + return resultLen; + } + + private void cleanupTee() throws IOException { + if (tee == null) return; + tee.flush(); + tee = null; + } + + @Override + public void close() throws IOException { + if (in == null) return; + in.close(); + in = null; + cleanupTee(); + } + + @Override + public int available() throws IOException { + if (in == null) return 0; + return in.available(); + } + + @Override + public boolean markSupported() { + if (in == null) return false; + return in.markSupported(); + } + + @Override + public synchronized void mark(int readLimit) { + if (in == null) return; + in.mark(readLimit); + } + + @Override + public synchronized void reset() throws IOException { + if (in == null) throw new IOException("Input Stream closed"); + in.reset(); + } + + @Override + public long skip(long n) throws IOException { + if (in == null) throw new IOException("Input Stream closed"); + return in.skip(n); + } + } } From aaa7c20d4ad5649e99e764089e36978f1ea66d68 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 3 Mar 2026 09:00:36 -0500 Subject: [PATCH 62/70] MLE-27388 Refactor: Cutting down on duplicated retry code Added RetryContext and tossed in some debug logging, but that may change a bit in the next PR. This gets rid of 9 occurrences of the same code and also means that logging for the retry-50x can be done in a single place now. --- .../marklogic/client/impl/OkHttpServices.java | 231 +++++++----------- .../marklogic/client/impl/RetryContext.java | 75 ++++++ 2 files changed, 164 insertions(+), 142 deletions(-) create mode 100644 marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 57ec71dc0..8ad5cec52 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -515,20 +515,18 @@ private Response sendRequestWithRetry(Request.Builder requestBldr, Function doFunction, Consumer resendableConsumer ) { + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; + /* * This loop is for retrying the request if the service is unavailable */ - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } /* @@ -567,19 +565,15 @@ private Response sendRequestWithRetry( /* * Calculate the delay before which we shouldn't retry */ - nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(retry)); + retryContext.calculateNextDelay(getRetryAfterTime(response), calculateDelay(retryContext.getRetry())); } /* * If the service is still unavailable after all the retries, we throw a * FailedRetryException indicating that the service is unavailable. */ if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } /* * Once we break from the retry loop, we just return the Response @@ -1197,18 +1191,15 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } boolean isResendable = handleBase.isResendable(); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; Headers responseHeaders = null; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } Object value = handleBase.sendContent(); @@ -1218,8 +1209,11 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } if (isFirstRequest() && !isResendable && isStreaming(value)) { - nextDelay = makeFirstRequest(retry); - if (nextDelay != 0) continue; + int firstRequestDelay = makeFirstRequest(retryContext.getRetry()); + if (firstRequestDelay != 0) { + retryContext.calculateNextDelay(0, firstRequestDelay); + continue; + } } MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); @@ -1260,15 +1254,11 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } if (status == STATUS_NOT_FOUND) { throw new ResourceNotFoundException( @@ -1358,18 +1348,15 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); } + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; Headers responseHeaders = null; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } MultipartBody.Builder multiPart = new MultipartBody.Builder(); @@ -1378,8 +1365,11 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth new AbstractWriteHandle[]{metadataHandle, contentHandle}); if (isFirstRequest() && hasStreamingPart) { - nextDelay = makeFirstRequest(retry); - if (nextDelay != 0) continue; + int firstRequestDelay = makeFirstRequest(retryContext.getRetry()); + if (firstRequestDelay != 0) { + retryContext.calculateNextDelay(0, firstRequestDelay); + continue; + } } requestBldr = ("put".equals(method)) ? requestBldr.put(multiPart.build()) : requestBldr.post(multiPart.build()); @@ -1402,15 +1392,11 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } if (status == STATUS_NOT_FOUND) { closeResponse(response); @@ -2126,17 +2112,14 @@ void init() { } Response getResponse() { + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, OkHttpServices.this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } if (queryDef instanceof StructuredQueryDefinition && !(queryDef instanceof RawQueryDefinition)) { @@ -2171,15 +2154,11 @@ Response getResponse() { closeResponse(response); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } if (status == STATUS_NOT_FOUND) { closeResponse(response); @@ -2665,17 +2644,14 @@ private void putPostValueImpl(RequestLogger reqlog, String method, String connectPath = null; Request.Builder requestBldr = null; + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } Object nextValue = (handle != null) ? handle.sendContent() : value; @@ -2685,7 +2661,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, sentValue = new StreamingOutputImpl( (OutputStreamSender) nextValue, reqlog, mediaType); } else { - if (reqlog != null && retry == 0) { + if (reqlog != null && retryContext.getRetry() == 0) { sentValue = new ObjectRequestBody(reqlog.copyContent(nextValue), mediaType); } else { sentValue = new ObjectRequestBody(nextValue, mediaType); @@ -2697,8 +2673,11 @@ private void putPostValueImpl(RequestLogger reqlog, String method, boolean isResendable = (handle == null) ? !isStreaming : handle.isResendable(); if (isFirstRequest() && !isResendable && isStreaming) { - nextDelay = makeFirstRequest(retry); - if (nextDelay != 0) continue; + int firstRequestDelay = makeFirstRequest(retryContext.getRetry()); + if (firstRequestDelay != 0) { + retryContext.calculateNextDelay(0, firstRequestDelay); + continue; + } } if ("put".equals(method)) { @@ -2747,15 +2726,11 @@ private void putPostValueImpl(RequestLogger reqlog, String method, } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to write " @@ -3098,17 +3073,14 @@ public R putResour String outputMimetype = outputBase.getMimetype(); Class as = outputBase.receiveAs(); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } MultipartBody.Builder multiPart = new MultipartBody.Builder(); @@ -3137,15 +3109,11 @@ public R putResour } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } checkStatus(response, status, "write", "resource", path, @@ -3272,17 +3240,14 @@ public R postResou String outputMimetype = outputBase != null ? outputBase.getMimetype() : null; Class as = outputBase != null ? outputBase.receiveAs() : null; + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } MultipartBody.Builder multiPart = new MultipartBody.Builder(); @@ -3311,15 +3276,11 @@ public R postResou } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } checkStatus(response, status, "apply", "resource", path, @@ -3882,17 +3843,14 @@ private U postIt throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { if (params == null) params = new RequestParameters(); if (transaction != null) params.add("txid", transaction.getTransactionId()); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } MultipartBody.Builder multiPart = new MultipartBody.Builder(); @@ -3924,15 +3882,11 @@ private U postIt } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } checkStatus(response, status, "apply", "resource", path, @@ -5001,17 +4955,14 @@ public InputStream match(QueryDefinition queryDef, MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); Response response = null; int status = -1; - long startTime = System.currentTimeMillis(); - int nextDelay = 0; - int retry = 0; - for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { - if (nextDelay > 0) { - try { - Thread.sleep(nextDelay); - } catch (InterruptedException e) { - } + for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { + try { + retryContext.sleepIfNeeded(); + } catch (InterruptedException e) { + // Ignore interruption } if (queryDef instanceof StructuredQueryDefinition) { @@ -5037,15 +4988,11 @@ public InputStream match(QueryDefinition queryDef, closeResponse(response); - nextDelay = Math.max(retryAfter, calculateDelay(retry)); + retryContext.calculateNextDelay(retryAfter, calculateDelay(retryContext.getRetry())); } if (RETRYABLE_STATUS_CODES.contains(status)) { - checkFirstRequest(); closeResponse(response); - throw new FailedRetryException( - "Service unavailable and maximum retry period elapsed: " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds after " + retry + " retries"); + retryContext.throwIfMaxRetriesExceeded(status); } if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to match", diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java new file mode 100644 index 000000000..4f0632f34 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl; + +import com.marklogic.client.FailedRetryException; +import org.slf4j.Logger; + +import java.util.Set; + +/** + * Helper class to consolidate retry logic across multiple HTTP operations. + * Tracks retry state, calculates delays, handles sleeping, and logs retry attempts. + */ +class RetryContext { + private final Logger logger; + private final Set retryableStatusCodes; + private final Runnable onMaxRetriesCallback; + + private int retry = 0; + private final long startTime = System.currentTimeMillis(); + private int nextDelay = 0; + + /** + * @param logger Logger for debug output + * @param retryableStatusCodes Set of HTTP status codes that trigger retries + * @param onMaxRetriesCallback Callback to invoke when max retries is exceeded (e.g., to reset first request flag) + */ + RetryContext(Logger logger, Set retryableStatusCodes, Runnable onMaxRetriesCallback) { + this.logger = logger; + this.retryableStatusCodes = retryableStatusCodes; + this.onMaxRetriesCallback = onMaxRetriesCallback; + } + + boolean shouldContinueRetrying(int minAttempts, int maxDelay) { + return retry < minAttempts || (System.currentTimeMillis() - startTime) < maxDelay; + } + + void sleepIfNeeded() throws InterruptedException { + if (nextDelay > 0) { + if (logger.isDebugEnabled()) { + logger.debug("Retrying request after {} ms delay (attempt {})", nextDelay, retry); + } + Thread.sleep(nextDelay); + } + } + + void calculateNextDelay(int retryAfter, int calculatedDelay) { + nextDelay = Math.max(retryAfter, calculatedDelay); + if (logger.isDebugEnabled()) { + logger.debug("Calculated next retry delay: {} ms (retryAfter: {}, calculatedDelay: {})", + nextDelay, retryAfter, calculatedDelay); + } + } + + void throwIfMaxRetriesExceeded(int status) { + if (retryableStatusCodes.contains(status)) { + if (onMaxRetriesCallback != null) { + onMaxRetriesCallback.run(); + } + throw new FailedRetryException( + "Service unavailable and maximum retry period elapsed: " + + ((System.currentTimeMillis() - startTime) / 1000) + + " seconds after " + retry + " retries"); + } + } + + void incrementRetry() { + retry++; + } + + int getRetry() { + return retry; + } +} From bf0ae480cf2b8541f7791af03b40311d4f2b740e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 3 Mar 2026 11:23:36 -0500 Subject: [PATCH 63/70] MLE-27388: Refactor: Renamed the "checkFirstRequest" stuff The new names actually convey what is going on as opposed to making you wonder what "check" might mean. Also modified sleepIfNeeded to catch the interrupted exception instead of repeating that catch. --- .../marklogic/client/impl/OkHttpServices.java | 145 ++++++++---------- .../marklogic/client/impl/RetryContext.java | 13 +- 2 files changed, 77 insertions(+), 81 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 8ad5cec52..bc2906909 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -120,7 +120,8 @@ public class OkHttpServices implements RESTServices { private static final Set RETRYABLE_STATUS_CODES = Set.of(STATUS_BAD_GATEWAY, STATUS_SERVICE_UNAVAILABLE, STATUS_GATEWAY_TIMEOUT); - private boolean checkFirstRequest = true; + // When true (digest auth), ping server before streaming non-resendable content to avoid wasted uploads + private boolean useDigestAuthPing = true; static protected class ThreadState { boolean isFirstRequest; @@ -131,7 +132,7 @@ static protected class ThreadState { } private final ThreadLocal threadState = - ThreadLocal.withInitial(() -> new ThreadState(checkFirstRequest)); + ThreadLocal.withInitial(() -> new ThreadState(useDigestAuthPing)); public record ConnectionConfig(String host, int port, String basePath, String database, SecurityContext securityContext, List clientConfigurators) { @@ -197,7 +198,7 @@ private OkHttpClient connect(ConnectionConfig config) { throw new IllegalArgumentException("No security context provided"); } - this.checkFirstRequest = config.securityContext instanceof DigestAuthContext; + this.useDigestAuthPing = config.securityContext instanceof DigestAuthContext; this.database = config.database; this.baseUri = HttpUrlBuilder.newBaseUrl(config.host, config.port, config.basePath, config.securityContext.getSSLContext()); @@ -283,15 +284,17 @@ private void setFirstRequest(boolean value) { threadState.get().isFirstRequest = value; } - private void checkFirstRequest() { - if (checkFirstRequest) setFirstRequest(true); + private void resetFirstRequestFlag() { + if (useDigestAuthPing) { + setFirstRequest(true); + } } - private int makeFirstRequest(int retry) { - return makeFirstRequest(baseUri, "ping", retry); + private int pingServerBeforeStreaming(int retry) { + return pingServer(baseUri, "ping", retry); } - private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { + private int pingServer(HttpUrl requestUri, String path, int retry) { Response response = sendRequestOnce(setupRequest(requestUri, path, null).head()); int statusCode = response.code(); if (!RETRYABLE_STATUS_CODES.contains(statusCode)) { @@ -515,7 +518,12 @@ private Response sendRequestWithRetry(Request.Builder requestBldr, Function doFunction, Consumer resendableConsumer ) { - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + // If the thread is already interrupted, fail fast rather than attempting a request + if (Thread.currentThread().isInterrupted()) { + throw new MarkLogicIOException("Request cancelled: thread was interrupted before request could be sent"); + } + + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; @@ -523,11 +531,7 @@ private Response sendRequestWithRetry( * This loop is for retrying the request if the service is unavailable */ for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); /* * Execute the function which is passed as an argument @@ -1191,16 +1195,12 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } boolean isResendable = handleBase.isResendable(); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; Headers responseHeaders = null; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); Object value = handleBase.sendContent(); if (value == null) { @@ -1209,7 +1209,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } if (isFirstRequest() && !isResendable && isStreaming(value)) { - int firstRequestDelay = makeFirstRequest(retryContext.getRetry()); + int firstRequestDelay = pingServerBeforeStreaming(retryContext.getRetry()); if (firstRequestDelay != 0) { retryContext.calculateNextDelay(0, firstRequestDelay); continue; @@ -1247,7 +1247,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth closeResponse(response); if (!isResendable) { - checkFirstRequest(); + resetFirstRequestFlag(); throw new ResourceNotResendableException( "Cannot retry request for " + ((uri != null) ? uri : "new document")); @@ -1348,16 +1348,12 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); } - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; Headers responseHeaders = null; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); MultipartBody.Builder multiPart = new MultipartBody.Builder(); boolean hasStreamingPart = addParts(multiPart, reqlog, @@ -1365,7 +1361,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth new AbstractWriteHandle[]{metadataHandle, contentHandle}); if (isFirstRequest() && hasStreamingPart) { - int firstRequestDelay = makeFirstRequest(retryContext.getRetry()); + int firstRequestDelay = pingServerBeforeStreaming(retryContext.getRetry()); if (firstRequestDelay != 0) { retryContext.calculateNextDelay(0, firstRequestDelay); continue; @@ -1923,7 +1919,6 @@ public T search(RequestLogger reqlog, T searchHandl addPointInTimeQueryParam(params, searchHandle); - @SuppressWarnings("rawtypes") HandleImplementation searchBase = HandleAccessor.checkHandle(searchHandle, "search"); Format searchFormat = searchBase.getFormat(); @@ -2112,15 +2107,11 @@ void init() { } Response getResponse() { - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, OkHttpServices.this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, OkHttpServices.this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); if (queryDef instanceof StructuredQueryDefinition && !(queryDef instanceof RawQueryDefinition)) { response = doPost(reqlog, requestBldr, structure); @@ -2644,15 +2635,11 @@ private void putPostValueImpl(RequestLogger reqlog, String method, String connectPath = null; Request.Builder requestBldr = null; - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); Object nextValue = (handle != null) ? handle.sendContent() : value; @@ -2673,7 +2660,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, boolean isResendable = (handle == null) ? !isStreaming : handle.isResendable(); if (isFirstRequest() && !isResendable && isStreaming) { - int firstRequestDelay = makeFirstRequest(retryContext.getRetry()); + int firstRequestDelay = pingServerBeforeStreaming(retryContext.getRetry()); if (firstRequestDelay != 0) { retryContext.calculateNextDelay(0, firstRequestDelay); continue; @@ -2720,7 +2707,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, closeResponse(response); if (!isResendable) { - checkFirstRequest(); + resetFirstRequestFlag(); throw new ResourceNotResendableException( "Cannot retry request for " + connectPath); } @@ -3029,7 +3016,7 @@ public R putResource(RequestLogger reqlog, Consumer resendableConsumer = (resendable) -> { if (!isResendable) { - checkFirstRequest(); + resetFirstRequestFlag(); throw new ResourceNotResendableException( "Cannot retry request for " + path); } @@ -3073,15 +3060,11 @@ public R putResour String outputMimetype = outputBase.getMimetype(); Class as = outputBase.receiveAs(); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); MultipartBody.Builder multiPart = new MultipartBody.Builder(); boolean hasStreamingPart = addParts(multiPart, reqlog, input); @@ -3182,7 +3165,7 @@ public R postResource(RequestLogger reqlog, Consumer resendableConsumer = new Consumer() { public void accept(Boolean resendable) { if (!isResendable) { - checkFirstRequest(); + resetFirstRequestFlag(); throw new ResourceNotResendableException("Cannot retry request for " + path); } } @@ -3240,15 +3223,11 @@ public R postResou String outputMimetype = outputBase != null ? outputBase.getMimetype() : null; Class as = outputBase != null ? outputBase.receiveAs() : null; - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); MultipartBody.Builder multiPart = new MultipartBody.Builder(); boolean hasStreamingPart = addParts(multiPart, reqlog, null, input, requestHeaders); @@ -3778,7 +3757,7 @@ private U postIteratedResourceImpl( Consumer resendableConsumer = resendable -> { if (!isResendable) { - checkFirstRequest(); + resetFirstRequestFlag(); throw new ResourceNotResendableException( "Cannot retry request for " + path); } @@ -3843,15 +3822,11 @@ private U postIt throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { if (params == null) params = new RequestParameters(); if (transaction != null) params.add("txid", transaction.getTransactionId()); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); MultipartBody.Builder multiPart = new MultipartBody.Builder(); boolean hasStreamingPart = addParts(multiPart, reqlog, input); @@ -3966,7 +3941,7 @@ private Request.Builder makePutWebResource(String path, private Response doPut(RequestLogger reqlog, Request.Builder requestBldr, Object value) { if (value == null) throw new IllegalArgumentException("Resource write with null value"); - if (isFirstRequest() && isStreaming(value)) makeFirstRequest(0); + if (isFirstRequest() && isStreaming(value)) pingServerBeforeStreaming(0); MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); if (value instanceof OutputStreamSender) { @@ -3987,7 +3962,7 @@ private Response doPut(RequestLogger reqlog, Request.Builder requestBldr, Object private Response doPut(Request.Builder requestBldr, MultipartBody.Builder multiPart, boolean hasStreamingPart) { - if (isFirstRequest() && hasStreamingPart) makeFirstRequest(0); + if (isFirstRequest() && hasStreamingPart) pingServerBeforeStreaming(0); requestBldr = requestBldr.put(multiPart.build()); Response response = sendRequestOnce(requestBldr); @@ -4007,7 +3982,7 @@ private Request.Builder makePostWebResource(String path, RequestParameters param private Response doPost(RequestLogger reqlog, Request.Builder requestBldr, Object value) { if (isFirstRequest() && isStreaming(value)) { - makeFirstRequest(0); + pingServerBeforeStreaming(0); } MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); @@ -4034,7 +4009,7 @@ private Response doPost(RequestLogger reqlog, Request.Builder requestBldr, Objec private Response doPost(Request.Builder requestBldr, MultipartBody.Builder multiPart, boolean hasStreamingPart) { - if (isFirstRequest() && hasStreamingPart) makeFirstRequest(0); + if (isFirstRequest() && hasStreamingPart) pingServerBeforeStreaming(0); Response response = sendRequestOnce(requestBldr.post(multiPart.build())); @@ -4953,17 +4928,11 @@ public InputStream match(QueryDefinition queryDef, } requestBldr = addTelemetryAgentId(requestBldr); - MediaType mediaType = makeType(requestBldr.build().header(HEADER_CONTENT_TYPE)); - - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::checkFirstRequest); + RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { - try { - retryContext.sleepIfNeeded(); - } catch (InterruptedException e) { - // Ignore interruption - } + retryContext.sleepIfNeeded(); if (queryDef instanceof StructuredQueryDefinition) { response = doPost(null, requestBldr, structure); @@ -5599,14 +5568,32 @@ private void executeRequest(CallResponseImpl responseImpl) { boolean hasStreamingPart = hasStreamingPart(); Consumer resendableConsumer = resendable -> { if (hasStreamingPart) { - checkFirstRequest(); + resetFirstRequestFlag(); throw new ResourceNotResendableException( "Cannot retry request for " + getEndpoint()); } }; Function sendRequestFunction = requestBldr -> { - if (isFirstRequest() && hasStreamingPart) makeFirstRequest(callBaseUri, "", 0); + if (isFirstRequest() && hasStreamingPart) { + // Ping the server before streaming; if unavailable, wait and retry the ping + int pingDelay = pingServer(callBaseUri, "", 0); + int pingRetries = 0; + int maxPingRetries = 10; // Prevent infinite loop + while (pingDelay > 0 && pingRetries < maxPingRetries && !Thread.currentThread().isInterrupted()) { + try { + Thread.sleep(pingDelay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting to ping server before streaming", e); + } + pingRetries++; + pingDelay = pingServer(callBaseUri, "", 0); + } + if (pingRetries >= maxPingRetries) { + logger.warn("Server still unavailable after {} ping attempts before streaming", maxPingRetries); + } + } Response response = sendRequestOnce(requestBldr); if (isFirstRequest()) setFirstRequest(false); return response; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java index 4f0632f34..fe17af67d 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java @@ -33,15 +33,24 @@ class RetryContext { } boolean shouldContinueRetrying(int minAttempts, int maxDelay) { + // Stop retrying if thread has been interrupted + if (Thread.currentThread().isInterrupted()) { + return false; + } return retry < minAttempts || (System.currentTimeMillis() - startTime) < maxDelay; } - void sleepIfNeeded() throws InterruptedException { + void sleepIfNeeded() { if (nextDelay > 0) { if (logger.isDebugEnabled()) { logger.debug("Retrying request after {} ms delay (attempt {})", nextDelay, retry); } - Thread.sleep(nextDelay); + try { + Thread.sleep(nextDelay); + } catch (InterruptedException e) { + // Restore interrupt status for higher-level cancellation/shutdown logic + Thread.currentThread().interrupt(); + } } } From 61f6540c4560a92853d15e4a1131b0ec9a7edc18 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 4 Mar 2026 11:23:12 -0500 Subject: [PATCH 64/70] MLE-27388 Added test for 50x retry --- .../marklogic/client/impl/OkHttpServices.java | 18 +- .../marklogic/client/impl/RetryContext.java | 9 +- .../impl/okhttp/RetryOn50XResponseTest.java | 160 ++++++++++++++++++ .../src/test/resources/logback-test.xml | 4 + 4 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/RetryOn50XResponseTest.java diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index bc2906909..573f89b0b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -523,7 +523,7 @@ private Response sendRequestWithRetry( throw new MarkLogicIOException("Request cancelled: thread was interrupted before request could be sent"); } - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; @@ -1195,7 +1195,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } boolean isResendable = handleBase.isResendable(); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; Headers responseHeaders = null; @@ -1348,7 +1348,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth requestBldr = addVersionHeader(desc, requestBldr, "If-Match"); } - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; Headers responseHeaders = null; @@ -2107,7 +2107,7 @@ void init() { } Response getResponse() { - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, OkHttpServices.this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, OkHttpServices.this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { @@ -2635,7 +2635,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, String connectPath = null; Request.Builder requestBldr = null; - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { @@ -3060,7 +3060,7 @@ public R putResour String outputMimetype = outputBase.getMimetype(); Class as = outputBase.receiveAs(); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { @@ -3223,7 +3223,7 @@ public R postResou String outputMimetype = outputBase != null ? outputBase.getMimetype() : null; Class as = outputBase != null ? outputBase.receiveAs() : null; - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { @@ -3822,7 +3822,7 @@ private U postIt throws ResourceNotFoundException, ResourceNotResendableException, ForbiddenUserException, FailedRequestException { if (params == null) params = new RequestParameters(); if (transaction != null) params.add("txid", transaction.getTransactionId()); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { @@ -4928,7 +4928,7 @@ public InputStream match(QueryDefinition queryDef, } requestBldr = addTelemetryAgentId(requestBldr); - RetryContext retryContext = new RetryContext(logger, RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); + RetryContext retryContext = new RetryContext(RETRYABLE_STATUS_CODES, this::resetFirstRequestFlag); Response response = null; int status = -1; for (; retryContext.shouldContinueRetrying(minRetryAttempts, maxDelayForRetries); retryContext.incrementRetry()) { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java index fe17af67d..67185804a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/RetryContext.java @@ -5,6 +5,7 @@ import com.marklogic.client.FailedRetryException; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Set; @@ -13,7 +14,9 @@ * Tracks retry state, calculates delays, handles sleeping, and logs retry attempts. */ class RetryContext { - private final Logger logger; + + static final private Logger logger = LoggerFactory.getLogger(RetryContext.class); + private final Set retryableStatusCodes; private final Runnable onMaxRetriesCallback; @@ -22,12 +25,10 @@ class RetryContext { private int nextDelay = 0; /** - * @param logger Logger for debug output * @param retryableStatusCodes Set of HTTP status codes that trigger retries * @param onMaxRetriesCallback Callback to invoke when max retries is exceeded (e.g., to reset first request flag) */ - RetryContext(Logger logger, Set retryableStatusCodes, Runnable onMaxRetriesCallback) { - this.logger = logger; + RetryContext(Set retryableStatusCodes, Runnable onMaxRetriesCallback) { this.retryableStatusCodes = retryableStatusCodes; this.onMaxRetriesCallback = onMaxRetriesCallback; } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/RetryOn50XResponseTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/RetryOn50XResponseTest.java new file mode 100644 index 000000000..db943ff4f --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/RetryOn50XResponseTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.impl.okhttp; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.FailedRequestException; +import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; +import com.marklogic.client.impl.RESTServices; +import com.marklogic.client.test.Common; +import okhttp3.Interceptor; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests the retry logic in OkHttpServices using a custom interceptor that simulates server errors. This is not a + * well-documented feature but users are currently able to discover it via the codebase. + */ +class RetryOn50XResponseTest { + + /** + * Custom interceptor that returns 502 Bad Gateway responses and counts how many times it's invoked. + */ + private static class BadGatewayInterceptor implements Interceptor { + + private final AtomicInteger invocationCount = new AtomicInteger(0); + private final int failureCount; + + /** + * @param failureCount Number of times to return 502 before allowing the request through + */ + public BadGatewayInterceptor(int failureCount) { + this.failureCount = failureCount; + } + + @Override + public Response intercept(Chain chain) throws IOException { + int count = invocationCount.incrementAndGet(); + Request request = chain.request(); + + // Fail the first N requests + if (count <= failureCount) { + return new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(502) + .message("Bad Gateway") + .body(okhttp3.ResponseBody.create("Simulated 502 error", null)) + .build(); + } + + // After N failures, let the request through + return chain.proceed(request); + } + + public int getInvocationCount() { + return invocationCount.get(); + } + + public void reset() { + invocationCount.set(0); + } + } + + @BeforeEach + void setUp() { + // Configure very short retry delays for testing + System.setProperty(RESTServices.MAX_DELAY_PROP, "3"); + System.setProperty(RESTServices.MIN_RETRY_PROP, "2"); + + // "Touch" the Common class to trigger the static block that removes existing configurators on + // DatabaseClientFactory. + Common.newServerPayload(); + } + + @AfterEach + void tearDown() { + DatabaseClientFactory.removeConfigurators(); + System.clearProperty(RESTServices.MAX_DELAY_PROP); + System.clearProperty(RESTServices.MIN_RETRY_PROP); + } + + @Test + void testRetryWith502Responses() { + // Create an interceptor that will fail 2 times, then succeed + BadGatewayInterceptor interceptor = new BadGatewayInterceptor(2); + + DatabaseClientFactory.addConfigurator((OkHttpClientConfigurator) builder -> + builder.addInterceptor(interceptor)); + + try (DatabaseClient client = Common.newClient()) { + client.checkConnection(); + assertEquals(3, interceptor.getInvocationCount(), + "Expected 3 invocations: 2 failures followed by 1 success"); + } + } + + @Test + void testRetryExceedsMaxAttempts() { + // Create an interceptor that will fail 10 times (more than minRetry) + BadGatewayInterceptor interceptor = new BadGatewayInterceptor(10); + + DatabaseClientFactory.addConfigurator((OkHttpClientConfigurator) builder -> + builder.addInterceptor(interceptor)); + + try (DatabaseClient client = Common.newClient()) { + assertThrows(FailedRequestException.class, () -> { + client.checkConnection(); + }, "Expected FailedRequestException after exhausting retries"); + + assertTrue(interceptor.getInvocationCount() >= 3, + "Expected at least 3 retry attempts, but got " + interceptor.getInvocationCount()); + } + } + + @Test + void testRetryCountIncreases() { + // Test that retry attempts increase as expected + BadGatewayInterceptor interceptor = new BadGatewayInterceptor(1); + + DatabaseClientFactory.addConfigurator((OkHttpClientConfigurator) builder -> + builder.addInterceptor(interceptor)); + + try (DatabaseClient client = Common.newClient()) { + // First request: fails once, then succeeds + client.checkConnection(); + assertEquals(2, interceptor.getInvocationCount(), "Expected 2 invocations for first request"); + + // Reset and try again + interceptor.reset(); + client.checkConnection(); + assertEquals(2, interceptor.getInvocationCount(), "Expected 2 invocations for second request"); + } + } + + @Test + void testNoRetryOnSuccessfulRequest() { + // Interceptor that never fails + BadGatewayInterceptor interceptor = new BadGatewayInterceptor(0); + + DatabaseClientFactory.addConfigurator((OkHttpClientConfigurator) builder -> + builder.addInterceptor(interceptor)); + + try (DatabaseClient client = Common.newClient()) { + client.checkConnection(); + assertEquals(1, interceptor.getInvocationCount(), + "Expected exactly 1 invocation when request succeeds immediately"); + } + } +} diff --git a/marklogic-client-api/src/test/resources/logback-test.xml b/marklogic-client-api/src/test/resources/logback-test.xml index 890be1792..6a3506a32 100644 --- a/marklogic-client-api/src/test/resources/logback-test.xml +++ b/marklogic-client-api/src/test/resources/logback-test.xml @@ -16,4 +16,8 @@ + + + + From f7bc1a41fa5ecb0ec815a79aba13c4478b89d119 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 4 Mar 2026 12:05:50 -0500 Subject: [PATCH 65/70] MLE-27841 Bumping dependencies before 8.1 release Added a few "since" annotations as well. --- build.gradle | 2 +- examples/build.gradle | 8 ++------ gradle.properties | 5 +++-- marklogic-client-api-functionaltests/build.gradle | 4 ++-- marklogic-client-api/build.gradle | 12 ++++++------ .../com/marklogic/client/expression/VecExpr.java | 7 +++++-- .../client/type/PlanTransitiveClosureOptions.java | 2 ++ ml-development-tools/build.gradle | 6 +++--- test-app/build.gradle | 6 +++--- 9 files changed, 27 insertions(+), 25 deletions(-) diff --git a/build.gradle b/build.gradle index d83bfdc8c..1b494c88d 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ subprojects { all { resolutionStrategy { // Forcing the latest commons-lang3 version to eliminate CVEs. - force "org.apache.commons:commons-lang3:3.19.0" + force "org.apache.commons:commons-lang3:3.20.0" } } } diff --git a/examples/build.gradle b/examples/build.gradle index 3e123799c..dc1b98764 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -1,10 +1,6 @@ -/* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. - */ - dependencies { implementation project(':marklogic-client-api') - implementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" + implementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.5" // The 'api' configuration is used so that the test configuration in marklogic-client-api doesn't have to declare // all of these dependencies. This library project won't otherwise be depended on by anything else as it's not @@ -23,5 +19,5 @@ dependencies { // passes without this on the classpath. exclude module: "commons-beanutils" } - api 'org.apache.commons:commons-lang3:3.19.0' + api 'org.apache.commons:commons-lang3:3.20.0' } diff --git a/gradle.properties b/gradle.properties index d109496b4..ccc8edbe0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,9 +5,10 @@ publishUrl=file:../marklogic-java/releases okhttpVersion=5.3.2 # See https://github.com/FasterXML/jackson for more information on the Jackson libraries. -jacksonVersion=2.20.1 +jacksonVersion=2.21.1 -junitVersion=6.0.1 +junitVersion=6.0.3 +logbackVersion=1.5.32 # Defined at this level so that they can be set as system properties and used by the marklogic-client-api and test-app # project diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index 2dcda2303..6de118eae 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -4,7 +4,7 @@ dependencies { testImplementation project(':marklogic-client-api') - testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" + testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.5" testImplementation 'org.skyscreamer:jsonassert:1.5.3' testImplementation 'org.slf4j:slf4j-api:2.0.17' testImplementation 'commons-io:commons-io:2.21.0' @@ -17,7 +17,7 @@ dependencies { testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" - testImplementation 'ch.qos.logback:logback-classic:1.5.19' + testImplementation "ch.qos.logback:logback-classic:${logbackVersion}" testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation 'org.xmlunit:xmlunit-legacy:2.11.0' diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index b292e3c79..3b888233e 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -17,7 +17,7 @@ dependencies { // This is now an implementation dependency as opposed to an api dependency in 7.x and earlier. // The only time it appears in the public API is when a user uses JAXBHandle. // But in that scenario, the user would already be using JAXB in their application. - implementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" + implementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.5" implementation "org.glassfish.jaxb:jaxb-runtime:4.0.6" implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" @@ -40,14 +40,14 @@ dependencies { // Dependencies for hash generation. Can be safely omitted if not using the incremental write feature. But neither // has any transitive dependencies, and thus their impact on the dependency tree is minimal. implementation "io.github.erdtman:java-json-canonicalization:1.1" - implementation "net.openhft:zero-allocation-hashing:0.27ea1" + implementation "net.openhft:zero-allocation-hashing:2026.0" // Only used by extras (which some examples then depend on) compileOnly 'org.jdom:jdom2:2.0.6.1' compileOnly 'org.dom4j:dom4j:2.2.0' compileOnly 'com.google.code.gson:gson:2.13.2' - testImplementation "com.marklogic:marklogic-junit5:2.0-SNAPSHOT" + testImplementation "com.marklogic:marklogic-junit5:2.0.0" testImplementation 'org.xmlunit:xmlunit-legacy:2.11.0' testImplementation project(':examples') @@ -57,13 +57,13 @@ dependencies { // Allows talking to the Manage API. testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" - testImplementation "org.mockito:mockito-core:5.21.0" - testImplementation "org.mockito:mockito-inline:5.2.0" + testImplementation "org.mockito:mockito-core:5.22.0" +// testImplementation "org.mockito:mockito-inline:5.2.0" testImplementation "com.squareup.okhttp3:mockwebserver3:${okhttpVersion}" testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" - testImplementation 'ch.qos.logback:logback-classic:1.5.23' + testImplementation "ch.qos.logback:logback-classic:${logbackVersion}" // Using this to avoid a schema validation issue with the regular xercesImpl testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java index 3942c4117..352de5209 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/VecExpr.java @@ -151,7 +151,7 @@ public interface VecExpr { * Returns a new vector which is a copy of the input vector with reduced precision. The precision reduction is achieved by clearing the bottom (32 - precision) bits of the mantissa for each dimension's float value. This can be useful for reducing storage requirements or for creating approximate vector representations. * * - + *

* Provides a client interface to the vec:precision server function. * @param vector The input vector to reduce precision. Can be a vector or an empty sequence. (of vec:vector) @@ -207,7 +207,7 @@ public interface VecExpr { * Returns a new vector which is a copy of the input vector with each element truncated to a specific number of digits. * * - + *

* Provides a client interface to the vec:trunc server function. * @param vector The input vector to truncate. (of vec:vector) @@ -253,6 +253,8 @@ public interface VecExpr { * @param score The cts:score of the matching document. (of xs:unsignedInt) * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) * @return a server expression with the xs:unsignedLong server data type + * + * @since 8.1.0, requires MarkLogic 12.1 */ public ServerExpression vectorScore(ServerExpression score, double distance); /** @@ -262,6 +264,7 @@ public interface VecExpr { * @param score The cts:score of the matching document. (of xs:unsignedInt) * @param distance The distance between the vector in the matching document and the query vector. Examples, the result of a call to ovec:cosine-distance() or ovec:euclidean-distance(). (of xs:double) * @return a server expression with the xs:unsignedLong server data type + * @since 8.1.0, requires MarkLogic 12.1 */ public ServerExpression vectorScore(ServerExpression score, ServerExpression distance); /** diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java index 3772571d9..67f1e0ccc 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanTransitiveClosureOptions.java @@ -8,6 +8,8 @@ /** * Options for controlling transitive closure operations, including minimum and maximum * path lengths. + * + * @since 8.1.0 */ public interface PlanTransitiveClosureOptions { XsLongVal getMinLength(); diff --git a/ml-development-tools/build.gradle b/ml-development-tools/build.gradle index 37836848e..294969679 100644 --- a/ml-development-tools/build.gradle +++ b/ml-development-tools/build.gradle @@ -9,21 +9,21 @@ plugins { id 'maven-publish' id "com.gradle.plugin-publish" version "1.2.1" id "java-gradle-plugin" - id 'org.jetbrains.kotlin.jvm' version '2.1.0' + id 'org.jetbrains.kotlin.jvm' version '2.2.21' } dependencies { compileOnly gradleApi() // This is a runtime dependency of marklogic-client-api but is needed for compiling. - compileOnly "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" + compileOnly "jakarta.xml.bind:jakarta.xml.bind-api:4.0.5" // Gradle 9 does not like for a plugin to have a project dependency; trying to publish it results in a // NoSuchMethodError pertaining to getProjectDependency. So treating this as a 3rd party dependency. This creates // additional work during development, though we rarely modify the code in this plugin anymore. implementation "com.marklogic:marklogic-client-api:${version}" - implementation 'org.jetbrains.kotlin:kotlin-stdlib:2.2.20' + implementation 'org.jetbrains.kotlin:kotlin-stdlib:2.2.21' implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${jacksonVersion}" // Sticking with this older version for now as the latest 1.x version introduces breaking changes. diff --git a/test-app/build.gradle b/test-app/build.gradle index a5a70c96e..7ed3929b8 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -31,10 +31,10 @@ mlWaitTillReady { } dependencies { - implementation "io.undertow:undertow-core:2.3.20.Final" - implementation "io.undertow:undertow-servlet:2.3.20.Final" + implementation "io.undertow:undertow-core:2.3.23.Final" + implementation "io.undertow:undertow-servlet:2.3.23.Final" implementation 'org.slf4j:slf4j-api:2.0.17' - implementation 'ch.qos.logback:logback-classic:1.5.23' + implementation "ch.qos.logback:logback-classic:${logbackVersion}" implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } From 46d1b409f30c75e86ab4c2eb3add5b135047ca9a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 4 Mar 2026 14:01:14 -0500 Subject: [PATCH 66/70] MLE-27841 More 'since' annotations --- .../java/com/marklogic/client/expression/PlanBuilder.java | 8 ++++++++ .../com/marklogic/client/expression/PlanBuilderBase.java | 1 + 2 files changed, 9 insertions(+) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java index 14c91d5e7..4ce05261e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilder.java @@ -249,6 +249,8 @@ protected PlanBuilder( /** * Create column definitions which can be used in op:from-docs. Below functions are used to create column definitions. op:add-column, op:type, op:xpath, op:expr, op:nullable, op:default, op:dimension, op:coordinate-system, op:units, op:collation. * @return a PlanColumnBuilder object + * + * @since 8.1.0; requires MarkLogic 12.1 or higher */ public abstract PlanColumnBuilder columnBuilder(); /** @@ -1189,6 +1191,8 @@ protected PlanBuilder( /** * This helper function returns the node from the current processing row. It is to be used in op:xpath, to reference the 'current item' instead of a doc column. * @return a PlanContextExprCall object + * + * @since 8.1.0; requires MarkLogic 12.1 or higher */ public abstract PlanContextExprCall context(); /** @@ -1236,6 +1240,8 @@ protected PlanBuilder( * @param expression The server expression (such as op:context()) from which to extract the child nodes. * @param path An XPath (specified as a string) to apply to each node. (of xs:string) * @return a server expression with the node server data type + * + * @since 8.1.0; requires MarkLogic 12.1 or higher */ public abstract ServerExpression xpath(ServerExpression expression, String path); /** @@ -1245,6 +1251,8 @@ protected PlanBuilder( * @param expression The server expression (such as op:context()) from which to extract the child nodes. * @param path An XPath to apply to each node. (of xs:string) * @return a server expression with the node server data type + * + * @since 8.1.0; requires MarkLogic 12.1 or higher */ public abstract ServerExpression xpath(ServerExpression expression, ServerExpression path); /** diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java index 378f7ae43..669730712 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/expression/PlanBuilderBase.java @@ -362,6 +362,7 @@ public interface PlanBuilderBase { * operator. Use the fluent methods of the transitive closure option object * to set the configuration. * @return the configuration object + * @since 8.1.0; requires MarkLogic 12.1 or higher */ PlanTransitiveClosureOptions transitiveClosureOptions(); From bdca77603d54c7ba686082f111808e9cee1b0911 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 11 Mar 2026 16:04:21 -0400 Subject: [PATCH 67/70] MLE-27481 Small Polaris fix --- .../java/com/marklogic/client/impl/OkHttpServices.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index 573f89b0b..86e912a96 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -2643,7 +2643,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, Object nextValue = (handle != null) ? handle.sendContent() : value; - RequestBody sentValue = null; + RequestBody sentValue; if (nextValue instanceof OutputStreamSender) { sentValue = new StreamingOutputImpl( (OutputStreamSender) nextValue, reqlog, mediaType); @@ -2676,9 +2676,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, requestBldr = addTelemetryAgentId(requestBldr); } - response = (sentValue == null) ? - sendRequestOnce(requestBldr.put(null).build()) : - sendRequestOnce(requestBldr.put(sentValue).build()); + response = sendRequestOnce(requestBldr.put(sentValue).build()); } else if ("post".equals(method)) { if (requestBldr == null) { connectPath = type; @@ -2688,9 +2686,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, requestBldr = addTelemetryAgentId(requestBldr); } - response = (sentValue == null) ? - sendRequestOnce(requestBldr.post(RequestBody.create("", null)).build()) : - sendRequestOnce(requestBldr.post(sentValue).build()); + response = sendRequestOnce(requestBldr.post(sentValue).build()); } else { throw new MarkLogicInternalException("unknown method type " + method); From 315ecd76ad2e9e1a339c0a7c6ac99123467f3dd1 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 12 Mar 2026 13:23:13 -0400 Subject: [PATCH 68/70] MLE-27481 Small tweaks pre release, plus NOTICE file --- NOTICE.txt | 960 +++++++++++++----- marklogic-client-api/build.gradle | 1 - .../client/type/PlanColumnBuilder.java | 2 + .../client/type/PlanContextExprCall.java | 2 + 4 files changed, 713 insertions(+), 252 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 700be2e15..a435604f3 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,309 +1,767 @@ -MarkLogic® for Java Client +Progress MarkLogic Java Client + +Portions of the Products include certain open source and commercial third-party +components listed below ("Third-Party Components"). The authors of the +Third-Party Components require Progress Software Corporation ("PSC") to include +the following notices and additional licensing terms as a condition of PSC's +use of such Third-Party Components. You acknowledge that the authors of the +Third-Party Components have no obligation to provide support to you for the +Third-Party Components or the Product. You hereby undertake to comply with all +licenses related to the applicable Third-Party Components. Notwithstanding +anything to the contrary, to the extent that any of the terms and conditions of +the Progress Agreement conflict, vary, or are in addition to the terms and +conditions of the aforementioned third-party licenses for these technologies, +such terms and conditions are offered by PSC alone and not by any other party. + +-------------------------------------------------------------------------------- +SUMMARY OF COMPONENTS: + +Vendor Name | Product Name | Version + +Apache License 2.0: + Chronicle Software : Open Source | Zero-allocation Hashing | 2026.0 + FasterXML | jackson-annotations | 2.21 + FasterXML | jackson-bom | 2.21.1 + FasterXML | jackson-core | 2.21.1 + FasterXML | jackson-databind | 2.21.1 + FasterXML | jackson-dataformat-csv | 2.21.1 + FasterXML | jackson-dataformat-yaml | 2.21.1 + FasterXML | jackson-module-kotlin | 2.21.1 + JetBrains | kotlin-reflect | 2.1.21 + Jetbrains | Jetbrains annotations | 13.0 + Samuel Erdtman | java-json-canonicalization | 1.1 + Square | OkHttp | 5.3.2 + Square | OkHttp Logging Interceptor | 5.3.2 + Square | squareokio | 3.16.4 + github.com/rburgst | okhttp-digest | 3.1.1 + +BSD 3-Clause "New" or "Revised" License: + Oracle and/or its affiliates | Jakarta Activation | 2.0.1 + +Eclipse Distribution License - v 1.0: + Eclipse Foundation | Angus Activation Registries | 2.0.3 + Eclipse Foundation | JAXB CORE | 4.0.6 + Eclipse Foundation | JAXB Runtime | 4.0.6 + Eclipse Foundation | Jakarta Activation API | 2.1.4 + Eclipse Foundation | TXW2 Runtime | 4.0.6 + Eclipse Foundation | istack common utility code runtime | 4.1.2 + Eclipse Foundation | jakarta.xml.bind:jakarta.xml.bind-api | 4.0.5 + +Eclipse Public License 2.0: + Oracle | Jakarta Mail | 2.0.2 + +MIT License: + QOS.ch | SLF4J API Module | 2.0.17 + +================================================================================ + +The Product contains Jakarta Mail 2.0.2. These third-party technologies are +licensed to PSC and User is subject to the terms of a third-party license, a +copy of which is included herein. PSC will, at Licensee's request, provide a +copy of the source code for this third-party technology, including +modifications, if any, made by PSC. PSC may charge reasonable shipping and +handling charges for such distribution. Licensee may also obtain the source +code for these third-party technologies through +http://iue.progress.com/3dpartysoftwares/Pages/default.aspx by following the +instructions set forth therein. + +1. Special Notices Regarding Open-Source Third-Party Components incorporated +into the Product. Such technology is subject to the following terms and +conditions: + +(1) Apache License 2.0: + +Progress MarkLogic Java Client incorporates Jetbrains annotations +13.0, OkHttp 5.3.2, OkHttp Logging Interceptor 5.3.2, Zero-allocation Hashing +2026.0, jackson-annotations 2.21, jackson-bom 2.21.1, jackson-core 2.21.1, +jackson-databind 2.21.1, jackson-dataformat-csv 2.21.1, jackson-dataformat-yaml +2.21.1, jackson-module-kotlin 2.21.1, java-json-canonicalization 1.1, +kotlin-reflect 2.1.21, okhttp-digest 3.1.1, and squareokio 3.16.4. + + Apache License Version 2.0, January 2004 + ========================= + + + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, and + distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by the + copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all other + entities that control, are controlled by, or are under common control with + that entity. For the purposes of this definition, "control" means (i) the + power, direct or indirect, to cause the direction or management of such + entity, whether by contract or otherwise, or (ii) ownership of fifty percent + (50%) or more of the outstanding shares, or (iii) beneficial ownership of + such entity. + + "You" (or "Your") shall mean an individual or Legal Entity exercising + permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation source, and + configuration files. + + "Object" form shall mean any form resulting from mechanical transformation or + translation of a Source form, including but not limited to compiled object + code, generated documentation, and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or Object form, + made available under the License, as indicated by a copyright notice that is + included in or attached to the work (an example is provided in the Appendix + below). + + "Derivative Works" shall mean any work, whether in Source or Object form, + that is based on (or derived from) the Work and for which the editorial + revisions, annotations, elaborations, or other modifications represent, as a + whole, an original work of authorship. For the purposes of this License, + Derivative Works shall not include works that remain separable from, or + merely link (or bind by name) to the interfaces of, the Work and Derivative + Works thereof. + + "Contribution" shall mean any work of authorship, including the original + version of the Work and any modifications or additions to that Work or + Derivative Works thereof, that is intentionally submitted to Licensor for + inclusion in the Work by the copyright owner or by an individual or Legal + Entity authorized to submit on behalf of the copyright owner. For the + purposes of this definition, "submitted" means any form of electronic, + verbal, or written communication sent to the Licensor or its representatives, + including but not limited to communication on electronic mailing lists, + source code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and improving the + Work, but excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity on + behalf of whom a Contribution has been received by Licensor and subsequently + incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of this + License, each Contributor hereby grants to You a perpetual, worldwide, + non-exclusive, no-charge, royalty-free, irrevocable copyright license to + reproduce, prepare Derivative Works of, publicly display, publicly perform, + sublicense, and distribute the Work and such Derivative Works in Source or + Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of this + License, each Contributor hereby grants to You a perpetual, worldwide, + non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this + section) patent license to make, have made, use, offer to sell, sell, import, + and otherwise transfer the Work, where such license applies only to those + patent claims licensable by such Contributor that are necessarily infringed + by their Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You institute + patent litigation against any entity (including a cross-claim or counterclaim + in a lawsuit) alleging that the Work or a Contribution incorporated within + the Work constitutes direct or contributory patent infringement, then any + patent licenses granted to You under this License for that Work shall + terminate as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the Work or + Derivative Works thereof in any medium, with or without modifications, and in + Source or Object form, provided that You meet the following conditions: + + a. You must give any other recipients of the Work or Derivative Works a + copy of + this License; and + + b. You must cause any modified files to carry prominent notices stating that + You changed the files; and + + c. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices + from the Source form of the Work, excluding those notices that do not + pertain to any part of the Derivative Works; and + + d. If the Work includes a "NOTICE" text file as part of its distribution, + then + any Derivative Works that You distribute must include a readable copy of + the attribution notices contained within such NOTICE file, excluding + those notices that do not pertain to any part of the Derivative Works, in + at least one of the following places: within a NOTICE text file + distributed as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, within a + display generated by the Derivative Works, if and wherever such + third-party notices normally appear. The contents of the NOTICE file are + for informational purposes only and do not modify the License. You may + add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the Work, + provided that such additional attribution notices cannot be construed as + modifying the License. + + You may add Your own copyright statement to Your modifications and may + provide additional or different license terms and conditions for use, + reproduction, or distribution of Your modifications, or for any such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions stated in + this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, any + Contribution intentionally submitted for inclusion in the Work by You to the + Licensor shall be under the terms and conditions of this License, without any + additional terms or conditions. Notwithstanding the above, nothing herein + shall supersede or modify the terms of any separate license agreement you may + have executed with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade names, + trademarks, service marks, or product names of the Licensor, except as + required for reasonable and customary use in describing the origin of the + Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in + writing, Licensor provides the Work (and each Contributor provides its + Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied, including, without limitation, any + warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or + FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining + the appropriateness of using or redistributing the Work and assume any risks + associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, whether in + tort (including negligence), contract, or otherwise, unless required by + applicable law (such as deliberate and grossly negligent acts) or agreed to + in writing, shall any Contributor be liable to You for damages, including any + direct, indirect, special, incidental, or consequential damages of any + character arising as a result of this License or out of the use or inability + to use the Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all other + commercial damages or losses), even if such Contributor has been advised of + the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing the Work + or Derivative Works thereof, You may choose to offer, and charge a fee for, + acceptance of support, warranty, indemnity, or other liability obligations + and/or rights consistent with this License. However, in accepting such + obligations, You may act only on Your own behalf and on Your sole + responsibility, not on behalf of any other Contributor, and only if You agree + to indemnify, defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason of your + accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work + + To apply the Apache License to your work, attach the following boilerplate + notice, with the fields enclosed by brackets "[]" replaced with your own + identifying information. (Don't include the brackets!) The text should be + enclosed in the appropriate comment syntax for the file format. We also + recommend that a file or class name and description of purpose be included on + the same "printed page" as the copyright notice for easier identification + within third-party archives. + + Copyright [yyyy] [name of copyright owner] Licensed under the Apache + License, Version 2.0 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + law or agreed to in writing, software distributed under the License is + distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the specific language + governing permissions and limitations under the License. + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Contents of NOTICE text file for jackson-annotations 2.21 and jackson-databind +2.21.1: + + # Jackson JSON processor -Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + Jackson is a high-performance, Free/Open Source JSON processing library. It + was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. It is currently developed by a community of + developers. + + ## Copyright -To the extent required by the applicable open-source license, a complete machine-readable copy of the source code -corresponding to such code is available upon request. This offer is valid to anyone in receipt of this information and -shall expire three years following the date of the final distribution of this product version by Progress Software -Corporation. To obtain such source code, send an email to Legal-thirdpartyreview@progress.com. Please specify the -product and version for which you are requesting source code. + Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) -Third Party Notices + ## Licensing -jackson-databind 2.20.0 (Apache-2.0) -jackson-dataformat-csv 2.20.0 (Apache-2.0) -okhttp 5.2.0 (Apache-2.0) -logging-interceptor 5.2.0 (Apache-2.0) -jakarta.mail 2.0.2 (EPL-1.0) -okhttp-digest 3.1.1 (Apache-2.0) -jakarta.xml.bind-api 4.0.4 (EPL-1.0) -jaxb-runtime 4.0.6 (CDDL-1.1) -slf4j-api 2.0.17 (Apache-2.0) + Jackson 2.x core and extension components are licensed under Apache License + 2.0 To find the details that apply to this artifact see the accompanying + LICENSE file. -Common Licenses + ## Credits -Apache License 2.0 (Apache-2.0) -Common Development and Distribution License 1.1 (CDDL-1.1) -Eclipse Public License 1.0 (EPL-1.0) + A list of contributors may be found from CREDITS(-2.x) file, which is + included in some artifacts (usually source distributions); but is always + available from the source code management (SCM) system project uses. -Third-Party Components + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Contents of NOTICE text file for jackson-core 2.21.1: -The following is a list of the third-party components used by the MarkLogic® for Java Client 8.0.0 (last updated October 29, 2025): + # Jackson JSON processor -jackson-databind 2.20.0 (Apache-2.0) -https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + Jackson is a high-performance, Free/Open Source JSON processing library. It + was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. It is currently developed by a community of + developers. -jackson-dataformat-csv 2.20.0 (Apache-2.0) -https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-csv/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + ## Copyright -okhttp 5.2.0 (Apache-2.0) -https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) -logging-interceptor 5.2.0 (Apache-2.0) -https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + ## Licensing -jakarta.mail 2.0.2 (Apache-2.0) -https://repo1.maven.org/maven2/com/sun/mail/jakarta.mail/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + Jackson 2.x core and extension components are licensed under Apache License + 2.0 To find the details that apply to this artifact see the accompanying + LICENSE file. -okhttp-digest 3.1.1 (Apache-2.0) -https://repo1.maven.org/maven2/io/github/rburgst/okhttp-digest/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + ## Credits -jakarta.xml.bind-api 4.0.4 (Apache-2.0) -https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + A list of contributors may be found from CREDITS(-2.x) file, which is + included in some artifacts (usually source distributions); but is always + available from the source code management (SCM) system project uses. -jaxb-runtime 4.0.6 (Apache-2.0) -https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + ## FastDoubleParser -slf4j-api 2.0.17 (Apache-2.0) -https://repo1.maven.org/maven2/org/slf4j/slf4j-api/ -For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0) + jackson-core bundles a shaded copy of FastDoubleParser + . That code is available + under an MIT license + under the following copyright. -Common Licenses + Copyright © 2023 Werner Randelshofer, Switzerland. MIT License. -The following is a list of the third-party components used by the MarkLogic® for Java Client 8.0.0 (last updated October 29, 2025): + See FastDoubleParser-NOTICE for details of other source code included in + FastDoubleParser and the licenses and copyrights that apply to that code. -Apache License 2.0 (Apache-2.0) -https://spdx.org/licenses/Apache-2.0.html + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Contents of NOTICE text file for jackson-dataformat-csv 2.21.1 and +jackson-dataformat-yaml 2.21.1: -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ + # Jackson JSON processor -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + Jackson is a high-performance, Free/Open Source JSON processing library. It + was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. It is currently developed by a community of + developers. -1. Definitions. + ## Copyright -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + ## Licensing -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + Jackson components are licensed under Apache (Software) License, version 2.0, + as per accompanying LICENSE file. -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + ## Credits -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + A list of contributors may be found from CREDITS file, which is included in + some artifacts (usually source distributions); but is always available from + the source code management (SCM) system project uses. -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Contents of NOTICE text file for jackson-module-kotlin 2.21.1: -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + # Jackson JSON processor -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + Jackson is a high-performance, Free/Open Source JSON processing library. It + was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. It is currently developed by a community of + developers, as well as supported commercially by FasterXML.com. -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + ## Copyright -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + ## Licensing -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + Jackson core and extension components may be licensed under different + licenses. To find the details that apply to this artifact see the + accompanying LICENSE file. For more information, including possible other + licensing options, contact FasterXML.com (http://fasterxml.com). -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + ## Credits - (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and + A list of contributors may be found from CREDITS file, which is included in + some artifacts (usually source distributions); but is always available from + the source code management (SCM) system project uses. - (b) You must cause any modified files to carry prominent notices stating that You changed the files; and +(2) BSD 3-Clause "New" or "Revised" License: - (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +Progress MarkLogic Java Client incorporates Jakarta Activation 2.0.1. - (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + Copyright (c) , Oracle and/or its affiliates All rights reserved. - You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + * Redistributions of source code must retain the above copyright notice, + this + list of conditions and the following disclaimer. -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + * Neither the name of the com.sun.activation nor the names of its + contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. -END OF TERMS AND CONDITIONS +(3) Eclipse Distribution License - v 1.0: -APPENDIX: How to apply the Apache License to your work. +Progress MarkLogic Java Client incorporates Angus Activation +Registries 2.0.3, JAXB CORE 4.0.6, JAXB Runtime 4.0.6, Jakarta Activation API +2.1.4, TXW2 Runtime 4.0.6, istack common utility code runtime 4.1.2, and +jakarta.xml.bind:jakarta.xml.bind-api 4.0.5. -To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. + Eclipse Distribution License - v 1.0 + ==================================== -Copyright [yyyy] [name of copyright owner] + Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + All rights reserved. -http://www.apache.org/licenses/LICENSE-2.0 + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + * Redistributions of source code must retain the above copyright notice, + this + list of conditions and the following disclaimer. -==================== + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. -Common Development and Distribution License 1.1 (CDDL-1.1) -https://spdx.org/licenses/CDDL-1.1.html + * Neither the name of the Eclipse Foundation, Inc. nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) -Version 1.1 + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +(4) Eclipse Public License 2.0: -1. Definitions. -1.1. "Contributor" means each individual or entity that creates or contributes to the creation of Modifications. -1.2. "Contributor Version" means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. -1.3. "Covered Software" means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. -1.4. "Executable" means the Covered Software in any form other than Source Code. -1.5. "Initial Developer" means the individual or entity that first makes Original Software available under this License. -1.6. "Larger Work" means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. -1.7. "License" means this document. -1.8. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. -1.9. "Modifications" means the Source Code and Executable form of any of the following: -A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; -B. Any new file that contains any part of the Original Software or previous Modification; or -C. Any new file that is contributed or otherwise made available under the terms of this License. -1.10. "Original Software" means the Source Code and Executable form of computer software code that is originally released under this License. -1.11. "Patent Claims" means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. -1.12. "Source Code" means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. -1.13. "You" (or "Your") means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. -2. License Grants. -2.1. The Initial Developer Grant. -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). -(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. -(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. -2.2. Contributor Grant. -Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. -3. Distribution Obligations. -3.1. Availability of Source Code. -Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. -3.2. Modifications. -The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. -3.3. Required Notices. -You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients' rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. -3.5. Distribution of Executable Versions. -You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient's rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. -3.6. Larger Works. -You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. -4. Versions of the License. -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. -4.3. Modified Versions. -When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. -5. DISCLAIMER OF WARRANTY. -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. -6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. -6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as "Participant") alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. -6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. -6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -8. U.S. GOVERNMENT END USERS. -The Covered Software is a "commercial item," as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer software" (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and "commercial computer software documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. - -9. MISCELLANEOUS. -This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys' fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. - -NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) -The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. - -==================== - -Eclipse Public License 1.0 (EPL-1.0) -https://spdx.org/licenses/EPL-1.0.html - -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and - b) in the case of each subsequent Contributor: - i) changes to the Program, and - ii) additions to the Program; - -where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. - -2. GRANT OF RIGHTS - - a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. - - b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. - - c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. - - d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. - -3. REQUIREMENTS -A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: - - a) it complies with the terms and conditions of this Agreement; and - - b) its license agreement: - i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; - ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; - iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and - iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. - -When the Program is made available in source code form: - - a) it must be made available under this Agreement; and - - b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the Program. - -Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION -Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. - -5. NO WARRANTY -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. +Progress MarkLogic Java Client incorporates Jakarta Mail 2.0.2. + + Eclipse Public License - v 2.0 + ============================== + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC + LICENSE (“AGREEMENT”). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM + CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + + 1. DEFINITIONS + -------------- + + “Contribution” means: + + a) in the case of the initial Contributor, the initial content Distributed + under this Agreement, and b) in the case of each subsequent Contributor: + + i) changes to the Program, and ii) additions to the Program; + + where such changes and/or additions to the Program originate from and are + Distributed by that particular Contributor. A Contribution “originates” + from a Contributor if it was added to the Program by such Contributor + itself or anyone acting on such Contributor's behalf. Contributions do not + include changes or additions to the Program that are not Modified Works. + + “Contributor” means any person or entity that Distributes the Program. + + “Licensed Patents” mean patent claims licensable by a Contributor which are + necessarily infringed by the use or sale of its Contribution alone or when + combined with the Program. + + “Program” means the Contributions Distributed in accordance with this + Agreement. + + “Recipient” means anyone who receives the Program under this Agreement or any + Secondary License (as applicable), including Contributors. + + “Derivative Works” shall mean any work, whether in Source Code or other form, + that is based on (or derived from) the Program and for which the editorial + revisions, annotations, elaborations, or other modifications represent, as a + whole, an original work of authorship. + + “Modified Works” shall mean any work in Source Code or other form that + results from an addition to, deletion from, or modification of the contents + of the Program, including, for purposes of clarity any new file in Source + Code form that contains any contents of the Program. Modified Works shall not + include works that contain only declarations, interfaces, types, classes, + structures, or files of the Program solely in each case in order to link to, + bind by name, or subclass the Program or Modified Works thereof. + + “Distribute” means the acts of a) distributing or b) making available in any + manner that enables the transfer of a copy. + + “Source Code” means the form of a Program preferred for making modifications, + including but not limited to software source code, documentation source, and + configuration files. + + “Secondary License” means either the GNU General Public License, Version 2.0, + or any later versions of that license, including any exceptions or additional + permissions as identified by the initial Contributor. + + + 2. GRANT OF RIGHTS + ------------------ + + a) Subject to the terms of this Agreement, each Contributor hereby grants + Recipient a non-exclusive, worldwide, royalty-free copyright license to + reproduce, prepare Derivative Works of, publicly display, publicly perform, + Distribute and sublicense the Contribution of such Contributor, if any, and + such Derivative Works. b) Subject to the terms of this Agreement, each + Contributor hereby grants Recipient a non-exclusive, worldwide, + royalty-free patent license under Licensed Patents to make, use, sell, + offer to sell, import and otherwise transfer the Contribution of such + Contributor, if any, in Source Code or other form. This patent license + shall apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition of the + Contribution causes such combination to be covered by the Licensed Patents. + The patent license shall not apply to any other combinations which include + the Contribution. No hardware per se is licensed hereunder. c) Recipient + understands that although each Contributor grants the licenses to its + Contributions set forth herein, no assurances are provided by any + Contributor that the Program does not infringe the patent or other + intellectual property rights of any other entity. Each Contributor + disclaims any liability to Recipient for claims brought by any other entity + based on infringement of intellectual property rights or otherwise. As a + condition to exercising the rights and licenses granted hereunder, each + Recipient hereby assumes sole responsibility to secure any other + intellectual property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the Program, it + is Recipient's responsibility to acquire that license before distributing + the Program. d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant the + copyright license set forth in this Agreement. e) Notwithstanding the terms + of any Secondary License, no Contributor makes additional grants to any + Recipient (other than those set forth in this Agreement) as a result of + such Recipient's receipt of the Program under the terms of a Secondary + License (if permitted under the terms of Section 3). + + + 3. REQUIREMENTS + --------------- + + 3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in accordance + with section 3.2, and the Contributor must accompany the Program with a + statement that the Source Code for the Program is available under this + Agreement, and informs Recipients how to obtain it in a reasonable manner + on or through a medium customarily used for software exchange; and b) the + Contributor may Distribute the Program under a license different than this + Agreement, provided that such license: + + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including warranties or + conditions of title and non-infringement, and implied warranties or + conditions of merchantability and fitness for a particular purpose; ii) + effectively excludes on behalf of all other Contributors all liability + for damages, including direct, indirect, special, incidental and + consequential damages, such as lost profits; iii) does not attempt to + limit or alter the recipients' rights in the Source Code under section + 3.2; and iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements of this + section 3. + + 3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the Program + + (i) is combined with other material in a separate file or files made + available under a Secondary License, and (ii) the initial Contributor + attached to the Source Code the notice described in Exhibit A of this + Agreement, then the Program may be made available under the terms of such + Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of the Program. + + 3.3 Contributors may not remove or alter any copyright, patent, trademark, + attribution notices, disclaimers of warranty, or limitations of liability + (‘notices’) contained within the Program from any copy of the Program which + they Distribute, provided that Contributors may add their own appropriate + notices. + + + 4. COMMERCIAL DISTRIBUTION + -------------------------- + + Commercial distributors of software may accept certain responsibilities with + respect to end users, business partners and the like. While this license is + intended to facilitate the commercial use of the Program, the Contributor who + includes the Program in a commercial product offering should do so in a + manner which does not create potential liability for other Contributors. + Therefore, if a Contributor includes the Program in a commercial product + offering, such Contributor (“Commercial Contributor”) hereby agrees to defend + and indemnify every other Contributor (“Indemnified Contributor”) against any + losses, damages and costs (collectively “Losses”) arising from claims, + lawsuits and other legal actions brought by a third party against the + Indemnified Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the Program in + a commercial product offering. The obligations in this section do not apply + to any claims or Losses relating to any actual or alleged intellectual + property infringement. In order to qualify, an Indemnified Contributor must: + a) promptly notify the Commercial Contributor in writing of such claim, and + b) allow the Commercial Contributor to control, and cooperate with the + Commercial Contributor in, the defense and any related settlement + negotiations. The Indemnified Contributor may participate in any such claim + at its own expense. + + For example, a Contributor might include the Program in a commercial product + offering, Product X. That Contributor is then a Commercial Contributor. If + that Commercial Contributor then makes performance claims, or offers + warranties related to Product X, those performance claims and warranties are + such Commercial Contributor's responsibility alone. Under this section, the + Commercial Contributor would have to defend claims against the other + Contributors related to those performance claims and warranties, and if a + court requires any other Contributor to pay any damages as a result, the + Commercial Contributor must pay those damages. + + + 5. NO WARRANTY + -------------- + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED + BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN “AS IS” BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, + WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, + MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely + responsible for determining the appropriateness of using and distributing the + Program and assumes all risks associated with its exercise of rights under + this Agreement, including but not limited to the risks and costs of program + errors, compliance with applicable laws, damage to or loss of data, programs + or equipment, and unavailability or interruption of operations. + + + 6. DISCLAIMER OF LIABILITY + -------------------------- + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED + BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY + LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED + HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + + 7. GENERAL + ---------- + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of the + remainder of the terms of this Agreement, and without further action by the + parties hereto, such provision shall be reformed to the minimum extent + necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Program itself + (excluding combinations of the Program with other software or hardware) + infringes such Recipient's patent(s), then such Recipient's rights granted + under Section 2(b) shall terminate as of the date such litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it fails to + comply with any of the material terms or conditions of this Agreement and + does not cure such failure in a reasonable period of time after becoming + aware of such noncompliance. If all Recipient's rights under this Agreement + terminate, Recipient agrees to cease use and distribution of the Program as + soon as reasonably practicable. However, Recipient's obligations under this + Agreement and any licenses granted by Recipient relating to the Program shall + continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, but in + order to avoid inconsistency the Agreement is copyrighted and may only be + modified in the following manner. The Agreement Steward reserves the right to + publish new versions (including revisions) of this Agreement from time to + time. No one other than the Agreement Steward has the right to modify this + Agreement. The Eclipse Foundation is the initial Agreement Steward. The + Eclipse Foundation may assign the responsibility to serve as the Agreement + Steward to a suitable separate entity. Each new version of the Agreement will + be given a distinguishing version number. The Program (including + Contributions) may always be Distributed subject to the version of the + Agreement under which it was received. In addition, after a new version of + the Agreement is published, Contributor may elect to Distribute the Program + (including its Contributions) under the new version. + + Except as expressly stated in Sections 2(a) and 2(b) above, Recipient + receives no rights or licenses to the intellectual property of any + Contributor under this Agreement, whether expressly, by implication, estoppel + or otherwise. All rights in the Program not expressly granted under this + Agreement are reserved. Nothing in this Agreement is intended to be + enforceable by any entity that is not a Contributor or Recipient. No + third-party beneficiary rights are created under this Agreement. + + + Exhibit A – Form of Secondary Licenses Notice + --------------------------------------------- + + “This Source Code may also be made available under the following Secondary + Licenses when the conditions for such availability set forth in the Eclipse + Public License, v. 2.0 are satisfied: {name license(s), version(s), and + exceptions or additional permissions here}.” + + Simply including a copy of this Agreement, including this Exhibit A is not + sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular file, + then You may include the notice in a location (such as a LICENSE file in a + relevant directory) where a recipient would be likely to look for such a + notice. + + You may add additional accurate notices of copyright ownership. + +(5) MIT License: + +Progress MarkLogic Java Client incorporates SLF4J API Module 2.0.17. + + The MIT License + =============== + Copyright (c) 2004-2025 QOS.ch + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +NOTICE FROM PROGRESS SOFTWARE CORPORATION: Additional notices may be included +in the release notes or other documentation that accompanies updates received +in connection with support of the Product. + +Updated March 12, 2026 diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index 3b888233e..60ff0ae94 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -58,7 +58,6 @@ dependencies { testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" testImplementation "org.mockito:mockito-core:5.22.0" -// testImplementation "org.mockito:mockito-inline:5.2.0" testImplementation "com.squareup.okhttp3:mockwebserver3:${okhttpVersion}" diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java index fe39ec616..f136884de 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanColumnBuilder.java @@ -8,6 +8,8 @@ /** * An instance of a column builder returned by the columnBuilder() method * in a row pipeline. Used to create column definitions for op:from-docs. + * + * @since 8.1.0; requires MarkLogic 12.1 or higher */ public interface PlanColumnBuilder extends ServerExpression { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java index 128e30f38..80e42f42c 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/type/PlanContextExprCall.java @@ -8,6 +8,8 @@ /** * An instance of a context expression call returned by the context() method * in a row pipeline. + * + * @since 8.1.0; requires MarkLogic 12.1 or higher */ public interface PlanContextExprCall extends ServerExpression { } From b0eca281b10fd34f05b3803e6cfbcc9b002257e2 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 12 Mar 2026 15:26:34 -0400 Subject: [PATCH 69/70] MLE-27481 Added SPDX file --- sbom.spdx.json | 491 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 491 insertions(+) create mode 100644 sbom.spdx.json diff --git a/sbom.spdx.json b/sbom.spdx.json new file mode 100644 index 000000000..1a1e57d6a --- /dev/null +++ b/sbom.spdx.json @@ -0,0 +1,491 @@ +{ + "SPDXID" : "SPDXRef-DOCUMENT", + "spdxVersion" : "2.3", + "creationInfo" : { + "comment" : "Progress MarkLogic Java Client 8.1.0\nCopyright (c) 2026 Progress Software Corporation and/or its subsidiaries or affiliates. All rights reserved.", + "created" : "2026-03-12T18:22:03+0000", + "creators" : [ "Tool: SBOMinator-1.1.0.617", "Organization: Progress Software Corporation" ] + }, + "name" : "MarkLogic-DevExp-javaapi.java client-8.1.0.0", + "documentName" : "java client-8.1.0.spdx", + "documentNamespace" : "https://www.progress.com/spdx/MarkLogic/DevExp/javaapi/java client-8.1.0.0-69f00983-1344-342a-b62a-db32c5399683", + "hasExtractedLicensingInfos" : [ { + "licenseId" : "LicenseRef-Apache-2.0-fab41b7c", + "extractedText" : "Apache License\nVersion 2.0, January 2004\n=========================\n\n\nhttp://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n\"License\" shall mean the terms and conditions for use, reproduction, and\ndistribution as defined by Sections 1 through 9 of this document.\n\n\"Licensor\" shall mean the copyright owner or entity authorized by the copyright\nowner that is granting the License.\n\n\"Legal Entity\" shall mean the union of the acting entity and all other entities\nthat control, are controlled by, or are under common control with that entity.\nFor the purposes of this definition, \"control\" means (i) the power, direct or\nindirect, to cause the direction or management of such entity, whether by\ncontract or otherwise, or (ii) ownership of fifty percent (50%) or more of the\noutstanding shares, or (iii) beneficial ownership of such entity.\n\n\"You\" (or \"Your\") shall mean an individual or Legal Entity exercising permissions\ngranted by this License.\n\n\"Source\" form shall mean the preferred form for making modifications, including\nbut not limited to software source code, documentation source, and configuration\nfiles.\n\n\"Object\" form shall mean any form resulting from mechanical transformation or\ntranslation of a Source form, including but not limited to compiled object code,\ngenerated documentation, and conversions to other media types.\n\n\"Work\" shall mean the work of authorship, whether in Source or Object form, made\navailable under the License, as indicated by a copyright notice that is included\nin or attached to the work (an example is provided in the Appendix below).\n\n\"Derivative Works\" shall mean any work, whether in Source or Object form, that is\nbased on (or derived from) the Work and for which the editorial revisions,\nannotations, elaborations, or other modifications represent, as a whole, an\noriginal work of authorship. For the purposes of this License, Derivative Works\nshall not include works that remain separable from, or merely link (or bind by\nname) to the interfaces of, the Work and Derivative Works thereof.\n\n\"Contribution\" shall mean any work of authorship, including the original version\nof the Work and any modifications or additions to that Work or Derivative Works\nthereof, that is intentionally submitted to Licensor for inclusion in the Work by\nthe copyright owner or by an individual or Legal Entity authorized to submit on\nbehalf of the copyright owner. For the purposes of this definition, \"submitted\"\nmeans any form of electronic, verbal, or written communication sent to the\nLicensor or its representatives, including but not limited to communication on\nelectronic mailing lists, source code control systems, and issue tracking systems\nthat are managed by, or on behalf of, the Licensor for the purpose of discussing\nand improving the Work, but excluding communication that is conspicuously marked\nor otherwise designated in writing by the copyright owner as \"Not a\nContribution.\"\n\n\"Contributor\" shall mean Licensor and any individual or Legal Entity on behalf of\nwhom a Contribution has been received by Licensor and subsequently incorporated\nwithin the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of this\nLicense, each Contributor hereby grants to You a perpetual, worldwide,\nnon-exclusive, no-charge, royalty-free, irrevocable copyright license to\nreproduce, prepare Derivative Works of, publicly display, publicly perform,\nsublicense, and distribute the Work and such Derivative Works in Source or Object\nform.\n\n3. Grant of Patent License. Subject to the terms and conditions of this License,\neach Contributor hereby grants to You a perpetual, worldwide, non-exclusive,\nno-charge, royalty-free, irrevocable (except as stated in this section) patent\nlicense to make, have made, use, offer to sell, sell, import, and otherwise\ntransfer the Work, where such license applies only to those patent claims\nlicensable by such Contributor that are necessarily infringed by their\nContribution(s) alone or by combination of their Contribution(s) with the Work to\nwhich such Contribution(s) was submitted. If You institute patent litigation\nagainst any entity (including a cross-claim or counterclaim in a lawsuit)\nalleging that the Work or a Contribution incorporated within the Work constitutes\ndirect or contributory patent infringement, then any patent licenses granted to\nYou under this License for that Work shall terminate as of the date such\nlitigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the Work or\nDerivative Works thereof in any medium, with or without modifications, and in\nSource or Object form, provided that You meet the following conditions:\n\n a. You must give any other recipients of the Work or Derivative Works a copy of\n this License; and\n\n b. You must cause any modified files to carry prominent notices stating that\n You changed the files; and\n\n c. You must retain, in the Source form of any Derivative Works that You\n distribute, all copyright, patent, trademark, and attribution notices from\n the Source form of the Work, excluding those notices that do not pertain to\n any part of the Derivative Works; and\n\n d. If the Work includes a \"NOTICE\" text file as part of its distribution, then\n any Derivative Works that You distribute must include a readable copy of the\n attribution notices contained within such NOTICE file, excluding those\n notices that do not pertain to any part of the Derivative Works, in at least\n one of the following places: within a NOTICE text file distributed as part of\n the Derivative Works; within the Source form or documentation, if provided\n along with the Derivative Works; or, within a display generated by the\n Derivative Works, if and wherever such third-party notices normally appear.\n The contents of the NOTICE file are for informational purposes only and do\n not modify the License. You may add Your own attribution notices within\n Derivative Works that You distribute, alongside or as an addendum to the\n NOTICE text from the Work, provided that such additional attribution notices\n cannot be construed as modifying the License.\n\nYou may add Your own copyright statement to Your modifications and may provide\nadditional or different license terms and conditions for use, reproduction, or\ndistribution of Your modifications, or for any such Derivative Works as a whole,\nprovided Your use, reproduction, and distribution of the Work otherwise complies\nwith the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise, any\nContribution intentionally submitted for inclusion in the Work by You to the\nLicensor shall be under the terms and conditions of this License, without any\nadditional terms or conditions. Notwithstanding the above, nothing herein shall\nsupersede or modify the terms of any separate license agreement you may have\nexecuted with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade names,\ntrademarks, service marks, or product names of the Licensor, except as required\nfor reasonable and customary use in describing the origin of the Work and\nreproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or agreed to in\nwriting, Licensor provides the Work (and each Contributor provides its\nContributions) on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,\neither express or implied, including, without limitation, any warranties or\nconditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\nPARTICULAR PURPOSE. You are solely responsible for determining the\nappropriateness of using or redistributing the Work and assume any risks\nassociated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory, whether in\ntort (including negligence), contract, or otherwise, unless required by\napplicable law (such as deliberate and grossly negligent acts) or agreed to in\nwriting, shall any Contributor be liable to You for damages, including any\ndirect, indirect, special, incidental, or consequential damages of any character\narising as a result of this License or out of the use or inability to use the\nWork (including but not limited to damages for loss of goodwill, work stoppage,\ncomputer failure or malfunction, or any and all other commercial damages or\nlosses), even if such Contributor has been advised of the possibility of such\ndamages.\n\n9. Accepting Warranty or Additional Liability. While redistributing the Work or\nDerivative Works thereof, You may choose to offer, and charge a fee for,\nacceptance of support, warranty, indemnity, or other liability obligations and/or\nrights consistent with this License. However, in accepting such obligations, You\nmay act only on Your own behalf and on Your sole responsibility, not on behalf of\nany other Contributor, and only if You agree to indemnify, defend, and hold each\nContributor harmless for any liability incurred by, or claims asserted against,\nsuch Contributor by reason of your accepting any such warranty or additional\nliability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work\n\nTo apply the Apache License to your work, attach the following boilerplate\nnotice, with the fields enclosed by brackets \"[]\" replaced with your own\nidentifying information. (Don't include the brackets!) The text should be\nenclosed in the appropriate comment syntax for the file format. We also recommend\nthat a file or class name and description of purpose be included on the same\n\"printed page\" as the copyright notice for easier identification within\nthird-party archives.\n\n Copyright [yyyy] [name of copyright owner] Licensed under the Apache License,\n Version 2.0 (the \"License\"); you may not use this file except in compliance\n with the License. You may obtain a copy of the License at\n http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law\n or agreed to in writing, software distributed under the License is\n distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n KIND, either express or implied. See the License for the specific language\n governing permissions and limitations under the License." + }, { + "licenseId" : "LicenseRef-Eclipse-Distribution-License-v-1.0-ca2a15f3", + "extractedText" : "Eclipse Distribution License - v 1.0\n====================================\n\nCopyright (c) 2007, Eclipse Foundation, Inc. and its licensors.\n\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n * Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n * Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n * Neither the name of the Eclipse Foundation, Inc. nor the names of its\n contributors may be used to endorse or promote products derived from this\n software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\nANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS\nOF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING\nNEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN\nIF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + }, { + "licenseId" : "LicenseRef-EPL-2.0-e7a7fa30", + "extractedText" : "Eclipse Public License - v 2.0\n==============================\n\nTHE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC\nLICENSE (\u201CAGREEMENT\u201D). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM\nCONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.\n\n\n1. DEFINITIONS\n--------------\n\n\u201CContribution\u201D means:\n\n a) in the case of the initial Contributor, the initial content Distributed\n under this Agreement, and\n b) in the case of each subsequent Contributor:\n\n i) changes to the Program, and\n ii) additions to the Program;\n\n where such changes and/or additions to the Program originate from and are\n Distributed by that particular Contributor. A Contribution \u201Coriginates\u201D from a\n Contributor if it was added to the Program by such Contributor itself or anyone\n acting on such Contributor's behalf. Contributions do not include changes or\n additions to the Program that are not Modified Works.\n\n\u201CContributor\u201D means any person or entity that Distributes the Program.\n\n\u201CLicensed Patents\u201D mean patent claims licensable by a Contributor which are\nnecessarily infringed by the use or sale of its Contribution alone or when\ncombined with the Program.\n\n\u201CProgram\u201D means the Contributions Distributed in accordance with this Agreement.\n\n\u201CRecipient\u201D means anyone who receives the Program under this Agreement or any\nSecondary License (as applicable), including Contributors.\n\n\u201CDerivative Works\u201D shall mean any work, whether in Source Code or other form,\nthat is based on (or derived from) the Program and for which the editorial\nrevisions, annotations, elaborations, or other modifications represent, as a\nwhole, an original work of authorship.\n\n\u201CModified Works\u201D shall mean any work in Source Code or other form that results\nfrom an addition to, deletion from, or modification of the contents of the\nProgram, including, for purposes of clarity any new file in Source Code form that\ncontains any contents of the Program. Modified Works shall not include works that\ncontain only declarations, interfaces, types, classes, structures, or files of\nthe Program solely in each case in order to link to, bind by name, or subclass\nthe Program or Modified Works thereof.\n\n\u201CDistribute\u201D means the acts of a) distributing or b) making available in any\nmanner that enables the transfer of a copy.\n\n\u201CSource Code\u201D means the form of a Program preferred for making modifications,\nincluding but not limited to software source code, documentation source, and\nconfiguration files.\n\n\u201CSecondary License\u201D means either the GNU General Public License, Version 2.0, or\nany later versions of that license, including any exceptions or additional\npermissions as identified by the initial Contributor.\n\n\n2. GRANT OF RIGHTS\n------------------\n\n a) Subject to the terms of this Agreement, each Contributor hereby grants\n Recipient a non-exclusive, worldwide, royalty-free copyright license to\n reproduce, prepare Derivative Works of, publicly display, publicly perform,\n Distribute and sublicense the Contribution of such Contributor, if any, and\n such Derivative Works.\n b) Subject to the terms of this Agreement, each Contributor hereby grants\n Recipient a non-exclusive, worldwide, royalty-free patent license under\n Licensed Patents to make, use, sell, offer to sell, import and otherwise\n transfer the Contribution of such Contributor, if any, in Source Code or other\n form. This patent license shall apply to the combination of the Contribution\n and the Program if, at the time the Contribution is added by the Contributor,\n such addition of the Contribution causes such combination to be covered by the\n Licensed Patents. The patent license shall not apply to any other combinations\n which include the Contribution. No hardware per se is licensed hereunder.\n c) Recipient understands that although each Contributor grants the licenses to\n its Contributions set forth herein, no assurances are provided by any\n Contributor that the Program does not infringe the patent or other intellectual\n property rights of any other entity. Each Contributor disclaims any liability\n to Recipient for claims brought by any other entity based on infringement of\n intellectual property rights or otherwise. As a condition to exercising the\n rights and licenses granted hereunder, each Recipient hereby assumes sole\n responsibility to secure any other intellectual property rights needed, if any.\n For example, if a third party patent license is required to allow Recipient to\n Distribute the Program, it is Recipient's responsibility to acquire that\n license before distributing the Program.\n d) Each Contributor represents that to its knowledge it has sufficient\n copyright rights in its Contribution, if any, to grant the copyright license\n set forth in this Agreement.\n e) Notwithstanding the terms of any Secondary License, no Contributor makes\n additional grants to any Recipient (other than those set forth in this\n Agreement) as a result of such Recipient's receipt of the Program under the\n terms of a Secondary License (if permitted under the terms of Section 3).\n\n\n3. REQUIREMENTS\n---------------\n\n3.1 If a Contributor Distributes the Program in any form, then:\n\n a) the Program must also be made available as Source Code, in accordance with\n section 3.2, and the Contributor must accompany the Program with a statement\n that the Source Code for the Program is available under this Agreement, and\n informs Recipients how to obtain it in a reasonable manner on or through a\n medium customarily used for software exchange; and\n b) the Contributor may Distribute the Program under a license different than\n this Agreement, provided that such license:\n\n i) effectively disclaims on behalf of all other Contributors all warranties\n and conditions, express and implied, including warranties or conditions of\n title and non-infringement, and implied warranties or conditions of\n merchantability and fitness for a particular purpose;\n ii) effectively excludes on behalf of all other Contributors all liability\n for damages, including direct, indirect, special, incidental and\n consequential damages, such as lost profits;\n iii) does not attempt to limit or alter the recipients' rights in the Source\n Code under section 3.2; and\n iv) requires any subsequent distribution of the Program by any party to be\n under a license that satisfies the requirements of this section 3.\n\n3.2 When the Program is Distributed as Source Code:\n\n a) it must be made available under this Agreement, or if the Program\n\n (i) is combined with other material in a separate file or files made\n available under a Secondary License, and\n (ii) the initial Contributor attached to the Source Code the notice described\n in Exhibit A of this Agreement, then the Program may be made available under\n the terms of such Secondary Licenses, and\n\n b) a copy of this Agreement must be included with each copy of the Program.\n\n3.3 Contributors may not remove or alter any copyright, patent, trademark,\nattribution notices, disclaimers of warranty, or limitations of liability\n(\u2018notices\u2019) contained within the Program from any copy of the Program which they\nDistribute, provided that Contributors may add their own appropriate notices.\n\n\n4. COMMERCIAL DISTRIBUTION\n--------------------------\n\nCommercial distributors of software may accept certain responsibilities with\nrespect to end users, business partners and the like. While this license is\nintended to facilitate the commercial use of the Program, the Contributor who\nincludes the Program in a commercial product offering should do so in a manner\nwhich does not create potential liability for other Contributors. Therefore, if a\nContributor includes the Program in a commercial product offering, such\nContributor (\u201CCommercial Contributor\u201D) hereby agrees to defend and indemnify\nevery other Contributor (\u201CIndemnified Contributor\u201D) against any losses, damages\nand costs (collectively \u201CLosses\u201D) arising from claims, lawsuits and other legal\nactions brought by a third party against the Indemnified Contributor to the\nextent caused by the acts or omissions of such Commercial Contributor in\nconnection with its distribution of the Program in a commercial product offering.\nThe obligations in this section do not apply to any claims or Losses relating to\nany actual or alleged intellectual property infringement. In order to qualify, an\nIndemnified Contributor must: a) promptly notify the Commercial Contributor in\nwriting of such claim, and b) allow the Commercial Contributor to control, and\ncooperate with the Commercial Contributor in, the defense and any related\nsettlement negotiations. The Indemnified Contributor may participate in any such\nclaim at its own expense.\n\nFor example, a Contributor might include the Program in a commercial product\noffering, Product X. That Contributor is then a Commercial Contributor. If that\nCommercial Contributor then makes performance claims, or offers warranties\nrelated to Product X, those performance claims and warranties are such Commercial\nContributor's responsibility alone. Under this section, the Commercial\nContributor would have to defend claims against the other Contributors related to\nthose performance claims and warranties, and if a court requires any other\nContributor to pay any damages as a result, the Commercial Contributor must pay\nthose damages.\n\n\n5. NO WARRANTY\n--------------\n\nEXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN \u201CAS IS\u201D BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT\nLIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,\nMERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely\nresponsible for determining the appropriateness of using and distributing the\nProgram and assumes all risks associated with its exercise of rights under this\nAgreement, including but not limited to the risks and costs of program errors,\ncompliance with applicable laws, damage to or loss of data, programs or\nequipment, and unavailability or interruption of operations.\n\n\n6. DISCLAIMER OF LIABILITY\n--------------------------\n\nEXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING\nNEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE\nPROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE\nPOSSIBILITY OF SUCH DAMAGES.\n\n\n7. GENERAL\n----------\n\nIf any provision of this Agreement is invalid or unenforceable under applicable\nlaw, it shall not affect the validity or enforceability of the remainder of the\nterms of this Agreement, and without further action by the parties hereto, such\nprovision shall be reformed to the minimum extent necessary to make such\nprovision valid and enforceable.\n\nIf Recipient institutes patent litigation against any entity (including a\ncross-claim or counterclaim in a lawsuit) alleging that the Program itself\n(excluding combinations of the Program with other software or hardware) infringes\nsuch Recipient's patent(s), then such Recipient's rights granted under Section\n2(b) shall terminate as of the date such litigation is filed.\n\nAll Recipient's rights under this Agreement shall terminate if it fails to comply\nwith any of the material terms or conditions of this Agreement and does not cure\nsuch failure in a reasonable period of time after becoming aware of such\nnoncompliance. If all Recipient's rights under this Agreement terminate,\nRecipient agrees to cease use and distribution of the Program as soon as\nreasonably practicable. However, Recipient's obligations under this Agreement and\nany licenses granted by Recipient relating to the Program shall continue and\nsurvive.\n\nEveryone is permitted to copy and distribute copies of this Agreement, but in\norder to avoid inconsistency the Agreement is copyrighted and may only be\nmodified in the following manner. The Agreement Steward reserves the right to\npublish new versions (including revisions) of this Agreement from time to time.\nNo one other than the Agreement Steward has the right to modify this Agreement.\nThe Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation\nmay assign the responsibility to serve as the Agreement Steward to a suitable\nseparate entity. Each new version of the Agreement will be given a distinguishing\nversion number. The Program (including Contributions) may always be Distributed\nsubject to the version of the Agreement under which it was received. In addition,\nafter a new version of the Agreement is published, Contributor may elect to\nDistribute the Program (including its Contributions) under the new version.\n\nExcept as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no\nrights or licenses to the intellectual property of any Contributor under this\nAgreement, whether expressly, by implication, estoppel or otherwise. All rights\nin the Program not expressly granted under this Agreement are reserved. Nothing\nin this Agreement is intended to be enforceable by any entity that is not a\nContributor or Recipient. No third-party beneficiary rights are created under\nthis Agreement.\n\n\nExhibit A \u2013 Form of Secondary Licenses Notice\n---------------------------------------------\n\n\u201CThis Source Code may also be made available under the following Secondary\nLicenses when the conditions for such availability set forth in the Eclipse\nPublic License, v. 2.0 are satisfied: {name license(s), version(s), and\nexceptions or additional permissions here}.\u201D\n\n Simply including a copy of this Agreement, including this Exhibit A is not\n sufficient to license the Source Code under Secondary Licenses.\n\n If it is not possible or desirable to put the notice in a particular file,\n then You may include the notice in a location (such as a LICENSE file in a\n relevant directory) where a recipient would be likely to look for such a\n notice.\n\n You may add additional accurate notices of copyright ownership." + }, { + "licenseId" : "LicenseRef-MIT-244c96f8", + "extractedText" : "The MIT License\n===============\nCopyright (c) 2004-2025 QOS.ch\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in the\nSoftware without restriction, including without limitation the rights to use,\ncopy, modify, merge, publish, distribute, sublicense, and/or sell copies of the\nSoftware, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN\nAN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." + }, { + "licenseId" : "LicenseRef-BSD-3-Clause-52a414cd", + "extractedText" : "Copyright (c) , Oracle and/or its affiliates\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without modification,\nare permitted provided that the following conditions are met:\n\n * Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n * Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n * Neither the name of the com.sun.activation nor the names of its contributors may\n be used to endorse or promote products derived from this software without\n specific prior written permission.\n\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\nANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS\nOF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING\nNEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN\nIF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + } ], + "packages" : [ { + "SPDXID" : "SPDXRef-C1:com.fasterxml.jackson.core:jackson-annotations:2.21", + "name" : "jackson-annotations", + "versionInfo" : "2.21", + "copyrightText" : "2007-, FasterXML", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "e0d0c3e7300954f73e43c67d933aaea4" + }, { + "algorithm" : "SHA1", + "checksumValue" : "b1bc1868bf02dc0bd6c7836257a036a331005309" + }, { + "algorithm" : "SHA256", + "checksumValue" : "53ca085f4a150f703f49e1aabd935bd03b43e1ea3d55d135438292af22cef56b" + }, { + "algorithm" : "SHA512", + "checksumValue" : "d832a99867acc2d5afb2596da760c50e6a6b54c1bbc8e6c4186f267a16e1d55c7a916ef8b034ad6b277c41e3342b91dcdc2e3dcea8bdccfe89e7c147894591be" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.21/jackson-annotations-2.21.jar" + }, { + "SPDXID" : "SPDXRef-C9:com.squareup.okhttp3:okhttp:5.3.2", + "name" : "okhttp", + "versionInfo" : "5.3.2", + "copyrightText" : "Square", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "7cefaebda364337f3770168187124f31" + }, { + "algorithm" : "SHA1", + "checksumValue" : "6891a4fd19bcab6f69fe091c2bc6077edec5258e" + }, { + "algorithm" : "SHA256", + "checksumValue" : "e720a383fdc3eb1df4aac77a085e61b7730837364151de867093fdbcafcb44aa" + }, { + "algorithm" : "SHA512", + "checksumValue" : "165c65503bded11490b09c46935864aa71ee8b9fc4ea1c01e1cb91b209a64af97e92d5d8fd9b4bc78209a8949a79f37ff49edd992171d25ba6f7e0d968cee87c" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/5.3.2/okhttp-5.3.2.jar" + }, { + "SPDXID" : "SPDXRef-C24:org.jetbrains:annotations:13.0", + "name" : "annotations", + "versionInfo" : "13.0", + "copyrightText" : "Jetbrains", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "f4fb462172517b46b6cd90003508515a" + }, { + "algorithm" : "SHA1", + "checksumValue" : "919f0dfe192fb4e063e7dacadee7f8bb9a2672a9" + }, { + "algorithm" : "SHA256", + "checksumValue" : "ace2a10dc8e2d5fd34925ecac03e4988b2c0f851650c94b8cef49ba1bd111478" + }, { + "algorithm" : "SHA512", + "checksumValue" : "5622d0ffe410e7272e2bb9fae1006caedeb86d0c62d2d9f3929a3b3cdcdef1963218fcf0cede82e95ef9f4da3ed4a173fa055ee6e4038886376181e0423e02ff" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/jetbrains/annotations/13.0/annotations-13.0.jar" + }, { + "SPDXID" : "SPDXRef-C3:com.fasterxml.jackson.core:jackson-databind:2.21.1", + "name" : "jackson-databind", + "versionInfo" : "2.21.1", + "copyrightText" : "2007-, FasterXML", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "35fe6cc9ff7990e67f86a92092aa0569" + }, { + "algorithm" : "SHA1", + "checksumValue" : "5615fb77652bfd386d87b95a1d663e1c0e38b372" + }, { + "algorithm" : "SHA256", + "checksumValue" : "b011eb5202d9ec889e27f1dcbdf6c63f06a76e7a16c0a1b30c6048d556c9a28e" + }, { + "algorithm" : "SHA512", + "checksumValue" : "837a94e20474d0ad3630663c0112e982914e5cfa3bbf397b8c31d00b68d7fc8e4514458b75dd39cf74edda7e33339504fa845864476786e94812bd36a5a428ee" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.21.1/jackson-databind-2.21.1.jar" + }, { + "SPDXID" : "SPDXRef-C17:jakarta.xml.bind:jakarta.xml.bind-api:4.0.5", + "name" : "jakarta.xml.bind-api", + "versionInfo" : "4.0.5", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "935053b2c792f34c7ea3a238ea96f3ac" + }, { + "algorithm" : "SHA1", + "checksumValue" : "161811f36cad3c65991502e80317f2f6703361df" + }, { + "algorithm" : "SHA256", + "checksumValue" : "5e489b6c874c4119e003ff1403db523ee3a8959ec499f3de29e77245efccf216" + }, { + "algorithm" : "SHA512", + "checksumValue" : "3f3e522bd716ab6842ba2eed8bd44fc7d859ca5a3798e5adf2447eabde1c5d3b74b6f58d217d6cd772cbb034e4f003d9f9e10fafa15f4e108ea802548d1e5044" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/jakarta/xml/bind/jakarta.xml.bind-api/4.0.5/jakarta.xml.bind-api-4.0.5.jar" + }, { + "SPDXID" : "SPDXRef-C22:org.glassfish.jaxb:txw2:4.0.6", + "name" : "txw2", + "versionInfo" : "4.0.6", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "0bf7070aee3bb53640d2ea6441e059fb" + }, { + "algorithm" : "SHA1", + "checksumValue" : "4f4cd53b5ff9a2c5aa1211f15ed2569c57dfb044" + }, { + "algorithm" : "SHA256", + "checksumValue" : "fcc749785412ef3806fde1ce70f93ef5a0065dcc47fe449bc871db0795cb11af" + }, { + "algorithm" : "SHA512", + "checksumValue" : "47eb0e4b199bb12804da94cf8a81a1e652e8ca31af68b65d52f1a0cda6e1c9b41276e4bc1e4ea510f83ecccff9978ff8bd05a56c5b16a4181d8e0673d608b2c4" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/glassfish/jaxb/txw2/4.0.6/txw2-4.0.6.jar" + }, { + "SPDXID" : "SPDXRef-C4:com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.21.1", + "name" : "jackson-dataformat-csv", + "versionInfo" : "2.21.1", + "copyrightText" : "2007-, FasterXML", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "334e2d7f64aa1553feda4e2f975244b2" + }, { + "algorithm" : "SHA1", + "checksumValue" : "b62877fbda7385a2ec4059f5cb2d54f8d323dad0" + }, { + "algorithm" : "SHA256", + "checksumValue" : "33e58728b1df1c8d59638a9c3c8750b9543655b8fb837712c22ce8e6395f4978" + }, { + "algorithm" : "SHA512", + "checksumValue" : "ff4e3878ce96a1e354da1138b1d78e10215db3f954fcc830473afd469c737d0caf6246c5bb1f12a98fe9d71dbb3c5dac5812e6fcd2791f46704925b4b931127a" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-csv/2.21.1/jackson-dataformat-csv-2.21.1.jar" + }, { + "SPDXID" : "SPDXRef-C6:com.fasterxml.jackson.module:jackson-module-kotlin:2.21.1", + "name" : "jackson-module-kotlin", + "versionInfo" : "2.21.1", + "copyrightText" : "FasterXML", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "a6e4a127bdd574a1e73823e341a6bee2" + }, { + "algorithm" : "SHA1", + "checksumValue" : "b9020c210dccee9e1d3d7b26d3b6b27f3e418e47" + }, { + "algorithm" : "SHA256", + "checksumValue" : "b141bfd34e457f95f4029744c7cf254f421bb2bd9687761d184a9c5fd5700f0e" + }, { + "algorithm" : "SHA512", + "checksumValue" : "916111563d4e54089015595e492f7801c89c8675273301e06c89638900ee203bfb533022e2f19fd7cf5c19bb97e3e93cd9c3d0d496c700c69a790481ecc0c5b2" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/module/jackson-module-kotlin/2.21.1/jackson-module-kotlin-2.21.1.jar" + }, { + "SPDXID" : "SPDXRef-C23:org.jetbrains.kotlin:kotlin-reflect:2.1.21", + "name" : "kotlin-reflect", + "versionInfo" : "2.1.21", + "copyrightText" : "JetBrains", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "0ed84417501a7c7f015dcc9be623bb22" + }, { + "algorithm" : "SHA1", + "checksumValue" : "4929da533b3f04a101e4bb9ae0ee51a1d8107614" + }, { + "algorithm" : "SHA256", + "checksumValue" : "bcd75a36ca4ad8e06117214ed807f8dea2fe61a71e07f91ca14f4335024b8463" + }, { + "algorithm" : "SHA512", + "checksumValue" : "b3f93ed791b0d47bab3e50ca789c3d1d90010fa37cc9c4a05f53537fdb9c7a998a8ed53be425bd55f43019374abd1a53c51d0f2ad9d717597f91d09170e4f5f9" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-reflect/2.1.21/kotlin-reflect-2.1.21.jar" + }, { + "SPDXID" : "SPDXRef-C13:com.sun.mail:jakarta.mail:2.0.2", + "name" : "jakarta.mail", + "versionInfo" : "2.0.2", + "copyrightText" : "Oracle", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "69753278ed0fa13af0a627de3df5d8b0" + }, { + "algorithm" : "SHA1", + "checksumValue" : "6dfe5d279fb579a41baa84a7728e31a40a50d90e" + }, { + "algorithm" : "SHA256", + "checksumValue" : "536f0ae5dd774e257f3064f19a2b982aa56072c07c675c70f98cdc262047b4b5" + }, { + "algorithm" : "SHA512", + "checksumValue" : "2f11a49f1afa5d86a31647e2a00632fadd98eb755e08c7cbc3eb1b0de584499dd8c0923b68d6c4cb3a8abeb383d35d6b02a43fd06da1fe0223c1589ec934e87d" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/sun/mail/jakarta.mail/2.0.2/jakarta.mail-2.0.2.jar" + }, { + "SPDXID" : "SPDXRef-C21:org.glassfish.jaxb:jaxb-runtime:4.0.6", + "name" : "jaxb-runtime", + "versionInfo" : "4.0.6", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "0e600d639f3a09ddd6fa91623a12b634" + }, { + "algorithm" : "SHA1", + "checksumValue" : "fb95ebb62564657b2fedfe165b859789ef3a8711" + }, { + "algorithm" : "SHA256", + "checksumValue" : "1c0d57f8c25f9605d5a2f7ad0a87581893776ac85b00b101b2651258edaa9118" + }, { + "algorithm" : "SHA512", + "checksumValue" : "e19db2669e916992ea6acc75174070a4b5f04bb7788df03b1ede3340f5d63cf1c5055568ba1d7738e9e018716d10c3f6b26d1951faf8bba75f6ebb6ac6b16314" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-runtime/4.0.6/jaxb-runtime-4.0.6.jar" + }, { + "SPDXID" : "SPDXRef-C10:com.squareup.okio:okio-jvm:3.16.4", + "name" : "okio-jvm", + "versionInfo" : "3.16.4", + "copyrightText" : "Square", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "34d866f45bb2a65bf601cde52b72d096" + }, { + "algorithm" : "SHA1", + "checksumValue" : "ceb794cf0bbf8d0d20f49aa91ce20db7fd77675d" + }, { + "algorithm" : "SHA256", + "checksumValue" : "2196b993cd34dbbd919e7e01f57a4781b58bee80f86106163e287c20343a96a7" + }, { + "algorithm" : "SHA512", + "checksumValue" : "1a7158538ae341b8a61fe7292506e12ca6063c5a51a1c122d0e934164395c7da585dc31b26ed88a993c47aeca89f6a854042c67a4bc7d771d35a0ece6c4dbde0" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/squareup/okio/okio-jvm/3.16.4/okio-jvm-3.16.4.jar" + }, { + "SPDXID" : "SPDXRef-C20:org.glassfish.jaxb:jaxb-core:4.0.6", + "name" : "jaxb-core", + "versionInfo" : "4.0.6", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "e36c915cf47342b4fe31ffba3407b928" + }, { + "algorithm" : "SHA1", + "checksumValue" : "8e61282303777fc98a00cc3affd0560d68748a75" + }, { + "algorithm" : "SHA256", + "checksumValue" : "ebbd274207b4860d0dc6e2d44d6dbdb5945cede01222d2e50661d45f5d46c0f7" + }, { + "algorithm" : "SHA512", + "checksumValue" : "f6ed3cc73361794a8e35fc04f09212aa21af9575f36f6440bf33a6d43708ded6142a9be51d9d08d6e5debc262d87c158f0e199ed041a78fb1e0086fd02868a28" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/glassfish/jaxb/jaxb-core/4.0.6/jaxb-core-4.0.6.jar" + }, { + "SPDXID" : "SPDXRef-C19:org.eclipse.angus:angus-activation:2.0.3", + "name" : "angus-activation", + "versionInfo" : "2.0.3", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "ad20392145690b36b4f950fe31a31a2a" + }, { + "algorithm" : "SHA1", + "checksumValue" : "7f80607ea5014fef0b1779e6c33d63a88a45a563" + }, { + "algorithm" : "SHA256", + "checksumValue" : "a6bd35c538cf90fff941ad6258c40c08fca0b5c9c3f536c657114f27ce0527a7" + }, { + "algorithm" : "SHA512", + "checksumValue" : "efb987b781f665589b2a524d86826ffcf37eaf105b2f823be124fded85bd118098c0749b5d268373d00b1d3b8bf0f89f86670444f9990b91948704a7052376e1" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/eclipse/angus/angus-activation/2.0.3/angus-activation-2.0.3.jar" + }, { + "SPDXID" : "SPDXRef-C14:io.github.erdtman:java-json-canonicalization:1.1", + "name" : "java-json-canonicalization", + "versionInfo" : "1.1", + "copyrightText" : "Samuel Erdtman", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "eb0d5f9152537adc89496b3a09792df4" + }, { + "algorithm" : "SHA1", + "checksumValue" : "605be827a90e12a465bbc30fcc5c775a1eb38f8f" + }, { + "algorithm" : "SHA256", + "checksumValue" : "ed12a01f28d147898312963a1f704e90290b67a61f34fa3a761f41c134f4e691" + }, { + "algorithm" : "SHA512", + "checksumValue" : "8c30197c069e27d33704aa69029a122d3a528235ab8baf5c7af22ebb13ff29d8d1888542bc5e29f4d2df654bc65b00a6777abe6d00e14a4589c2c75935c5466c" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/io/github/erdtman/java-json-canonicalization/1.1/java-json-canonicalization-1.1.jar" + }, { + "SPDXID" : "SPDXRef-C25:org.slf4j:slf4j-api:2.0.17", + "name" : "slf4j-api", + "versionInfo" : "2.0.17", + "copyrightText" : "2004-2025 QOS.ch", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "b6480d114a23683498ac3f746f959d2f" + }, { + "algorithm" : "SHA1", + "checksumValue" : "d9e58ac9c7779ba3bf8142aff6c830617a7fe60f" + }, { + "algorithm" : "SHA256", + "checksumValue" : "7b751d952061954d5abfed7181c1f645d336091b679891591d63329c622eb832" + }, { + "algorithm" : "SHA512", + "checksumValue" : "9a3e79db6666a6096a3021bb2e1d918f30f589d8de51d6b600f8ebd92515a510ae2d8f87919cc2dfa8365d64f10194cac8dfa0fb950160eef0e9da06f6caaeb9" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/org/slf4j/slf4j-api/2.0.17/slf4j-api-2.0.17.jar" + }, { + "SPDXID" : "SPDXRef-C16:jakarta.activation:jakarta.activation-api:2.1.4", + "name" : "jakarta.activation-api", + "versionInfo" : "2.1.4", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "bc1602eee7bc61a0b86f14bbbb0cc794" + }, { + "algorithm" : "SHA1", + "checksumValue" : "9e5c2a0d75dde71a0bedc4dbdbe47b78a5dc50f8" + }, { + "algorithm" : "SHA256", + "checksumValue" : "c9db52100ce6c8aac95cc39075f95720d2e561b11f8051b81c121ad4effd7004" + }, { + "algorithm" : "SHA512", + "checksumValue" : "cd078772acb5ebf1f90f7c737f372b7e86a5ce31b995ed759526a9eee73fd89f97e506f9a208ba3b73e757cb78830b829733411cf7989680aea2272abba7323b" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/jakarta/activation/jakarta.activation-api/2.1.4/jakarta.activation-api-2.1.4.jar" + }, { + "SPDXID" : "SPDXRef-C2:com.fasterxml.jackson.core:jackson-core:2.21.1", + "name" : "jackson-core", + "versionInfo" : "2.21.1", + "copyrightText" : "2007-, FasterXML", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "a526a530a07fd25140dee4695e686051" + }, { + "algorithm" : "SHA1", + "checksumValue" : "47b013fc85dbb819f3ba51e95a5560d0f1c4121c" + }, { + "algorithm" : "SHA256", + "checksumValue" : "1edd5f2e49dca5f8e4519957c24b7b3050bd1c7ee883920da33cff031ff1f7c0" + }, { + "algorithm" : "SHA512", + "checksumValue" : "f4b153a0a7c974620dec3fcfe4a4869c2632ab97f73c2662041af7d0eb0c42ad462bfee8cdcab5542842175b96197df1d910a8841067654f6e763b370c18ccfa" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.21.1/jackson-core-2.21.1.jar" + }, { + "SPDXID" : "SPDXRef-C18:net.openhft:zero-allocation-hashing:2026.0", + "name" : "zero-allocation-hashing", + "versionInfo" : "2026.0", + "copyrightText" : "Chronicle Software : Open Source", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "9f1ecd845fe93004aa6170d7dc90c47f" + }, { + "algorithm" : "SHA1", + "checksumValue" : "2447494f7842ca377b4ccdba2089a8da5c1d1b50" + }, { + "algorithm" : "SHA256", + "checksumValue" : "22b4328b4cd751790ba3222320c7aae3f7927e6cdd064cd8a60d2a179be4ada3" + }, { + "algorithm" : "SHA512", + "checksumValue" : "32006a9caddb63b72f7f49c3bb314511fd1c04b974bc5608ad4c3d46ad876039137b0e7705dbda7a2dd506a08d41987bed8fc5b3027abd4e34108b5553e80379" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/net/openhft/zero-allocation-hashing/2026.0/zero-allocation-hashing-2026.0.jar" + }, { + "SPDXID" : "SPDXRef-C15:io.github.rburgst:okhttp-digest:3.1.1", + "name" : "okhttp-digest", + "versionInfo" : "3.1.1", + "copyrightText" : "github.com/rburgst", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "273d1f50be9bc156c2110dd315b7370c" + }, { + "algorithm" : "SHA1", + "checksumValue" : "c076a26e02cb44b414459cdc1ae62b2b587f4bb0" + }, { + "algorithm" : "SHA256", + "checksumValue" : "86c3e09d956cd0b7b785472adfc650e7e87d381cc06ba6f77e48501896f1515a" + }, { + "algorithm" : "SHA512", + "checksumValue" : "e4fc133a8093e72a1d0d58c6525019c355a9f0a2dd4785e6a969502dbe07d538cf21930e2eb6cb7fb231caec9c9b66034661bc1ec11a7fe7c09726745a100654" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/io/github/rburgst/okhttp-digest/3.1.1/okhttp-digest-3.1.1.jar" + }, { + "SPDXID" : "SPDXRef-C7:com.fasterxml.jackson:jackson-bom:2.21.1", + "name" : "jackson-bom", + "versionInfo" : "2.21.1", + "copyrightText" : "FasterXML", + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/jackson-bom/2.21.1/jackson-bom-2.21.1.jar" + }, { + "SPDXID" : "SPDXRef-C5:com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.21.1", + "name" : "jackson-dataformat-yaml", + "versionInfo" : "2.21.1", + "copyrightText" : "2007-, FasterXML", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "f0a647f76bb70da49b12f9d862cc6d77" + }, { + "algorithm" : "SHA1", + "checksumValue" : "334d9c06a7b63ae67b72b5926dd10fb35cff0c0d" + }, { + "algorithm" : "SHA256", + "checksumValue" : "5c94fa55d4b93bd4ea9ac6f2cf4928ff50822d1c43e521e715d3abf23031a06d" + }, { + "algorithm" : "SHA512", + "checksumValue" : "a4af873838cab58bdef7d953c4f081fae1d6354b05f8b1937ab62b30f61be73e576c998b01e712c1fa42fd7c3de951e302aa807b2d7f5fe362c3e3603410e1d9" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.21.1/jackson-dataformat-yaml-2.21.1.jar" + }, { + "SPDXID" : "SPDXRef-C11:com.sun.activation:jakarta.activation:2.0.1", + "name" : "jakarta.activation", + "versionInfo" : "2.0.1", + "copyrightText" : "Oracle and/or its affiliates", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "39228ac67f033514a0ccb3360ac461f3" + }, { + "algorithm" : "SHA1", + "checksumValue" : "828b80e886a52bb09fe41ff410b10b342f533ce1" + }, { + "algorithm" : "SHA256", + "checksumValue" : "b9e24b7dd6e07495562ea96531be3130c96dba4d78e1dfd88adbbdebf4332871" + }, { + "algorithm" : "SHA512", + "checksumValue" : "d1c214d2d6ecc61bba396d913c66c977418e74dfe96e5558951ccf99a60cfca6adfea3fc64b6e2d15f698e30dac870b8f6e8d7a22e0dff975f8dd83864714fc5" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/sun/activation/jakarta.activation/2.0.1/jakarta.activation-2.0.1.jar" + }, { + "SPDXID" : "SPDXRef-C8:com.squareup.okhttp3:logging-interceptor:5.3.2", + "name" : "logging-interceptor", + "versionInfo" : "5.3.2", + "copyrightText" : "Square", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "afaee5dc0f5adb6013b7fb6a20ed7430" + }, { + "algorithm" : "SHA1", + "checksumValue" : "c3e383f1e11079b94509f025628959ebf7742d92" + }, { + "algorithm" : "SHA256", + "checksumValue" : "86574a5e35f483e6d6ba97beaf84420996dffca19f4d5cf3a41636365d403239" + }, { + "algorithm" : "SHA512", + "checksumValue" : "88a2411d54fda221560a9ce70362a572f434881cbe1e047ae3b268798fa83475884d1e8fdd7cda5ed04bbbdf621b7d538e3f9672499960180dfca30ac65827fc" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/squareup/okhttp3/logging-interceptor/5.3.2/logging-interceptor-5.3.2.jar" + }, { + "SPDXID" : "SPDXRef-C12:com.sun.istack:istack-commons-runtime:4.1.2", + "name" : "istack-commons-runtime", + "versionInfo" : "4.1.2", + "copyrightText" : "Eclipse Foundation", + "checksums" : [ { + "algorithm" : "MD5", + "checksumValue" : "535154ef647af2a52478c4debec93659" + }, { + "algorithm" : "SHA1", + "checksumValue" : "18ec117c85f3ba0ac65409136afa8e42bc74e739" + }, { + "algorithm" : "SHA256", + "checksumValue" : "7fd6792361f4dd00f8c56af4a20cecc0066deea4a8f3dec38348af23fc2296ee" + }, { + "algorithm" : "SHA512", + "checksumValue" : "c3b191409b9ace8cccca6be103b684a25f10675977d38f608036ffb687651a74fd4581a66e1c38e588e77165d32614e4b547bff412379f7a84b926ccb93515bb" + } ], + "downloadLocation" : "https://repo1.maven.org/maven2/com/sun/istack/istack-commons-runtime/4.1.2/istack-commons-runtime-4.1.2.jar" + } ] +} From 467e17a54eb302b823171674571cce6eb5dc8e54 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 12 Mar 2026 14:05:07 -0400 Subject: [PATCH 70/70] ML-27841 Bump to 8.1.0 --- NOTICE.txt | 20 ++++++++++---------- gradle.properties | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index a435604f3..be7922b95 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,4 +1,4 @@ -Progress MarkLogic Java Client +Progress MarkLogic Java Client 8.1.0 Portions of the Products include certain open source and commercial third-party components listed below ("Third-Party Components"). The authors of the @@ -71,9 +71,9 @@ conditions: (1) Apache License 2.0: -Progress MarkLogic Java Client incorporates Jetbrains annotations -13.0, OkHttp 5.3.2, OkHttp Logging Interceptor 5.3.2, Zero-allocation Hashing -2026.0, jackson-annotations 2.21, jackson-bom 2.21.1, jackson-core 2.21.1, +Progress MarkLogic Java Client 8.1.0 incorporates Jetbrains annotations 13.0, +OkHttp 5.3.2, OkHttp Logging Interceptor 5.3.2, Zero-allocation Hashing 2026.0, +jackson-annotations 2.21, jackson-bom 2.21.1, jackson-core 2.21.1, jackson-databind 2.21.1, jackson-dataformat-csv 2.21.1, jackson-dataformat-yaml 2.21.1, jackson-module-kotlin 2.21.1, java-json-canonicalization 1.1, kotlin-reflect 2.1.21, okhttp-digest 3.1.1, and squareokio 3.16.4. @@ -386,7 +386,7 @@ Contents of NOTICE text file for jackson-module-kotlin 2.21.1: (2) BSD 3-Clause "New" or "Revised" License: -Progress MarkLogic Java Client incorporates Jakarta Activation 2.0.1. +Progress MarkLogic Java Client 8.1.0 incorporates Jakarta Activation 2.0.1. Copyright (c) , Oracle and/or its affiliates All rights reserved. @@ -421,9 +421,9 @@ Progress MarkLogic Java Client incorporates Jakarta Activation 2.0.1. (3) Eclipse Distribution License - v 1.0: -Progress MarkLogic Java Client incorporates Angus Activation -Registries 2.0.3, JAXB CORE 4.0.6, JAXB Runtime 4.0.6, Jakarta Activation API -2.1.4, TXW2 Runtime 4.0.6, istack common utility code runtime 4.1.2, and +Progress MarkLogic Java Client 8.1.0 incorporates Angus Activation Registries +2.0.3, JAXB CORE 4.0.6, JAXB Runtime 4.0.6, Jakarta Activation API 2.1.4, TXW2 +Runtime 4.0.6, istack common utility code runtime 4.1.2, and jakarta.xml.bind:jakarta.xml.bind-api 4.0.5. Eclipse Distribution License - v 1.0 @@ -462,7 +462,7 @@ jakarta.xml.bind:jakarta.xml.bind-api 4.0.5. (4) Eclipse Public License 2.0: -Progress MarkLogic Java Client incorporates Jakarta Mail 2.0.2. +Progress MarkLogic Java Client 8.1.0 incorporates Jakarta Mail 2.0.2. Eclipse Public License - v 2.0 ============================== @@ -736,7 +736,7 @@ Progress MarkLogic Java Client incorporates Jakarta Mail 2.0.2. (5) MIT License: -Progress MarkLogic Java Client incorporates SLF4J API Module 2.0.17. +Progress MarkLogic Java Client 8.1.0 incorporates SLF4J API Module 2.0.17. The MIT License =============== diff --git a/gradle.properties b/gradle.properties index ccc8edbe0..a9b1b620f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=8.1-SNAPSHOT +version=8.1.0 publishUrl=file:../marklogic-java/releases okhttpVersion=5.3.2