diff --git a/.copyrightconfig b/.copyrightconfig index ba242e11f..c87b8a91b 100644 --- a/.copyrightconfig +++ b/.copyrightconfig @@ -11,4 +11,4 @@ startyear: 2010 # - Dotfiles already skipped automatically # Enable by removing the leading '# ' from the next line and editing values. # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js -filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml +filesexcluded: .github/*, README.md, Jenkinsfile, gradle/*, docker-compose.yaml, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, *.md, pom.xml, *.properties, *.json, *.xml, CODEOWNERS diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 623c3f1bd..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -So we can incorporate your pull request, please share the following: -* What issue are you addressing with this pull request? -* Are you modifying the correct branch? (See CONTRIBUTING.md) -* Have you run unit tests? (See CONTRIBUTING.md) -* Version of MarkLogic Java Client API (see Readme.txt) -* Version of MarkLogic Server (see admin gui on port 8001) -* Java version (`java -version`) -* OS and version -* What Changed: What happened before this change? What happens without this change? diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index d11ced4a0..4562e18d6 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -18,6 +18,6 @@ jobs: name: © Validate Copyright Headers uses: marklogic/pr-workflows/.github/workflows/copyright-check.yml@main permissions: - contents: read + contents: write pull-requests: write - issues: write \ No newline at end of file + issues: write diff --git a/CODEOWNERS b/CODEOWNERS index 94edd30a2..4eff9ce7f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,4 +2,4 @@ # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. -* @anu3990 @billfarber @rjrudin @stevebio +* @billfarber @rjrudin @stevebio diff --git a/Jenkinsfile b/Jenkinsfile index 080b15cf3..5225054e6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,12 +20,13 @@ def setupDockerMarkLogic(String image) { echo "Using image: "''' + image + ''' docker pull ''' + image + ''' MARKLOGIC_IMAGE=''' + image + ''' MARKLOGIC_LOGS_VOLUME=marklogicLogs docker compose up -d --build - echo "Waiting for MarkLogic server to initialize." - sleep 60s export JAVA_HOME=$JAVA_HOME_DIR export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR - export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH - ./gradlew mlTestConnections + export PATH=$JAVA_HOME/bin:$PATH + ./gradlew -i mlWaitTillReady + sleep 3 + ./gradlew -i mlWaitTillReady + ./gradlew mlTestConnections ./gradlew -i mlDeploy mlReloadSchemas ''' } @@ -171,6 +172,7 @@ pipeline { parameters { booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions') string(name: 'JAVA_VERSION', defaultValue: 'JAVA17', description: 'Either JAVA17 or JAVA21') + string(name: 'MARKLOGIC_IMAGE_TAGS', defaultValue: 'marklogic-server-ubi:latest-11,marklogic-server-ubi:latest-12', description: 'Comma-delimited list of MarkLogic image tags including variant (e.g., marklogic-server-ubi:latest-11,marklogic-server-ubi-rootless:11.3.2). The registry/org (ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic) path will be prepended automatically.') } environment { @@ -218,6 +220,7 @@ pipeline { } } } + stage('publish') { when { branch 'develop' @@ -237,60 +240,33 @@ pipeline { } } - stage('regressions-11') { + stage('regressions') { when { allOf { branch 'develop' expression { return params.regressions } } } - steps { - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11") - } - post { - always { - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() - } - } - } - // Latest run had 87 errors, which have been added to MLE-24523 for later research. -// stage('regressions-12-reverseProxy') { -// when { -// allOf { -// branch 'develop' -// expression {return params.regressions} -// } -// } -// steps { -// runTestsWithReverseProxy("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") -// } -// post { -// always { -// junit '**/build/**/TEST*.xml' -// updateWorkspacePermissions() -// tearDownDocker() -// } -// } -// } - - stage('regressions-12') { - when { - allOf { - branch 'develop' - expression { return params.regressions } - } - } steps { - runTests("ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12") - } - post { - always { - junit '**/build/**/TEST*.xml' - updateWorkspacePermissions() - tearDownDocker() + script { + def imageTags = params.MARKLOGIC_IMAGE_TAGS.split(',') + def imagePrefix = 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/' + + imageTags.each { tag -> + def fullImage = imagePrefix + tag.trim() + def stageName = "regressions-${tag.trim().replace(':', '-')}" + + stage(stageName) { + try { + runTests(fullImage) + } finally { + junit '**/build/**/TEST*.xml' + updateWorkspacePermissions() + tearDownDocker() + } + } + } } } } diff --git a/build.gradle b/build.gradle index 6ad238fe4..d83bfdc8c 100644 --- a/build.gradle +++ b/build.gradle @@ -23,9 +23,10 @@ subprojects { } repositories { - mavenLocal() mavenCentral() + mavenLocal() + // Needed so that ml-development-tools can resolve snapshots of marklogic-client-api. maven { url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" diff --git a/gradle.properties b/gradle.properties index dc8a5f9c0..d109496b4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,11 +1,13 @@ group=com.marklogic -version=8.0.0 +version=8.1-SNAPSHOT publishUrl=file:../marklogic-java/releases -okhttpVersion=5.2.0 +okhttpVersion=5.3.2 # See https://github.com/FasterXML/jackson for more information on the Jackson libraries. -jacksonVersion=2.20.0 +jacksonVersion=2.20.1 + +junitVersion=6.0.1 # Defined at this level so that they can be set as system properties and used by the marklogic-client-api and test-app # project diff --git a/marklogic-client-api-functionaltests/build.gradle b/marklogic-client-api-functionaltests/build.gradle index d9cebfa63..2dcda2303 100755 --- a/marklogic-client-api-functionaltests/build.gradle +++ b/marklogic-client-api-functionaltests/build.gradle @@ -7,26 +7,23 @@ dependencies { testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.4" testImplementation 'org.skyscreamer:jsonassert:1.5.3' testImplementation 'org.slf4j:slf4j-api:2.0.17' - testImplementation 'commons-io:commons-io:2.20.0' + testImplementation 'commons-io:commons-io:2.21.0' testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" testImplementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" testImplementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" testImplementation "org.jdom:jdom2:2.0.6.1" - testImplementation 'org.apache.commons:commons-lang3:3.19.0' + testImplementation 'org.apache.commons:commons-lang3:3.20.0' - // Allows talking to the Manage API. - testImplementation("com.marklogic:ml-app-deployer:6.0.1") { - exclude module: "marklogic-client-api" - } + testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" - testImplementation 'ch.qos.logback:logback-classic:1.5.18' - testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' + testImplementation 'ch.qos.logback:logback-classic:1.5.19' + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" + testImplementation 'org.xmlunit:xmlunit-legacy:2.11.0' // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.13.4" + testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junitVersion}" } tasks.withType(Test).configureEach { diff --git a/marklogic-client-api/build.gradle b/marklogic-client-api/build.gradle index a8c48a096..b292e3c79 100644 --- a/marklogic-client-api/build.gradle +++ b/marklogic-client-api/build.gradle @@ -6,6 +6,12 @@ plugins { id 'maven-publish' } +configurations.all { + // Ensure that no test dependencies below, namely marklogic-junit5, bring in their own version of marklogic-client-api, + // as we only want to use the one built by this project. + exclude group: 'com.marklogic', module: 'marklogic-client-api' +} + dependencies { // Using the latest version now that the 8.0.0 release requires Java 17. // This is now an implementation dependency as opposed to an api dependency in 7.x and earlier. @@ -31,32 +37,33 @@ dependencies { implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:${jacksonVersion}" + // Dependencies for hash generation. Can be safely omitted if not using the incremental write feature. But neither + // has any transitive dependencies, and thus their impact on the dependency tree is minimal. + implementation "io.github.erdtman:java-json-canonicalization:1.1" + implementation "net.openhft:zero-allocation-hashing:0.27ea1" + // Only used by extras (which some examples then depend on) compileOnly 'org.jdom:jdom2:2.0.6.1' compileOnly 'org.dom4j:dom4j:2.2.0' compileOnly 'com.google.code.gson:gson:2.13.2' - testImplementation 'org.junit.jupiter:junit-jupiter:5.13.4' + testImplementation "com.marklogic:marklogic-junit5:2.0-SNAPSHOT" - // Forcing junit version to avoid vulnerability with older version in xmlunit - testImplementation 'junit:junit:4.13.2' - testImplementation 'org.xmlunit:xmlunit-legacy:2.10.4' + testImplementation 'org.xmlunit:xmlunit-legacy:2.11.0' testImplementation project(':examples') - testImplementation 'org.apache.commons:commons-lang3:3.19.0' + testImplementation 'org.apache.commons:commons-lang3:3.20.0' // Allows talking to the Manage API. - testImplementation("com.marklogic:ml-app-deployer:6.0.1") { - exclude module: "marklogic-client-api" - } + testImplementation "com.marklogic:ml-app-deployer:6.2-SNAPSHOT" + + testImplementation "org.mockito:mockito-core:5.21.0" + testImplementation "org.mockito:mockito-inline:5.2.0" - // Starting with mockito 5.x, Java 11 is required, so sticking with 4.x as we have to support Java 8. - testImplementation "org.mockito:mockito-core:4.11.0" - testImplementation "org.mockito:mockito-inline:4.11.0" - testImplementation "com.squareup.okhttp3:mockwebserver3:5.1.0" + testImplementation "com.squareup.okhttp3:mockwebserver3:${okhttpVersion}" testImplementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${jacksonVersion}" - testImplementation 'ch.qos.logback:logback-classic:1.5.18' + testImplementation 'ch.qos.logback:logback-classic:1.5.23' // Using this to avoid a schema validation issue with the regular xercesImpl testImplementation 'org.opengis.cite.xerces:xercesImpl-xsd11:2.12-beta-r1667115' @@ -73,7 +80,7 @@ dependencies { // https://docs.gradle.org/current/userguide/upgrading_version_8.html#test_framework_implementation_dependencies // Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of: // org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests - testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.13.4" + testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junitVersion}" } // Ensure that mlHost and mlPassword can override the defaults of localhost/admin if they've been modified diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java index 976f870e6..a39d00802 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientBuilder.java @@ -29,7 +29,15 @@ public class DatabaseClientBuilder { public final static String PREFIX = "marklogic.client."; public final static String AUTH_TYPE_BASIC = "basic"; public final static String AUTH_TYPE_DIGEST = "digest"; - public final static String AUTH_TYPE_MARKLOGIC_CLOUD = "cloud"; + + public final static String AUTH_TYPE_CLOUD = "cloud"; + + /** + * @deprecated as of 8.1.0, use AUTH_TYPE_CLOUD instead + */ + @Deprecated + public final static String AUTH_TYPE_MARKLOGIC_CLOUD = AUTH_TYPE_CLOUD; + public final static String AUTH_TYPE_KERBEROS = "kerberos"; public final static String AUTH_TYPE_CERTIFICATE = "certificate"; public final static String AUTH_TYPE_SAML = "saml"; @@ -150,7 +158,7 @@ public DatabaseClientBuilder withDigestAuth(String username, String password) { } public DatabaseClientBuilder withCloudAuth(String apiKey, String basePath) { - return withAuthType(AUTH_TYPE_MARKLOGIC_CLOUD) + return withAuthType(AUTH_TYPE_CLOUD) .withCloudApiKey(apiKey) .withBasePath(basePath); } @@ -163,7 +171,7 @@ public DatabaseClientBuilder withCloudAuth(String apiKey, String basePath) { * @since 6.3.0 */ public DatabaseClientBuilder withCloudAuth(String apiKey, String basePath, Integer tokenDuration) { - return withAuthType(AUTH_TYPE_MARKLOGIC_CLOUD) + return withAuthType(AUTH_TYPE_CLOUD) .withCloudApiKey(apiKey) .withBasePath(basePath) .withCloudTokenDuration(tokenDuration != null ? tokenDuration.toString() : null); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java index 63e4cd212..ce053799c 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java @@ -7,6 +7,7 @@ import com.marklogic.client.impl.*; import com.marklogic.client.io.marker.ContentHandle; import com.marklogic.client.io.marker.ContentHandleFactory; +import com.progress.pdc.auth.okhttp.TokenInputs; import javax.naming.InvalidNameException; import javax.naming.ldap.LdapName; @@ -350,7 +351,7 @@ public SecurityContext withSSLContext(SSLContext context, X509TrustManager trust /** * @since 7.2.0 Replaced {@code MarkLogicCloudAuthContext} which was removed in 8.0.0 */ - public static class ProgressDataCloudAuthContext extends AuthContext { + public static class ProgressDataCloudAuthContext extends AuthContext implements TokenInputs { private String tokenEndpoint; private String grantType; private String apiKey; @@ -399,18 +400,22 @@ public ProgressDataCloudAuthContext(String apiKey, String tokenEndpoint, String this.tokenDuration = tokenDuration; } + @Override public String getTokenEndpoint() { return tokenEndpoint; } + @Override public String getGrantType() { return grantType; } + @Override public String getApiKey() { return apiKey; } + @Override public Integer getTokenDuration() { return tokenDuration; } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java index 31152d6ef..f024b55bf 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DataMovementManager.java @@ -7,6 +7,7 @@ import com.marklogic.client.io.marker.ContentHandle; import com.marklogic.client.query.*; +import java.io.Closeable; import java.util.Iterator; /** @@ -33,7 +34,15 @@ * dataMovementManager.release(); *} */ -public interface DataMovementManager { +public interface DataMovementManager extends Closeable { + + /** + * @since 8.1.0 + */ + default void close() { + release(); + } + /** Calls release() on all host-specific DatabaseClient instances (but not on * the DatabaseClient instance used to create this DataMovementManager * instance). diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java new file mode 100644 index 000000000..7f00d8161 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/DocumentWriteSetFilter.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.document.DocumentWriteSet; + +import java.util.function.Function; + +/** + * A filter that can modify a DocumentWriteSet before it is written to the database. + * + * @since 8.1.0 + */ +public interface DocumentWriteSetFilter extends Function { + + interface Context { + /** + * @return the DocumentWriteSet to be written + */ + DocumentWriteSet getDocumentWriteSet(); + + /** + * @return the batch number + */ + long getBatchNumber(); + + /** + * @return the DatabaseClient being used for this batch + */ + DatabaseClient getDatabaseClient(); + + /** + * @return the temporal collection name, or null if not writing to a temporal collection + */ + String getTemporalCollection(); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java index 2757ad58b..0facc145f 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/WriteBatcher.java @@ -3,15 +3,15 @@ */ package com.marklogic.client.datamovement; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - import com.marklogic.client.document.DocumentWriteOperation; import com.marklogic.client.document.ServerTransform; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.marker.AbstractWriteHandle; import com.marklogic.client.io.marker.DocumentMetadataWriteHandle; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + /** *

To facilitate long-running write jobs, batches documents added by many * external threads and coordinates internal threads to send the batches @@ -182,12 +182,7 @@ WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle, * * @param queryEvent the information about the batch that failed */ - public void retry(WriteBatch queryEvent); - - /* - public WriteBatcher withTransactionSize(int transactionSize); - public int getTransactionSize(); - */ + void retry(WriteBatch queryEvent); /** * Get the array of WriteBatchListener instances registered via @@ -361,5 +356,18 @@ WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle, * * @param writeBatch the information about the batch that failed */ - public void retryWithFailureListeners(WriteBatch writeBatch); + void retryWithFailureListeners(WriteBatch writeBatch); + + /** + * Sets a filter to modify or replace the DocumentWriteSet before it is written. + * The filter can return either the modified DocumentWriteSet or a new one. + * If the filter returns null or an empty DocumentWriteSet, no write will occur. + * + * @param filter the function to apply before writing + * @return this instance for method chaining + * @since 8.1.0 + */ + default WriteBatcher withDocumentWriteSetFilter(DocumentWriteSetFilter filter) { + return this; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java new file mode 100644 index 000000000..39cd5aee2 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/ContentExclusionUtil.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.core.JsonPointer; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.XmlFactories; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.namespace.NamespaceContext; +import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import java.io.ByteArrayInputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.Map; + +/** + * Utility class for applying content exclusions to documents before hash calculation. + * Supports removing specific paths from JSON and XML documents using JSON Pointer and XPath expressions. + * + * @since 8.1.0 + */ +class ContentExclusionUtil { + + private static final Logger logger = LoggerFactory.getLogger(ContentExclusionUtil.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + /** + * Applies JSON Pointer exclusions to JSON content by removing the specified paths. + * + * @param uri the document URI (used for logging purposes) + * @param jsonContent the JSON content as a string + * @param jsonPointers array of RFC 6901 JSON Pointer expressions identifying properties to exclude + * @return the modified JSON content with specified paths removed + * @throws JsonProcessingException if the JSON content cannot be parsed or serialized + */ + static String applyJsonExclusions(String uri, String jsonContent, String[] jsonPointers) throws JsonProcessingException { + if (jsonPointers == null || jsonPointers.length == 0) { + return jsonContent; + } + + JsonNode rootNode = OBJECT_MAPPER.readTree(jsonContent); + for (String jsonPointer : jsonPointers) { + removeNodeAtPointer(uri, rootNode, jsonPointer); + } + return OBJECT_MAPPER.writeValueAsString(rootNode); + } + + /** + * Removes a node at the specified JSON Pointer path from the given root node. + * + * @param uri the document URI (used for logging purposes) + * @param rootNode the root JSON node + * @param jsonPointer the JSON Pointer expression identifying the node to remove + */ + private static void removeNodeAtPointer(String uri, JsonNode rootNode, String jsonPointer) { + JsonPointer pointer = JsonPointer.compile(jsonPointer); + JsonNode targetNode = rootNode.at(pointer); + + if (targetNode.isMissingNode()) { + logger.debug("JSONPointer '{}' does not exist in document {}, skipping", jsonPointer, uri); + return; + } + + // Use Jackson's JsonPointer API to get parent and field name + JsonPointer parentPointer = pointer.head(); + JsonNode parentNode = rootNode.at(parentPointer); + + if (parentNode.isObject()) { + JsonPointer lastSegment = pointer.last(); + if (lastSegment != null) { + String fieldName = lastSegment.getMatchingProperty(); + ((ObjectNode) parentNode).remove(fieldName); + } + } else if (parentNode.isArray()) { + logger.warn("Array element exclusion not supported for JSONPointer '{}'. " + + "Consider excluding the entire array property instead.", jsonPointer); + } + } + + /** + * Applies XPath exclusions to XML content by removing the specified elements. + * + * @param uri the document URI (used for logging purposes) + * @param xmlContent the XML content as a string + * @param namespaces a map of namespace prefixes to URIs for use in XPath expressions, or null + * @param xpathExpressions array of XPath expressions identifying elements to exclude + * @return the modified XML content with specified elements removed + * @throws Exception if the XML content cannot be parsed or serialized + */ + static String applyXmlExclusions(String uri, String xmlContent, Map namespaces, String... xpathExpressions) throws Exception { + if (xpathExpressions == null || xpathExpressions.length == 0) { + return xmlContent; + } + + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); + Document document = builder.parse(new ByteArrayInputStream(xmlContent.getBytes(StandardCharsets.UTF_8))); + applyXmlExclusions(uri, document, namespaces, xpathExpressions); + return serializeDocument(document); + } + + private static void applyXmlExclusions(String uri, Document document, Map namespaces, String[] xpathExpressions) { + final XPath xpath = XmlFactories.getXPathFactory().newXPath(); + if (namespaces != null && !namespaces.isEmpty()) { + xpath.setNamespaceContext(new SimpleNamespaceContext(namespaces)); + } + + for (String xpathExpression : xpathExpressions) { + try { + XPathExpression expr = xpath.compile(xpathExpression); + QName returnType = XPathConstants.NODESET; + NodeList nodes = (NodeList) expr.evaluate(document, returnType); + + if (nodes.getLength() == 0) { + logger.debug("XPath '{}' does not match any nodes in document {}, skipping", xpathExpression, uri); + continue; + } + + // Remove nodes in reverse order to avoid index issues + for (int i = nodes.getLength() - 1; i >= 0; i--) { + Node node = nodes.item(i); + Node parent = node.getParentNode(); + if (parent != null) { + parent.removeChild(node); + } + } + } catch (XPathExpressionException e) { + logger.warn("Invalid XPath expression '{}' for document {}: {}", xpathExpression, uri, e.getMessage()); + } + } + } + + private static String serializeDocument(Document document) throws TransformerException { + Transformer transformer = XmlFactories.newTransformer(); + transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.setOutputProperty(OutputKeys.INDENT, "no"); + StringWriter writer = new StringWriter(); + transformer.transform(new DOMSource(document), new StreamResult(writer)); + return writer.toString(); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java new file mode 100644 index 000000000..fc0546798 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteEvalFilter.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.marklogic.client.FailedRequestException; +import com.marklogic.client.datamovement.DocumentWriteSetFilter; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.io.JacksonHandle; + +import java.util.Map; +import java.util.function.Consumer; + +/** + * Uses server-side JavaScript code to get the existing hash values for a set of URIs. + * + * @since 8.1.0 + */ +class IncrementalWriteEvalFilter extends IncrementalWriteFilter { + + private static final String EVAL_SCRIPT = """ + const tuples = cts.valueTuples([cts.uriReference(), cts.fieldReference(hashKeyName)], null, cts.documentQuery(uris)); + const response = {}; + for (var tuple of tuples) { + response[tuple[0]] = tuple[1]; + } + response + """; + + IncrementalWriteEvalFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + } + + @Override + public DocumentWriteSet apply(DocumentWriteSetFilter.Context context) { + ArrayNode uris = new ObjectMapper().createArrayNode(); + for (DocumentWriteOperation doc : context.getDocumentWriteSet()) { + if (DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(doc.getOperationType())) { + uris.add(doc.getUri()); + } + } + + try { + JsonNode response = context.getDatabaseClient().newServerEval().javascript(EVAL_SCRIPT) + .addVariable("hashKeyName", hashKeyName) + .addVariable("uris", new JacksonHandle(uris)) + .evalAs(JsonNode.class); + + return filterDocuments(context, uri -> { + if (response.has(uri)) { + return response.get(uri).asText(); + } + return null; + }); + } catch (FailedRequestException e) { + String message = "Unable to query for existing incremental write hashes; cause: " + e.getMessage(); + throw new FailedRequestException(message, e.getFailedRequest()); + } + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java new file mode 100644 index 000000000..730910c0b --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.core.JsonPointer; +import com.marklogic.client.datamovement.DocumentWriteSetFilter; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.impl.HandleAccessor; +import com.marklogic.client.impl.XmlFactories; +import com.marklogic.client.io.BaseHandle; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.Format; +import net.openhft.hashing.LongHashFunction; +import org.erdtman.jcs.JsonCanonicalizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A DocumentWriteSetFilter that skips writing documents whose content has not changed since the last write + * based on a hash value stored in a MarkLogic field. + * + * @since 8.1.0 + */ +public abstract class IncrementalWriteFilter implements DocumentWriteSetFilter { + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + + private String hashKeyName = "incrementalWriteHash"; + private String timestampKeyName = "incrementalWriteTimestamp"; + private boolean canonicalizeJson = true; + private boolean useEvalQuery = false; + private Consumer skippedDocumentsConsumer; + private String[] jsonExclusions; + private String[] xmlExclusions; + private Map xmlNamespaces; + + /** + * @param keyName the name of the MarkLogic metadata key that will hold the hash value; defaults to "incrementalWriteHash". + */ + public Builder hashKeyName(String keyName) { + // Don't let user shoot themselves in the foot with an empty key name. + if (keyName != null && !keyName.trim().isEmpty()) { + this.hashKeyName = keyName; + } + return this; + } + + /** + * @param keyName the name of the MarkLogic metadata key that will hold the timestamp value; defaults to "incrementalWriteTimestamp". + */ + public Builder timestampKeyName(String keyName) { + // Don't let user shoot themselves in the foot with an empty key name. + if (keyName != null && !keyName.trim().isEmpty()) { + this.timestampKeyName = keyName; + } + return this; + } + + /** + * @param canonicalizeJson whether to canonicalize JSON content before hashing; defaults to true. + * Delegates to https://github.com/erdtman/java-json-canonicalization for canonicalization. + */ + public Builder canonicalizeJson(boolean canonicalizeJson) { + this.canonicalizeJson = canonicalizeJson; + return this; + } + + /** + * @param useEvalQuery if true, evaluate server-side JavaScript instead of an Optic query for retrieving hash values; defaults to false. + */ + public Builder useEvalQuery(boolean useEvalQuery) { + this.useEvalQuery = useEvalQuery; + return this; + } + + /** + * @param skippedDocumentsConsumer a consumer that will be called with any documents in a batch that were skipped because their content had not changed. + */ + public Builder onDocumentsSkipped(Consumer skippedDocumentsConsumer) { + this.skippedDocumentsConsumer = skippedDocumentsConsumer; + return this; + } + + /** + * @param jsonPointers JSON Pointer expressions (RFC 6901) identifying JSON properties to exclude from hash calculation. + * For example, "/metadata/timestamp" or "/user/lastModified". + */ + public Builder jsonExclusions(String... jsonPointers) { + this.jsonExclusions = jsonPointers; + return this; + } + + /** + * @param xpathExpressions XPath expressions identifying XML elements to exclude from hash calculation. + * For example, "//timestamp" or "//metadata/lastModified". + */ + public Builder xmlExclusions(String... xpathExpressions) { + this.xmlExclusions = xpathExpressions; + return this; + } + + /** + * @param namespaces a map of namespace prefixes to URIs for use in XPath exclusion expressions. + * For example, Map.of("ns", "http://example.com/ns") allows XPath like "//ns:timestamp". + */ + public Builder xmlNamespaces(Map namespaces) { + this.xmlNamespaces = namespaces; + return this; + } + + public IncrementalWriteFilter build() { + validateJsonExclusions(); + validateXmlExclusions(); + if (useEvalQuery) { + return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + } + return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + } + + private void validateJsonExclusions() { + if (jsonExclusions == null) { + return; + } + for (String jsonPointer : jsonExclusions) { + if (jsonPointer == null || jsonPointer.trim().isEmpty()) { + throw new IllegalArgumentException( + "Empty JSON Pointer expression is not valid for excluding content from incremental write hash calculation; " + + "it would exclude the entire document. JSON Pointer expressions must start with '/'."); + } + try { + JsonPointer.compile(jsonPointer); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format("Invalid JSON Pointer expression '%s' for excluding content from incremental write hash calculation. " + + "JSON Pointer expressions must start with '/'; cause: %s", jsonPointer, e.getMessage()), e); + } + } + } + + private void validateXmlExclusions() { + if (xmlExclusions == null) { + return; + } + XPath xpath = XmlFactories.getXPathFactory().newXPath(); + if (xmlNamespaces != null && !xmlNamespaces.isEmpty()) { + xpath.setNamespaceContext(new SimpleNamespaceContext(xmlNamespaces)); + } + for (String xpathExpression : xmlExclusions) { + if (xpathExpression == null || xpathExpression.trim().isEmpty()) { + throw new IllegalArgumentException( + "Empty XPath expression is not valid for excluding content from incremental write hash calculation."); + } + try { + xpath.compile(xpathExpression); + } catch (XPathExpressionException e) { + throw new IllegalArgumentException( + String.format("Invalid XPath expression '%s' for excluding content from incremental write hash calculation; cause: %s", + xpathExpression, e.getMessage()), e); + } + } + } + } + + protected final String hashKeyName; + private final String timestampKeyName; + private final boolean canonicalizeJson; + private final Consumer skippedDocumentsConsumer; + private final String[] jsonExclusions; + private final String[] xmlExclusions; + private final Map xmlNamespaces; + + // Hardcoding this for now, with a good general purpose hashing function. + // See https://xxhash.com for benchmarks. + private final LongHashFunction hashFunction = LongHashFunction.xx3(); + + public IncrementalWriteFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { + this.hashKeyName = hashKeyName; + this.timestampKeyName = timestampKeyName; + this.canonicalizeJson = canonicalizeJson; + this.skippedDocumentsConsumer = skippedDocumentsConsumer; + this.jsonExclusions = jsonExclusions; + this.xmlExclusions = xmlExclusions; + this.xmlNamespaces = xmlNamespaces; + } + + protected final DocumentWriteSet filterDocuments(Context context, Function hashRetriever) { + final DocumentWriteSet newWriteSet = context.getDatabaseClient().newDocumentManager().newWriteSet(); + final List skippedDocuments = new ArrayList<>(); + final String timestamp = Instant.now().toString(); + + for (DocumentWriteOperation doc : context.getDocumentWriteSet()) { + if (!DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(doc.getOperationType())) { + newWriteSet.add(doc); + continue; + } + + final String serializedContent = serializeContent(doc); + if (serializedContent == null) { + // Not sure if the doc can have null content - possibly for a naked properties document? - but if it + // does, just include it in the write set. + newWriteSet.add(doc); + continue; + } + + final String contentHash = computeHash(serializedContent); + final String existingHash = hashRetriever.apply(doc.getUri()); + if (logger.isTraceEnabled()) { + logger.trace("URI: {}, existing Hash: {}, new Hash: {}", doc.getUri(), existingHash, contentHash); + } + + if (existingHash != null) { + if (!existingHash.equals(contentHash)) { + newWriteSet.add(addHashToMetadata(doc, hashKeyName, contentHash, timestampKeyName, timestamp)); + } else if (skippedDocumentsConsumer != null) { + skippedDocuments.add(doc); + } else { + // No consumer, so skip the document silently. + } + } else { + newWriteSet.add(addHashToMetadata(doc, hashKeyName, contentHash, timestampKeyName, timestamp)); + } + } + + if (!skippedDocuments.isEmpty() && skippedDocumentsConsumer != null) { + skippedDocumentsConsumer.accept(skippedDocuments.toArray(new DocumentWriteOperation[0])); + } + + return newWriteSet; + } + + private String serializeContent(DocumentWriteOperation doc) { + String content = HandleAccessor.contentAsString(doc.getContent()); + if (content == null) { + return null; + } + + Format format = null; + if (doc.getContent() instanceof BaseHandle baseHandle) { + format = baseHandle.getFormat(); + } + + if (canonicalizeJson && (Format.JSON.equals(format) || isPossiblyJsonContent(content))) { + JsonCanonicalizer jc; + try { + if (jsonExclusions != null && jsonExclusions.length > 0) { + content = ContentExclusionUtil.applyJsonExclusions(doc.getUri(), content, jsonExclusions); + } + jc = new JsonCanonicalizer(content); + return jc.getEncodedString(); + } catch (IOException e) { + // If the Format is actually JSON, then the write to MarkLogic should ultimately fail, which is the + // error message the user would want to see via a batch failure listener. So in all cases, if we cannot + // canonicalize something that appears to be JSON, we log a warning and return the original content for hashing. + logger.warn("Unable to canonicalize JSON content for URI {}, using original content for hashing; cause: {}", + doc.getUri(), e.getMessage()); + } + } else if (xmlExclusions != null && xmlExclusions.length > 0) { + try { + content = ContentExclusionUtil.applyXmlExclusions(doc.getUri(), content, xmlNamespaces, xmlExclusions); + } catch (Exception e) { + logger.warn("Unable to apply XML exclusions for URI {}, using original content for hashing; cause: {}", + doc.getUri(), e.getMessage()); + } + } + + return content; + } + + private boolean isPossiblyJsonContent(String content) { + // This isn't 100% reliable, as the content could be text that just happens to start with { or [, and so + // we'll still need to catch an exception if we try to canonicalize non-JSON content. + String trimmed = content.trim(); + return trimmed.startsWith("{") || trimmed.startsWith("["); + } + + private String computeHash(String content) { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + long hash = hashFunction.hashBytes(bytes); + return Long.toHexString(hash); + } + + protected static DocumentWriteOperation addHashToMetadata(DocumentWriteOperation op, String hashKeyName, String hash, + String timestampKeyName, String timestamp) { + DocumentMetadataHandle newMetadata = new DocumentMetadataHandle(); + if (op.getMetadata() != null) { + DocumentMetadataHandle originalMetadata = (DocumentMetadataHandle) op.getMetadata(); + newMetadata.setPermissions(originalMetadata.getPermissions()); + newMetadata.setCollections(originalMetadata.getCollections()); + newMetadata.setQuality(originalMetadata.getQuality()); + newMetadata.setProperties(originalMetadata.getProperties()); + newMetadata.getMetadataValues().putAll(originalMetadata.getMetadataValues()); + } + + newMetadata.getMetadataValues().put(hashKeyName, hash); + newMetadata.getMetadataValues().put(timestampKeyName, timestamp); + + return new DocumentWriteOperationImpl(op.getUri(), newMetadata, op.getContent(), op.getTemporalDocumentURI()); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java new file mode 100644 index 000000000..a52d21ad3 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteOpticFilter.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.FailedRequestException; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.row.RowTemplate; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Uses an Optic query to get the existing hash values for a set of URIs. + * + * @since 8.1.0 + */ +class IncrementalWriteOpticFilter extends IncrementalWriteFilter { + + IncrementalWriteOpticFilter(String hashKeyName, String timestampKeyName, boolean canonicalizeJson, + Consumer skippedDocumentsConsumer, String[] jsonExclusions, String[] xmlExclusions, Map xmlNamespaces) { + super(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions, xmlNamespaces); + } + + @Override + public DocumentWriteSet apply(Context context) { + final String[] uris = context.getDocumentWriteSet().stream() + .filter(op -> DocumentWriteOperation.OperationType.DOCUMENT_WRITE.equals(op.getOperationType())) + .map(DocumentWriteOperation::getUri) + .toArray(String[]::new); + + // It doesn't seem possible yet to use a DSL query and bind an array of strings to a "uris" param, so using + // a serialized query instead. That doesn't allow a user to override the query though. + RowTemplate rowTemplate = new RowTemplate(context.getDatabaseClient()); + + try { + Map existingHashes = rowTemplate.query(op -> + op.fromLexicons(Map.of( + "uri", op.cts.uriReference(), + "hash", op.cts.fieldReference(super.hashKeyName) + )).where( + op.cts.documentQuery(op.xs.stringSeq(uris)) + ), + + rows -> { + Map map = new HashMap<>(); + rows.forEach(row -> { + String uri = row.getString("uri"); + String existingHash = row.getString("hash"); + map.put(uri, existingHash); + }); + return map; + } + ); + + return filterDocuments(context, uri -> existingHashes.get(uri)); + } catch (FailedRequestException e) { + String message = "Unable to query for existing incremental write hashes; cause: " + e.getMessage(); + throw new FailedRequestException(message, e.getFailedRequest()); + } + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java new file mode 100644 index 000000000..196b36d71 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/SimpleNamespaceContext.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import javax.xml.namespace.NamespaceContext; +import java.util.Iterator; +import java.util.Map; + +/** + * A simple implementation of {@link NamespaceContext} backed by a Map of prefix to namespace URI mappings. + * Used for XPath evaluation with namespace-qualified expressions. + * + * @since 8.1.0 + */ +class SimpleNamespaceContext implements NamespaceContext { + + private final Map prefixToNamespaceUri; + + SimpleNamespaceContext(Map prefixToNamespaceUri) { + this.prefixToNamespaceUri = prefixToNamespaceUri; + } + + @Override + public String getNamespaceURI(String prefix) { + return prefixToNamespaceUri.get(prefix); + } + + @Override + public String getPrefix(String namespaceURI) { + for (Map.Entry entry : prefixToNamespaceUri.entrySet()) { + if (entry.getValue().equals(namespaceURI)) { + return entry.getKey(); + } + } + return null; + } + + @Override + public Iterator getPrefixes(String namespaceURI) { + return prefixToNamespaceUri.entrySet().stream() + .filter(entry -> entry.getValue().equals(namespaceURI)) + .map(Map.Entry::getKey) + .iterator(); + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java index ce4426563..f6e91c91a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriteSet.java @@ -3,121 +3,119 @@ */ package com.marklogic.client.datamovement.impl; -import java.util.function.Consumer; - import com.marklogic.client.DatabaseClient; -import com.marklogic.client.document.DocumentWriteSet; -import com.marklogic.client.document.ServerTransform; +import com.marklogic.client.datamovement.DocumentWriteSetFilter; import com.marklogic.client.datamovement.WriteBatch; import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.datamovement.WriteEvent; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.document.ServerTransform; + +import java.util.function.Consumer; -public class BatchWriteSet { - private WriteBatcher batcher; - private DocumentWriteSet writeSet; - private long batchNumber; - private long itemsSoFar; - private DatabaseClient client; - private ServerTransform transform; - private String temporalCollection; - private Runnable onSuccess; - private Consumer onFailure; - private Runnable onBeforeWrite; - - public BatchWriteSet(WriteBatcher batcher, DocumentWriteSet writeSet, DatabaseClient client, - ServerTransform transform, String temporalCollection) - { - this.batcher = batcher; - this.writeSet = writeSet; - this.client = client; - this.transform = transform; - this.temporalCollection = temporalCollection; - } - - public DocumentWriteSet getWriteSet() { - return writeSet; - } - - public void setWriteSet(DocumentWriteSet writeSet) { - this.writeSet = writeSet; - } - - public long getBatchNumber() { - return batchNumber; - } - - public void setBatchNumber(long batchNumber) { - this.batchNumber = batchNumber; - } - - public void setItemsSoFar(long itemsSoFar) { - this.itemsSoFar = itemsSoFar; - } - - public DatabaseClient getClient() { - return client; - } - - public void setClient(DatabaseClient client) { - this.client = client; - } - - public ServerTransform getTransform() { - return transform; - } - - public void setTransform(ServerTransform transform) { - this.transform = transform; - } - - public String getTemporalCollection() { - return temporalCollection; - } - - public void setTemporalCollection(String temporalCollection) { - this.temporalCollection = temporalCollection; - } - - public Runnable getOnSuccess() { - return onSuccess; - } - - public void onSuccess(Runnable onSuccess) { - this.onSuccess = onSuccess; - } - - public Consumer getOnFailure() { - return onFailure; - } - - public void onFailure(Consumer onFailure) { - this.onFailure = onFailure; - } - - public Runnable getOnBeforeWrite() { - return onBeforeWrite; - } - - public void onBeforeWrite(Runnable onBeforeWrite) { - this.onBeforeWrite = onBeforeWrite; - } - - public WriteBatch getBatchOfWriteEvents() { - WriteBatchImpl batch = new WriteBatchImpl() - .withBatcher(batcher) - .withClient(client) - .withJobBatchNumber(batchNumber) - .withJobWritesSoFar(itemsSoFar) - .withJobTicket(batcher.getJobTicket()); - WriteEvent[] writeEvents = getWriteSet().stream() - .map(writeOperation -> - new WriteEventImpl() - .withTargetUri(writeOperation.getUri()) - .withContent(writeOperation.getContent()) - .withMetadata(writeOperation.getMetadata()) - ) - .toArray(WriteEventImpl[]::new); - batch.withItems(writeEvents); - return batch; - } +/** + * Mutable class that captures the documents to be written. Documents are added via calls to "getDocumentWriteSet()", where the + * DocumentWriteSet is empty when this class is constructed. + */ +class BatchWriteSet implements DocumentWriteSetFilter.Context { + + private final WriteBatcher batcher; + private final long batchNumber; + private final DatabaseClient client; + private final ServerTransform transform; + private final String temporalCollection; + + // Can be overridden after creation + private DocumentWriteSet documentWriteSet; + + private long itemsSoFar; + private Runnable onSuccess; + private Consumer onFailure; + + BatchWriteSet(WriteBatcher batcher, DatabaseClient hostClient, ServerTransform transform, String temporalCollection, long batchNumber) { + this.batcher = batcher; + this.documentWriteSet = hostClient.newDocumentManager().newWriteSet(); + this.client = hostClient; + this.transform = transform; + this.temporalCollection = temporalCollection; + this.batchNumber = batchNumber; + } + + /** + * Must be called if a DocumentWriteSetFilter modified the DocumentWriteSet owned by this class. + * + * @since 8.1.0 + */ + void updateWithFilteredDocumentWriteSet(DocumentWriteSet filteredDocumentWriteSet) { + this.documentWriteSet = filteredDocumentWriteSet; + } + + @Override + public DocumentWriteSet getDocumentWriteSet() { + return documentWriteSet; + } + + @Override + public long getBatchNumber() { + return batchNumber; + } + + public void setItemsSoFar(long itemsSoFar) { + this.itemsSoFar = itemsSoFar; + } + + @Override + public DatabaseClient getDatabaseClient() { + return client; + } + + public DatabaseClient getClient() { + return client; + } + + public ServerTransform getTransform() { + return transform; + } + + @Override + public String getTemporalCollection() { + return temporalCollection; + } + + public Runnable getOnSuccess() { + return onSuccess; + } + + public void onSuccess(Runnable onSuccess) { + this.onSuccess = onSuccess; + } + + public Consumer getOnFailure() { + return onFailure; + } + + public void onFailure(Consumer onFailure) { + this.onFailure = onFailure; + } + + public WriteBatch getBatchOfWriteEvents() { + WriteBatchImpl batch = new WriteBatchImpl() + .withBatcher(batcher) + .withClient(client) + .withJobBatchNumber(batchNumber) + .withJobWritesSoFar(itemsSoFar) + .withJobTicket(batcher.getJobTicket()); + + WriteEvent[] writeEvents = getDocumentWriteSet().stream() + .map(writeOperation -> + new WriteEventImpl() + .withTargetUri(writeOperation.getUri()) + .withContent(writeOperation.getContent()) + .withMetadata(writeOperation.getMetadata()) + ) + .toArray(WriteEventImpl[]::new); + + batch.withItems(writeEvents); + return batch; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java new file mode 100644 index 000000000..a2ebe835d --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/BatchWriter.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.impl; + +import com.marklogic.client.datamovement.DocumentWriteSetFilter; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.document.DocumentWriteSet; +import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.io.Format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.util.function.Consumer; + +record BatchWriter(BatchWriteSet batchWriteSet, DocumentWriteSetFilter filter) implements Runnable { + + private static Logger logger = LoggerFactory.getLogger(WriteBatcherImpl.class); + + @Override + public void run() { + if (batchWriteSet.getDocumentWriteSet() == null || batchWriteSet.getDocumentWriteSet().isEmpty()) { + logger.debug("Unexpected empty batch {}, skipping", batchWriteSet.getBatchNumber()); + return; + } + + try { + logger.trace("Begin write batch {} to forest on host '{}'", batchWriteSet.getBatchNumber(), batchWriteSet.getClient().getHost()); + + DocumentWriteSet documentWriteSet = batchWriteSet.getDocumentWriteSet(); + if (filter != null) { + documentWriteSet = filter.apply(batchWriteSet); + if (documentWriteSet == null || documentWriteSet.isEmpty()) { + logger.debug("Filter returned empty write set for batch {}, skipping write", batchWriteSet.getBatchNumber()); + closeAllHandles(); + return; + } + batchWriteSet.updateWithFilteredDocumentWriteSet(documentWriteSet); + } + + writeDocuments(documentWriteSet); + + // This seems like it should be part of a finally block - but it's able to throw an exception. Which implies + // that onFailure() should occur when this fails, which seems odd??? + closeAllHandles(); + + onSuccess(); + } catch (Throwable t) { + onFailure(t); + } + } + + private void writeDocuments(DocumentWriteSet documentWriteSet) { + if (batchWriteSet.getTemporalCollection() == null) { + batchWriteSet.getClient().newDocumentManager().write(documentWriteSet, batchWriteSet.getTransform(), null); + } else { + // to get access to the TemporalDocumentManager write overload we need to instantiate + // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content + // format, so we'll set the default content format to unknown + XMLDocumentManager docMgr = batchWriteSet.getClient().newXMLDocumentManager(); + docMgr.setContentFormat(Format.UNKNOWN); + docMgr.write(documentWriteSet, batchWriteSet.getTransform(), null, batchWriteSet.getTemporalCollection()); + } + } + + private void onSuccess() { + Runnable onSuccess = batchWriteSet.getOnSuccess(); + if (onSuccess != null) { + onSuccess.run(); + } + } + + private void onFailure(Throwable t) { + logger.trace("Failed batch sent to forest on host \"{}\"", batchWriteSet.getClient().getHost()); + Consumer onFailure = batchWriteSet.getOnFailure(); + if (onFailure != null) { + onFailure.accept(t); + } + } + + private void closeAllHandles() throws Throwable { + Throwable lastThrowable = null; + for (DocumentWriteOperation doc : batchWriteSet.getDocumentWriteSet()) { + try { + if (doc.getContent() instanceof Closeable closeable) { + closeable.close(); + } + if (doc.getMetadata() instanceof Closeable closeable) { + closeable.close(); + } + } catch (Throwable t) { + logger.error("Error closing all handles in BatchWriter", t); + lastThrowable = t; + } + } + if (lastThrowable != null) throw lastThrowable; + } +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java index d31a46fbe..7b2362c8a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementManagerImpl.java @@ -1,11 +1,10 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientBuilder; -import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.datamovement.*; import com.marklogic.client.impl.DatabaseClientImpl; import com.marklogic.client.io.marker.ContentHandle; @@ -123,10 +122,8 @@ private QueryBatcher newQueryBatcherImpl(SearchQueryDefinition query) { QueryBatcherImpl queryBatcher = null; // preprocess the query if the effective version is at least 10.0-5 if (Long.compareUnsigned(getServerVersion(), Long.parseUnsignedLong("10000500")) >= 0) { - DataMovementServices.QueryConfig queryConfig = service.initConfig("POST", query); - queryBatcher = new QueryBatcherImpl(query, this, queryConfig.forestConfig, - queryConfig.serializedCtsQuery, queryConfig.filtered, - queryConfig.maxDocToUriBatchRatio, queryConfig.defaultDocBatchSize, queryConfig.maxUriBatchSize); + QueryConfig queryConfig = service.initConfig("POST", query); + queryBatcher = new QueryBatcherImpl(query, this, queryConfig); } else { queryBatcher = new QueryBatcherImpl(query, this, getForestConfig()); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java index 8699c81e8..de1754a50 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; @@ -23,7 +23,8 @@ import java.util.List; public class DataMovementServices { - private static Logger logger = LoggerFactory.getLogger(DataMovementServices.class); + + private static final Logger logger = LoggerFactory.getLogger(DataMovementServices.class); private DatabaseClient client; @@ -36,59 +37,65 @@ public DataMovementServices setClient(DatabaseClient client) { return this; } - QueryConfig initConfig(String method, SearchQueryDefinition qdef) { - logger.debug("initializing forest configuration with query"); - if (qdef == null) throw new IllegalArgumentException("null query definition"); - - JsonNode result = ((DatabaseClientImpl) this.client).getServices() - .forestInfo(null, method, new RequestParameters(), qdef, new JacksonHandle()) - .get(); - // System.out.println(result.toPrettyString()); - - QueryConfig queryConfig = new QueryConfig(); - - try { - ObjectMapper mapper = new ObjectMapper(); - JsonNode queryResult = result.get("query"); - if (queryResult != null && queryResult.isObject() && queryResult.has("ctsquery")) { - queryConfig.serializedCtsQuery = mapper.writeValueAsString(queryResult); - logger.debug("initialized query to: {}", queryConfig.serializedCtsQuery); - } - JsonNode filteredResult = result.get("filtered"); - if (filteredResult != null && filteredResult.isBoolean()) { - queryConfig.filtered = filteredResult.asBoolean(); - logger.debug("initialized filtering to: {}", queryConfig.filtered.toString()); - } - JsonNode maxDocToUriBatchRatio = result.get("maxDocToUriBatchRatio"); - if (maxDocToUriBatchRatio != null && maxDocToUriBatchRatio.isInt()) { - queryConfig.maxDocToUriBatchRatio = maxDocToUriBatchRatio.asInt(); - logger.debug("initialized maxDocToUriBatchRatio to : {}", queryConfig.maxDocToUriBatchRatio); - } else { - queryConfig.maxDocToUriBatchRatio = -1; - } - JsonNode defaultDocBatchSize = result.get("defaultDocBatchSize"); - if (defaultDocBatchSize != null && defaultDocBatchSize.isInt()) { - queryConfig.defaultDocBatchSize = defaultDocBatchSize.asInt(); - logger.debug("initialized defaultDocBatchSize to : {}", queryConfig.defaultDocBatchSize); - } else { - queryConfig.defaultDocBatchSize = -1; - } - JsonNode maxUriBatchSize = result.get("maxUriBatchSize"); - if (maxUriBatchSize != null && maxUriBatchSize.isInt()) { - queryConfig.maxUriBatchSize = maxUriBatchSize.asInt(); - logger.debug("initialized maxUriBatchSize to : {}", queryConfig.maxUriBatchSize); - } else { - queryConfig.maxUriBatchSize = -1; - } - - } catch (JsonProcessingException e) { - logger.error("failed to initialize query", e); - } - - queryConfig.forestConfig = makeForestConfig(result.has("forests") ? result.get("forests") : result); - - return queryConfig; - } + QueryConfig initConfig(String method, SearchQueryDefinition qdef) { + logger.debug("initializing forest configuration with query"); + if (qdef == null) throw new IllegalArgumentException("null query definition"); + + JsonNode result = ((DatabaseClientImpl) this.client).getServices() + .forestInfo(null, method, new RequestParameters(), qdef, new JacksonHandle()) + .get(); + + JsonNode queryResult = result.get("query"); + + String serializedCtsQuery = null; + if (queryResult != null && queryResult.isObject() && queryResult.has("ctsquery")) { + try { + serializedCtsQuery = new ObjectMapper().writeValueAsString(queryResult); + logger.debug("initialized query to: {}", serializedCtsQuery); + } catch (JsonProcessingException e) { + logger.warn("Unable to serialize query result while initializing QueryBatcher; cause: {}", e.getMessage()); + } + } + + JsonNode filteredResult = result.get("filtered"); + Boolean filtered = null; + if (filteredResult != null && filteredResult.isBoolean()) { + filtered = filteredResult.asBoolean(); + logger.debug("initialized filtering to: {}", filtered); + } + + JsonNode maxDocToUriBatchRatioNode = result.get("maxDocToUriBatchRatio"); + int maxDocToUriBatchRatio = -1; + if (maxDocToUriBatchRatioNode != null && maxDocToUriBatchRatioNode.isInt()) { + maxDocToUriBatchRatio = maxDocToUriBatchRatioNode.asInt(); + logger.debug("initialized maxDocToUriBatchRatio to : {}", maxDocToUriBatchRatio); + } + + // Per GitHub bug 1872 and MLE-26460, the server may return -1 when there are fewer server threads than forests. + // A value of -1 will cause later problems when constructing a LinkedBlockingQueue with a negative capacity. + // So defaulting this to 1 to avoid later errors. + if (maxDocToUriBatchRatio <= 0) { + maxDocToUriBatchRatio = 1; + } + + JsonNode defaultDocBatchSizeNode = result.get("defaultDocBatchSize"); + int defaultDocBatchSize = -1; + if (defaultDocBatchSizeNode != null && defaultDocBatchSizeNode.isInt()) { + defaultDocBatchSize = defaultDocBatchSizeNode.asInt(); + logger.debug("initialized defaultDocBatchSize to : {}", defaultDocBatchSize); + } + + JsonNode maxUriBatchSizeNode = result.get("maxUriBatchSize"); + int maxUriBatchSize = -1; + if (maxUriBatchSizeNode != null && maxUriBatchSizeNode.isInt()) { + maxUriBatchSize = maxUriBatchSizeNode.asInt(); + logger.debug("initialized maxUriBatchSize to : {}", maxUriBatchSize); + } + + ForestConfiguration forestConfig = makeForestConfig(result.has("forests") ? result.get("forests") : result); + return new QueryConfig(serializedCtsQuery, forestConfig, filtered, + maxDocToUriBatchRatio, defaultDocBatchSize, maxUriBatchSize); + } ForestConfigurationImpl readForestConfig() { logger.debug("initializing forest configuration"); @@ -183,12 +190,4 @@ private String generateJobId() { return UUID.randomUUID().toString(); } - static class QueryConfig { - String serializedCtsQuery; - ForestConfiguration forestConfig; - Boolean filtered; - int maxDocToUriBatchRatio; - int defaultDocBatchSize; - int maxUriBatchSize; - } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java index f9581867a..031e8b3df 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryBatcherImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.datamovement.impl; @@ -42,16 +42,17 @@ * startIterating, withForestConfig, and retry. */ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { - private static Logger logger = LoggerFactory.getLogger(QueryBatcherImpl.class); + private static final Logger logger = LoggerFactory.getLogger(QueryBatcherImpl.class); private String queryMethod; private SearchQueryDefinition query; - private SearchQueryDefinition originalQuery; private Boolean filtered; private Iterator iterator; private boolean threadCountSet = false; - private List urisReadyListeners = new ArrayList<>(); - private List failureListeners = new ArrayList<>(); - private List jobCompletionListeners = new ArrayList<>(); + + private final List urisReadyListeners = new ArrayList<>(); + private final List failureListeners = new ArrayList<>(); + private final List jobCompletionListeners = new ArrayList<>(); + private QueryThreadPoolExecutor threadPool; private boolean consistentSnapshot = false; private final AtomicLong batchNumber = new AtomicLong(0); @@ -61,10 +62,13 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { private Map forestResults = new HashMap<>(); private Map forestIsDone = new HashMap<>(); private Map retryForestMap = new HashMap<>(); - private AtomicBoolean runJobCompletionListeners = new AtomicBoolean(false); + + private final AtomicBoolean runJobCompletionListeners = new AtomicBoolean(false); private final Object lock = new Object(); private final Map> blackListedTasks = new HashMap<>(); + private boolean isSingleThreaded = false; + private long maxUris = Long.MAX_VALUE; private long maxBatches = Long.MAX_VALUE; private int maxDocToUriBatchRatio; @@ -72,40 +76,37 @@ public class QueryBatcherImpl extends BatcherImpl implements QueryBatcher { private int defaultDocBatchSize; private int maxUriBatchSize; - QueryBatcherImpl( - SearchQueryDefinition originalQuery, DataMovementManager moveMgr, ForestConfiguration forestConfig, - String serializedCtsQuery, Boolean filtered, int maxDocToUriBatchRatio, int defaultDocBatchSize, int maxUriBatchSize - ) { - this(moveMgr, forestConfig, maxDocToUriBatchRatio, defaultDocBatchSize, maxUriBatchSize); - // TODO: skip conversion in DataMovementManagerImpl.newQueryBatcherImpl() unless canSerializeQueryAsJSON() - if (serializedCtsQuery != null && serializedCtsQuery.length() > 0 && - originalQuery instanceof AbstractSearchQueryDefinition && - ((AbstractSearchQueryDefinition) originalQuery).canSerializeQueryAsJSON()) { - QueryManagerImpl queryMgr = (QueryManagerImpl) getPrimaryClient().newQueryManager(); - this.queryMethod = "POST"; - this.query = queryMgr.newRawCtsQueryDefinition(new StringHandle(serializedCtsQuery).withFormat(Format.JSON)); - this.originalQuery = originalQuery; - if (filtered != null) { - this.filtered = filtered; - } - } else { - initQuery(originalQuery); - } - } + QueryBatcherImpl(SearchQueryDefinition originalQuery, DataMovementManager moveMgr, QueryConfig queryConfig) { + this(moveMgr, queryConfig); + + final String serializedCtsQuery = queryConfig.serializedCtsQuery(); + if (serializedCtsQuery != null && !serializedCtsQuery.isEmpty() && + originalQuery instanceof AbstractSearchQueryDefinition && + ((AbstractSearchQueryDefinition) originalQuery).canSerializeQueryAsJSON()) { + QueryManagerImpl queryMgr = (QueryManagerImpl) getPrimaryClient().newQueryManager(); + this.queryMethod = "POST"; + this.query = queryMgr.newRawCtsQueryDefinition(new StringHandle(serializedCtsQuery).withFormat(Format.JSON)); + this.filtered = queryConfig.filtered(); + } else { + initQuery(originalQuery); + } + } + public QueryBatcherImpl(SearchQueryDefinition query, DataMovementManager moveMgr, ForestConfiguration forestConfig) { this(moveMgr, forestConfig); initQuery(query); } + public QueryBatcherImpl(Iterator iterator, DataMovementManager moveMgr, ForestConfiguration forestConfig) { this(moveMgr, forestConfig); this.iterator = iterator; } - private QueryBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forestConfig, - int maxDocToUriBatchRatio, int defaultDocBatchSize, int maxUriBatchSize) { - this(moveMgr, forestConfig); - this.maxDocToUriBatchRatio = maxDocToUriBatchRatio; - this.defaultDocBatchSize = defaultDocBatchSize; - this.maxUriBatchSize = maxUriBatchSize; + + private QueryBatcherImpl(DataMovementManager moveMgr, QueryConfig queryConfig) { + this(moveMgr, queryConfig.forestConfig()); + this.maxDocToUriBatchRatio = queryConfig.maxDocToUriBatchRatio(); + this.defaultDocBatchSize = queryConfig.defaultDocBatchSize(); + this.maxUriBatchSize = queryConfig.maxUriBatchSize(); withBatchSize(defaultDocBatchSize); } private QueryBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forestConfig) { @@ -187,7 +188,7 @@ public void retryWithFailureListeners(QueryEvent queryEvent) { } private void retry(QueryEvent queryEvent, boolean callFailListeners) { - if ( isStopped() == true ) { + if ( isStopped()) { logger.warn("Job is now stopped, aborting the retry"); return; } @@ -449,7 +450,7 @@ public synchronized void start(JobTicket ticket) { private synchronized void initialize() { Forest[] forests = getForestConfig().listForests(); - if ( threadCountSet == false ) { + if ( !threadCountSet ) { if ( query != null ) { logger.warn("threadCount not set--defaulting to number of forests ({})", forests.length); withThreadCount(forests.length * docToUriBatchRatio); @@ -529,7 +530,7 @@ public synchronized QueryBatcher withForestConfig(ForestConfiguration forestConf List newClientList = clients(hostNames); clientList.set(newClientList); boolean started = (threadPool != null); - if ( started == true && oldForests.size() > 0 ) calculateDeltas(oldForests, forests); + if ( started && !oldForests.isEmpty() ) calculateDeltas(oldForests, forests); return this; } @@ -550,7 +551,7 @@ private synchronized void calculateDeltas(Set oldForests, Forest[] fores // this forest is not black-listed blackListedForests.remove(forest); } - if ( blackListedForests.size() > 0 ) { + if ( !blackListedForests.isEmpty() ) { DataMovementManagerImpl moveMgrImpl = getMoveMgr(); String primaryHost = moveMgrImpl.getPrimaryClient().getHost(); if ( getHostNames(blackListedForests).contains(primaryHost) ) { @@ -562,7 +563,7 @@ private synchronized void calculateDeltas(Set oldForests, Forest[] fores } private synchronized void cleanupExistingTasks(Set addedForests, Set restartedForests, Set blackListedForests) { - if ( blackListedForests.size() > 0 ) { + if ( !blackListedForests.isEmpty() ) { logger.warn("removing jobs related to hosts [{}] from the queue", getHostNames(blackListedForests)); // since some forests have been removed, let's remove from the queue any jobs that were targeting that forest List tasks = new ArrayList<>(); diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java new file mode 100644 index 000000000..f46883622 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/QueryConfig.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.impl; + +import com.marklogic.client.datamovement.ForestConfiguration; + +record QueryConfig( + String serializedCtsQuery, + ForestConfiguration forestConfig, + Boolean filtered, + int maxDocToUriBatchRatio, + int defaultDocBatchSize, + int maxUriBatchSize +) { +} diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java index a87775285..1b376fb85 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/datamovement/impl/WriteBatcherImpl.java @@ -3,45 +3,25 @@ */ package com.marklogic.client.datamovement.impl; -import java.io.Closeable; -import java.util.*; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; -import java.util.stream.Stream; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.marklogic.client.DatabaseClient; import com.marklogic.client.DatabaseClientFactory; +import com.marklogic.client.datamovement.*; import com.marklogic.client.document.DocumentWriteOperation; -import com.marklogic.client.document.ServerTransform; -import com.marklogic.client.document.XMLDocumentManager; import com.marklogic.client.document.DocumentWriteOperation.OperationType; -import com.marklogic.client.io.DocumentMetadataHandle; -import com.marklogic.client.io.Format; +import com.marklogic.client.document.ServerTransform; import com.marklogic.client.impl.DocumentWriteOperationImpl; import com.marklogic.client.impl.Utilities; +import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.marker.AbstractWriteHandle; import com.marklogic.client.io.marker.ContentHandle; import com.marklogic.client.io.marker.DocumentMetadataWriteHandle; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import com.marklogic.client.datamovement.DataMovementException; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.Forest; -import com.marklogic.client.datamovement.ForestConfiguration; -import com.marklogic.client.datamovement.JobTicket; -import com.marklogic.client.datamovement.WriteBatch; -import com.marklogic.client.datamovement.WriteBatchListener; -import com.marklogic.client.datamovement.WriteEvent; -import com.marklogic.client.datamovement.WriteFailureListener; -import com.marklogic.client.datamovement.WriteBatcher; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; /** * The implementation of WriteBatcher. @@ -146,6 +126,7 @@ public class WriteBatcherImpl private boolean initialized = false; private CompletableThreadPoolExecutor threadPool = null; private DocumentMetadataHandle defaultMetadata; + private DocumentWriteSetFilter documentWriteSetFilter; public WriteBatcherImpl(DataMovementManager moveMgr, ForestConfiguration forestConfig) { super(moveMgr); @@ -208,7 +189,7 @@ public WriteBatcher add(DocumentWriteOperation writeOperation) { BatchWriteSet writeSet = newBatchWriteSet(); int minBatchSize = 0; if(defaultMetadata != null) { - writeSet.getWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); + writeSet.getDocumentWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); minBatchSize = 1; } for (int i=0; i < getBatchSize(); i++ ) { @@ -217,10 +198,10 @@ public WriteBatcher add(DocumentWriteOperation writeOperation) { // strange, there should have been a full batch of docs in the queue... break; } - writeSet.getWriteSet().add(doc); + writeSet.getDocumentWriteSet().add(doc); } - if ( writeSet.getWriteSet().size() > minBatchSize ) { - threadPool.submit( new BatchWriter(writeSet) ); + if ( writeSet.getDocumentWriteSet().size() > minBatchSize ) { + threadPool.submit( new BatchWriter(writeSet, documentWriteSetFilter) ); } } return this; @@ -258,19 +239,19 @@ public WriteBatcher addAs(String uri, DocumentMetadataWriteHandle metadataHandle } private void requireInitialized() { - if ( initialized == false ) { + if (!initialized) { throw new IllegalStateException("This operation must be called after starting this job"); } } private void requireNotInitialized() { - if ( initialized == true ) { + if (initialized) { throw new IllegalStateException("Configuration cannot be changed after starting this job or calling add or addAs"); } } private void requireNotStopped() { - if ( isStopped() == true ) throw new IllegalStateException("This instance has been stopped"); + if (isStopped()) throw new IllegalStateException("This instance has been stopped"); } private BatchWriteSet newBatchWriteSet() { @@ -281,16 +262,9 @@ private BatchWriteSet newBatchWriteSet() { private BatchWriteSet newBatchWriteSet(long batchNum) { int hostToUse = (int) (batchNum % hostInfos.length); HostInfo host = hostInfos[hostToUse]; - DatabaseClient hostClient = host.client; - BatchWriteSet batchWriteSet = new BatchWriteSet(this, hostClient.newDocumentManager().newWriteSet(), - hostClient, getTransform(), getTemporalCollection()); - batchWriteSet.setBatchNumber(batchNum); - batchWriteSet.onSuccess( () -> { - sendSuccessToListeners(batchWriteSet); - }); - batchWriteSet.onFailure( (throwable) -> { - sendThrowableToListeners(throwable, "Error writing batch: {}", batchWriteSet); - }); + BatchWriteSet batchWriteSet = new BatchWriteSet(this, host.client, getTransform(), getTemporalCollection(), batchNum); + batchWriteSet.onSuccess( () -> sendSuccessToListeners(batchWriteSet)); + batchWriteSet.onFailure(throwable -> sendThrowableToListeners(throwable, batchWriteSet)); return batchWriteSet; } @@ -318,7 +292,7 @@ public void retry(WriteBatch batch) { } private void retry(WriteBatch batch, boolean callFailListeners) { - if ( isStopped() == true ) { + if (isStopped()) { logger.warn("Job is now stopped, aborting the retry"); return; } @@ -333,9 +307,9 @@ private void retry(WriteBatch batch, boolean callFailListeners) { }); } for (WriteEvent doc : batch.getItems()) { - writeSet.getWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); + writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } - BatchWriter runnable = new BatchWriter(writeSet); + BatchWriter runnable = new BatchWriter(writeSet, documentWriteSetFilter); runnable.run(); } @Override @@ -392,28 +366,28 @@ private void flush(boolean waitForCompletion) { } Iterator iter = docs.iterator(); for ( int i=0; iter.hasNext(); i++ ) { - if ( isStopped() == true ) { + if (isStopped()) { logger.warn("Job is now stopped, preventing the flush of {} queued docs", docs.size() - i); - if ( waitForCompletion == true ) awaitCompletion(); + if (waitForCompletion) awaitCompletion(); return; } BatchWriteSet writeSet = newBatchWriteSet(); if(defaultMetadata != null) { - writeSet.getWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); + writeSet.getDocumentWriteSet().add(new DocumentWriteOperationImpl(OperationType.METADATA_DEFAULT, null, defaultMetadata, null)); } int j=0; for ( ; j < getBatchSize() && iter.hasNext(); j++ ) { DocumentWriteOperation doc = iter.next(); - writeSet.getWriteSet().add(doc); + writeSet.getDocumentWriteSet().add(doc); } - threadPool.submit( new BatchWriter(writeSet) ); + threadPool.submit( new BatchWriter(writeSet, documentWriteSetFilter) ); } - if ( waitForCompletion == true ) awaitCompletion(); + if (waitForCompletion) awaitCompletion(); } private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { - batchWriteSet.setItemsSoFar(itemsSoFar.addAndGet(batchWriteSet.getWriteSet().size())); + batchWriteSet.setItemsSoFar(itemsSoFar.addAndGet(batchWriteSet.getDocumentWriteSet().size())); WriteBatch batch = batchWriteSet.getBatchOfWriteEvents(); for ( WriteBatchListener successListener : successListeners ) { try { @@ -424,7 +398,7 @@ private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { } } - private void sendThrowableToListeners(Throwable t, String message, BatchWriteSet batchWriteSet) { + private void sendThrowableToListeners(Throwable t, BatchWriteSet batchWriteSet) { batchWriteSet.setItemsSoFar(itemsSoFar.get()); WriteBatch batch = batchWriteSet.getBatchOfWriteEvents(); for ( WriteFailureListener failureListener : failureListeners ) { @@ -434,7 +408,7 @@ private void sendThrowableToListeners(Throwable t, String message, BatchWriteSet logger.error("Exception thrown by an onBatchFailure listener", t2); } } - if ( message != null ) logger.warn(message, t.toString()); + logger.warn("Error writing batch: {}", t.toString()); } @Override @@ -613,18 +587,18 @@ public synchronized WriteBatcher withForestConfig(ForestConfiguration forestConf for ( Runnable task : tasks ) { if ( task instanceof BatchWriter ) { BatchWriter writerTask = (BatchWriter) task; - if ( removedHostInfos.containsKey(writerTask.writeSet.getClient().getHost()) ) { + if ( removedHostInfos.containsKey(writerTask.batchWriteSet().getClient().getHost()) ) { // this batch was targeting a host that's no longer on the list // if we re-add these docs they'll now be in batches that target acceptable hosts - BatchWriteSet writeSet = newBatchWriteSet(writerTask.writeSet.getBatchNumber()); + BatchWriteSet writeSet = newBatchWriteSet(writerTask.batchWriteSet().getBatchNumber()); writeSet.onFailure(throwable -> { if ( throwable instanceof RuntimeException ) throw (RuntimeException) throwable; else throw new DataMovementException("Failed to retry batch after failover", throwable); }); - for ( WriteEvent doc : writerTask.writeSet.getBatchOfWriteEvents().getItems() ) { - writeSet.getWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); + for ( WriteEvent doc : writerTask.batchWriteSet().getBatchOfWriteEvents().getItems() ) { + writeSet.getDocumentWriteSet().add(doc.getTargetUri(), doc.getMetadata(), doc.getContent()); } - BatchWriter retryWriterTask = new BatchWriter(writeSet); + BatchWriter retryWriterTask = new BatchWriter(writeSet, documentWriteSetFilter); Runnable fretryWriterTask = (Runnable) threadPool.submit(retryWriterTask); threadPool.replaceTask(writerTask, fretryWriterTask); // jump to the next task @@ -649,71 +623,6 @@ public static class HostInfo { public DatabaseClient client; } - public static class BatchWriter implements Runnable { - private BatchWriteSet writeSet; - - public BatchWriter(BatchWriteSet writeSet) { - if ( writeSet.getWriteSet().size() == 0 ) { - throw new IllegalStateException("Attempt to write an empty batch"); - } - this.writeSet = writeSet; - } - - @Override - public void run() { - try { - Runnable onBeforeWrite = writeSet.getOnBeforeWrite(); - if ( onBeforeWrite != null ) { - onBeforeWrite.run(); - } - logger.trace("begin write batch {} to forest on host \"{}\"", writeSet.getBatchNumber(), writeSet.getClient().getHost()); - if ( writeSet.getTemporalCollection() == null ) { - writeSet.getClient().newDocumentManager().write( - writeSet.getWriteSet(), writeSet.getTransform(), null - ); - } else { - // to get access to the TemporalDocumentManager write overload we need to instantiate - // a JSONDocumentManager or XMLDocumentManager, but we don't want to make assumptions about content - // format, so we'll set the default content format to unknown - XMLDocumentManager docMgr = writeSet.getClient().newXMLDocumentManager(); - docMgr.setContentFormat(Format.UNKNOWN); - docMgr.write( - writeSet.getWriteSet(), writeSet.getTransform(), null, writeSet.getTemporalCollection() - ); - } - closeAllHandles(); - Runnable onSuccess = writeSet.getOnSuccess(); - if ( onSuccess != null ) { - onSuccess.run(); - } - } catch (Throwable t) { - logger.trace("failed batch sent to forest on host \"{}\"", writeSet.getClient().getHost()); - Consumer onFailure = writeSet.getOnFailure(); - if ( onFailure != null ) { - onFailure.accept(t); - } - } - } - - private void closeAllHandles() throws Throwable { - Throwable lastThrowable = null; - for ( DocumentWriteOperation doc : writeSet.getWriteSet() ) { - try { - if ( doc.getContent() instanceof Closeable ) { - ((Closeable) doc.getContent()).close(); - } - if ( doc.getMetadata() instanceof Closeable ) { - ((Closeable) doc.getMetadata()).close(); - } - } catch (Throwable t) { - logger.error("error calling close()", t); - lastThrowable = t; - } - } - if ( lastThrowable != null ) throw lastThrowable; - } - } - /** * The following classes and CompletableThreadPoolExecutor * CompletableRejectedExecutionHandler exist exclusively to enable the @@ -938,4 +847,10 @@ public void addAll(Stream operations) { public DocumentMetadataHandle getDocumentMetadata() { return defaultMetadata; } + + @Override + public WriteBatcher withDocumentWriteSetFilter(DocumentWriteSetFilter filter) { + this.documentWriteSetFilter = filter; + return this; + } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java index 806815e05..ceffa379a 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/DatabaseClientPropertySource.java @@ -185,7 +185,7 @@ private DatabaseClientFactory.SecurityContext newSecurityContext(String type, Co return newBasicAuthContext(connectionString); case DatabaseClientBuilder.AUTH_TYPE_DIGEST: return newDigestAuthContext(connectionString); - case DatabaseClientBuilder.AUTH_TYPE_MARKLOGIC_CLOUD: + case DatabaseClientBuilder.AUTH_TYPE_CLOUD: return newCloudAuthContext(); case DatabaseClientBuilder.AUTH_TYPE_KERBEROS: return newKerberosAuthContext(); @@ -400,7 +400,7 @@ private String getSSLProtocol(String authType) { } // For convenience for Progress Data Cloud users, assume the JVM's default SSLContext should trust the certificate // used by Progress Data Cloud. A user can always override this default behavior by providing their own SSLContext. - if ((sslProtocol == null || sslProtocol.length() == 0) && DatabaseClientBuilder.AUTH_TYPE_MARKLOGIC_CLOUD.equalsIgnoreCase(authType)) { + if ((sslProtocol == null || sslProtocol.length() == 0) && DatabaseClientBuilder.AUTH_TYPE_CLOUD.equalsIgnoreCase(authType)) { sslProtocol = "default"; } return sslProtocol; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java index a885dc6e8..1baf50ba5 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/NodeConverter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -35,7 +35,6 @@ public class NodeConverter { static private ObjectMapper mapper; - static private DocumentBuilderFactory documentBuilderFactory; static private XMLInputFactory xmlInputFactory; private NodeConverter() { @@ -49,16 +48,7 @@ static private ObjectMapper getMapper() { } return mapper; } - static private DocumentBuilderFactory getDocumentBuilderFactory() { - // okay if one thread overwrites another during lazy initialization - if (documentBuilderFactory == null) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - documentBuilderFactory = factory; - } - return documentBuilderFactory; - } + static private XMLInputFactory getXMLInputFactory() { // okay if one thread overwrites another during lazy initialization if (xmlInputFactory == null) { @@ -265,7 +255,7 @@ static public Stream ReaderToJsonParser(Stream val static public Document InputStreamToDocument(InputStream inputStream) { try { - return (inputStream == null) ? null : getDocumentBuilderFactory().newDocumentBuilder().parse(inputStream); + return (inputStream == null) ? null : XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().parse(inputStream); } catch(SAXException | IOException | ParserConfigurationException e) { throw new RuntimeException(e); } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java index cd7bf9e6f..3d367aa2e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/OkHttpServices.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -106,10 +106,17 @@ public class OkHttpServices implements RESTServices { * in several places and is slightly different in each place. It's also not possible to implement this logic in an * OkHttp interceptor as the logic needs access to details that are not available to an interceptor. */ - private final Random randRetry = new Random(); - private int maxDelay = DEFAULT_MAX_DELAY; - private int minRetry = DEFAULT_MIN_RETRY; - private final Set retryStatus = new HashSet<>(); + private final Random randomForRetryDelay = new Random(); + + // The maximum amount of time to spend retrying requests. + private int maxDelayForRetries = DEFAULT_MAX_DELAY; + + // The minimum number of retry attempts to make regardless of the max delay. + private int minRetryAttempts = DEFAULT_MIN_RETRY; + + // The HTTP status codes that are retryable. + private static final Set RETRYABLE_STATUS_CODES = + Set.of(STATUS_BAD_GATEWAY, STATUS_SERVICE_UNAVAILABLE, STATUS_GATEWAY_TIMEOUT); private boolean checkFirstRequest = true; @@ -129,10 +136,6 @@ public record ConnectionConfig(String host, int port, String basePath, String da } public OkHttpServices(ConnectionConfig connectionConfig) { - retryStatus.add(STATUS_BAD_GATEWAY); - retryStatus.add(STATUS_SERVICE_UNAVAILABLE); - retryStatus.add(STATUS_GATEWAY_TIMEOUT); - this.okHttpClient = connect(connectionConfig); } @@ -239,13 +242,13 @@ private void configureDelayAndRetry(Properties props) { if (props.containsKey(MAX_DELAY_PROP)) { int max = Utilities.parseInt(props.getProperty(MAX_DELAY_PROP)); if (max > 0) { - maxDelay = max * 1000; + maxDelayForRetries = max * 1000; } } if (props.containsKey(MIN_RETRY_PROP)) { int min = Utilities.parseInt(props.getProperty(MIN_RETRY_PROP)); if (min > 0) { - minRetry = min; + minRetryAttempts = min; } } } @@ -289,7 +292,7 @@ private int makeFirstRequest(int retry) { private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { Response response = sendRequestOnce(setupRequest(requestUri, path, null).head()); int statusCode = response.code(); - if (!retryStatus.contains(statusCode)) { + if (!RETRYABLE_STATUS_CODES.contains(statusCode)) { closeResponse(response); return 0; } @@ -298,7 +301,7 @@ private int makeFirstRequest(HttpUrl requestUri, String path, int retry) { closeResponse(response); int retryAfter = Utilities.parseInt(retryAfterRaw); - return Math.max(retryAfter, calculateDelay(randRetry, retry)); + return Math.max(retryAfter, calculateDelay(retry)); } private RequestParameters addTemporalProtectionParams(RequestParameters params, String uri, ProtectionLevel level, @@ -518,7 +521,7 @@ private Response sendRequestWithRetry( /* * This loop is for retrying the request if the service is unavailable */ - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -537,7 +540,7 @@ private Response sendRequestWithRetry( ); } status = response.code(); - if (!isRetryable || !retryStatus.contains(status)) { + if (!isRetryable || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); /* * If we don't get a service unavailable status or if the request @@ -562,13 +565,13 @@ private Response sendRequestWithRetry( /* * Calculate the delay before which we shouldn't retry */ - nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(randRetry, retry)); + nextDelay = Math.max(getRetryAfterTime(response), calculateDelay(retry)); } /* * If the service is still unavailable after all the retries, we throw a * FailedRetryException indicating that the service is unavailable. */ - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -618,6 +621,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_NOT_FOUND) { @@ -895,6 +899,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doGetFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_NOT_FOUND) { throw new ResourceNotFoundException( @@ -1196,7 +1201,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -1236,7 +1241,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth status = response.code(); responseHeaders = response.headers(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -1253,9 +1258,9 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -1357,7 +1362,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -1380,7 +1385,7 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth status = response.code(); responseHeaders = response.headers(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -1395,9 +1400,9 @@ private TemporalDescriptor putPostDocumentImpl(RequestLogger reqlog, String meth } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -1501,6 +1506,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doPostFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to open transactions", extractErrorFields(response)); @@ -1510,12 +1516,14 @@ public Response apply(Request.Builder funcBuilder) { getReasonPhrase(response), extractErrorFields(response)); } - String location = response.headers().get("Location"); - List cookies = new ArrayList<>(); + final String location = response.headers().get("Location"); + + final List cookies = new ArrayList<>(); for (String setCookie : response.headers(HEADER_SET_COOKIE)) { ClientCookie cookie = parseClientCookie(requestBldr.build().url(), setCookie); cookies.add(cookie); } + closeResponse(response); if (location == null) throw new MarkLogicInternalException("transaction open failed to provide location"); if (!location.contains("/")) { @@ -2087,7 +2095,7 @@ Response getResponse() { long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -2116,7 +2124,7 @@ Response getResponse() { status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -2127,9 +2135,9 @@ Response getResponse() { closeResponse(response); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -2562,6 +2570,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to read " @@ -2625,7 +2634,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -2687,7 +2696,7 @@ private void putPostValueImpl(RequestLogger reqlog, String method, status = response.code(); - if (!retryStatus.contains(status)) { + if (!RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; } @@ -2702,9 +2711,9 @@ private void putPostValueImpl(RequestLogger reqlog, String method, } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -2749,6 +2758,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doDeleteFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to delete " @@ -3057,7 +3067,7 @@ public R putResour long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -3076,7 +3086,7 @@ public R putResour response = doPut(requestBldr, multiPart, hasStreamingPart); status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -3091,9 +3101,9 @@ public R putResour } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -3181,6 +3191,7 @@ public Response apply(Request.Builder funcBuilder) { }; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + Objects.requireNonNull(response); int status = response.code(); checkStatus(response, status, operation, "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); @@ -3230,7 +3241,7 @@ public R postResou long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -3249,7 +3260,7 @@ public R postResou response = doPost(requestBldr, multiPart, hasStreamingPart); status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -3264,9 +3275,9 @@ public R postResou } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -3712,6 +3723,7 @@ public RESTServiceResultIterator postMultipartForm( Consumer resendableConsumer = null; Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + Objects.requireNonNull(response); int status = response.code(); checkStatus(response, status, "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); return makeResults(OkHttpServiceResultIterator::new, reqlog, "apply", "resource", response); @@ -3782,6 +3794,7 @@ private U postIteratedResourceImpl( ); Response response = sendRequestWithRetry(requestBldr, (transaction == null), doPostFunction, resendableConsumer); + Objects.requireNonNull(response); checkStatus(response, response.code(), "apply", "resource", path, ResponseStatus.OK_OR_CREATED_OR_NO_CONTENT); boolean shouldStreamResults = "eval".equalsIgnoreCase(path) || "invoke".equalsIgnoreCase(path); @@ -3838,7 +3851,7 @@ private U postIt long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -3860,7 +3873,7 @@ private U postIt response = doPost(requestBldr, multiPart, hasStreamingPart); status = response.code(); - if (transaction != null || !retryStatus.contains(status)) { + if (transaction != null || !RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -3875,9 +3888,9 @@ private U postIt } int retryAfter = Utilities.parseInt(retryAfterRaw); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -4443,17 +4456,17 @@ private String stringJoin(Collection collection, String separator, return (builder != null) ? builder.toString() : null; } - private int calculateDelay(Random rand, int i) { + private int calculateDelay(int attempt) { int min = - (i > 6) ? DELAY_CEILING : - (i == 0) ? DELAY_FLOOR : - DELAY_FLOOR + (1 << i) * DELAY_MULTIPLIER; + (attempt > 6) ? DELAY_CEILING : + (attempt == 0) ? DELAY_FLOOR : + DELAY_FLOOR + (1 << attempt) * DELAY_MULTIPLIER; int range = - (i > 6) ? DELAY_FLOOR : - (i == 0) ? 2 * DELAY_MULTIPLIER : - (i == 6) ? DELAY_CEILING - min : - (1 << i) * DELAY_MULTIPLIER; - return min + randRetry.nextInt(range); + (attempt > 6) ? DELAY_FLOOR : + (attempt == 0) ? 2 * DELAY_MULTIPLIER : + (attempt == 6) ? DELAY_CEILING - min : + (1 << attempt) * DELAY_MULTIPLIER; + return min + randomForRetryDelay.nextInt(range); } static class OkHttpResult { @@ -4820,6 +4833,7 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); + Objects.requireNonNull(response); int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException( @@ -4956,7 +4970,7 @@ public InputStream match(QueryDefinition queryDef, long startTime = System.currentTimeMillis(); int nextDelay = 0; int retry = 0; - for (; retry < minRetry || (System.currentTimeMillis() - startTime) < maxDelay; retry++) { + for (; retry < minRetryAttempts || (System.currentTimeMillis() - startTime) < maxDelayForRetries; retry++) { if (nextDelay > 0) { try { Thread.sleep(nextDelay); @@ -4976,7 +4990,7 @@ public InputStream match(QueryDefinition queryDef, } status = response.code(); - if (!retryStatus.contains(status)) { + if (!RETRYABLE_STATUS_CODES.contains(status)) { if (isFirstRequest()) setFirstRequest(false); break; @@ -4987,9 +5001,9 @@ public InputStream match(QueryDefinition queryDef, closeResponse(response); - nextDelay = Math.max(retryAfter, calculateDelay(randRetry, retry)); + nextDelay = Math.max(retryAfter, calculateDelay(retry)); } - if (retryStatus.contains(status)) { + if (RETRYABLE_STATUS_CODES.contains(status)) { checkFirstRequest(); closeResponse(response); throw new FailedRetryException( @@ -5040,7 +5054,8 @@ public Response apply(Request.Builder funcBuilder) { } }; Response response = sendRequestWithRetry(requestBldr, doGetFunction, null); - int status = response.code(); + Objects.requireNonNull(response); + final int status = response.code(); if (status == STATUS_FORBIDDEN) { throw new ForbiddenUserException("User is not allowed to match", extractErrorFields(response)); @@ -5615,7 +5630,7 @@ private void executeRequest(CallResponseImpl responseImpl) { }; Response response = sendRequestWithRetry(requestBldr, sendRequestFunction, resendableConsumer); - + Objects.requireNonNull(response); if (session != null) { List cookies = new ArrayList<>(); for (String setCookie : response.headers(HEADER_SET_COOKIE)) { diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java index 6f1484159..5f931dfba 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/XmlFactories.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.impl; @@ -7,24 +7,33 @@ import org.slf4j.LoggerFactory; import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.FactoryConfigurationError; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; +import javax.xml.transform.Transformer; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; +import javax.xml.xpath.XPathFactory; import java.lang.ref.SoftReference; +import java.util.function.Supplier; public final class XmlFactories { private static final Logger logger = LoggerFactory.getLogger(XmlFactories.class); private static final CachedInstancePerThreadSupplier cachedOutputFactory = - new CachedInstancePerThreadSupplier(new Supplier() { - @Override - public XMLOutputFactory get() { - return makeNewOutputFactory(); - } - }); + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewOutputFactory); + + private static final CachedInstancePerThreadSupplier cachedDocumentBuilderFactory = + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewDocumentBuilderFactory); + + private static final CachedInstancePerThreadSupplier cachedXPathFactory = + new CachedInstancePerThreadSupplier<>(XPathFactory::newInstance); + + private static final CachedInstancePerThreadSupplier cachedTransformerFactory = + new CachedInstancePerThreadSupplier<>(XmlFactories::makeNewTransformerFactory); private XmlFactories() {} // preventing instances of utility class @@ -62,21 +71,78 @@ public static TransformerFactory makeNewTransformerFactory() { try { factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); } catch (TransformerConfigurationException e) { - logger.warn("Unable to set {} on TransformerFactory; cause: {}", XMLConstants.FEATURE_SECURE_PROCESSING, e.getMessage()); + logTransformerWarning(XMLConstants.FEATURE_SECURE_PROCESSING, e.getMessage()); } try { factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); } catch (IllegalArgumentException e) { - logger.warn("Unable to set {} on TransformerFactory; cause: {}", XMLConstants.ACCESS_EXTERNAL_DTD, e.getMessage()); + logTransformerWarning(XMLConstants.ACCESS_EXTERNAL_DTD, e.getMessage()); } try { factory.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); } catch (IllegalArgumentException e) { - logger.warn("Unable to set {} on TransformerFactory; cause: {}", XMLConstants.ACCESS_EXTERNAL_STYLESHEET, e.getMessage()); + logTransformerWarning(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, e.getMessage()); + } + return factory; + } + + private static void logTransformerWarning(String xmlConstant, String errorMessage) { + logger.warn("Unable to set {} on TransformerFactory; cause: {}", xmlConstant, errorMessage); + } + + private static DocumentBuilderFactory makeNewDocumentBuilderFactory() { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + // Default to best practices for conservative security including recommendations per + // https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.md + try { + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set FEATURE_SECURE_PROCESSING on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set disallow-doctype-decl on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set external-general-entities on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set external-parameter-entities on DocumentBuilderFactory; cause: {}", e.getMessage()); + } + try { + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + } catch (ParserConfigurationException e) { + logger.warn("Unable to set load-external-dtd on DocumentBuilderFactory; cause: {}", e.getMessage()); } + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + factory.setNamespaceAware(true); + factory.setValidating(false); + return factory; } + /** + * Returns a shared {@link DocumentBuilderFactory} configured with secure defaults. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a securely configured {@link DocumentBuilderFactory} + * + * @since 8.1.0 + * + * @see #makeNewDocumentBuilderFactory() if you really (really?) need a non-shared instance + */ + public static DocumentBuilderFactory getDocumentBuilderFactory() { + return cachedDocumentBuilderFactory.get(); + } + /** * Returns a shared {@link XMLOutputFactory}. This factory will have its * {@link XMLOutputFactory#IS_REPAIRING_NAMESPACES} property set to {@code true}. @@ -88,30 +154,52 @@ public static TransformerFactory makeNewTransformerFactory() { * * @throws FactoryConfigurationError see {@link XMLOutputFactory#newInstance()} * - * @see #makeNewOutputFactory() if you really (really?) need an non-shared instance + * @see #makeNewOutputFactory() if you really (really?) need a non-shared instance */ public static XMLOutputFactory getOutputFactory() { return cachedOutputFactory.get(); } - /** - * Represents a supplier of results. - * - *

There is no requirement that a new or distinct result be returned each - * time the supplier is invoked. - * - * @param the type of results supplied by this supplier - */ - // TODO replace with java.util.function.Supplier after Java 8 migration - interface Supplier { + /** + * Returns a shared {@link XPathFactory}. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a {@link XPathFactory} + * + * @since 8.1.0 + */ + public static XPathFactory getXPathFactory() { + return cachedXPathFactory.get(); + } - /** - * Gets a result. - * - * @return a result - */ - T get(); - } + /** + * Returns a shared {@link TransformerFactory} configured with secure defaults. + *

+ * Creating XML factories is potentially a pretty expensive operation. Using a shared instance helps to amortize + * this initialization cost via reuse. + * + * @return a securely configured {@link TransformerFactory} + * + * @since 8.1.0 + */ + public static TransformerFactory getTransformerFactory() { + return cachedTransformerFactory.get(); + } + + /** + * Creates a new {@link Transformer} from the shared {@link TransformerFactory}. + * + * @since 8.1.0 + */ + public static Transformer newTransformer() { + try { + return getTransformerFactory().newTransformer(); + } catch (TransformerConfigurationException e) { + throw new RuntimeException("Unable to create new Transformer from TransformerFactory", e); + } + } /** * A supplier that caches results per thread. @@ -129,7 +217,7 @@ interface Supplier { */ private static class CachedInstancePerThreadSupplier implements Supplier { - private final ThreadLocal> cachedInstances = new ThreadLocal>(); + private final ThreadLocal> cachedInstances = new ThreadLocal<>(); /** * The underlying supplier, invoked to originally retrieve the per-thread result @@ -167,7 +255,7 @@ public T get() { } // ... and retain it for later re-use - cachedInstances.set(new SoftReference(cachedInstance)); + cachedInstances.set(new SoftReference<>(cachedInstance)); } return cachedInstance; diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java index afd35b527..ca30e32cc 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurer.java @@ -3,188 +3,33 @@ */ package com.marklogic.client.impl.okhttp; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.marklogic.client.DatabaseClientFactory; -import com.marklogic.client.ProgressDataCloudException; -import okhttp3.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.progress.pdc.auth.okhttp.TokenGenerator; +import com.progress.pdc.auth.okhttp.TokenAuthenticationInterceptor; +import okhttp3.OkHttpClient; -import java.io.IOException; +import java.util.function.Supplier; -class ProgressDataCloudAuthenticationConfigurer implements AuthenticationConfigurer { - - private final String host; - - ProgressDataCloudAuthenticationConfigurer(String host) { - this.host = host; - } +record ProgressDataCloudAuthenticationConfigurer( + String host) implements AuthenticationConfigurer { @Override public void configureAuthentication(OkHttpClient.Builder clientBuilder, DatabaseClientFactory.ProgressDataCloudAuthContext securityContext) { final String apiKey = securityContext.getApiKey(); - if (apiKey == null || apiKey.trim().length() < 1) { + if (apiKey == null || apiKey.trim().isEmpty()) { throw new IllegalArgumentException("No API key provided"); } - TokenGenerator tokenGenerator = new DefaultTokenGenerator(this.host, securityContext); - clientBuilder.addInterceptor(new TokenAuthenticationInterceptor(tokenGenerator)); - } - /** - * Exists solely for mocking in unit tests. - */ - public interface TokenGenerator { - String generateToken(); - } - - /** - * Knows how to call the "/token" endpoint in Progress Data Cloud to generate a new token based on the - * user-provided API key. - */ - static class DefaultTokenGenerator implements TokenGenerator { - - private final static Logger logger = LoggerFactory.getLogger(DefaultTokenGenerator.class); - private final String host; - private final DatabaseClientFactory.ProgressDataCloudAuthContext securityContext; - - public DefaultTokenGenerator(String host, DatabaseClientFactory.ProgressDataCloudAuthContext securityContext) { - this.host = host; - this.securityContext = securityContext; - } - - public String generateToken() { - final Response tokenResponse = callTokenEndpoint(); - String token = getAccessTokenFromResponse(tokenResponse); - if (logger.isInfoEnabled()) { - logger.info("Successfully obtained authentication token"); - } - return token; - } - - private Response callTokenEndpoint() { - final HttpUrl tokenUrl = buildTokenUrl(); - OkHttpClient.Builder clientBuilder = OkHttpUtil.newClientBuilder(); + Supplier okHttpClientBuilderSupplier = () -> { + OkHttpClient.Builder tokenClientBuilder = OkHttpUtil.newClientBuilder(); // Current assumption is that the SSL config provided for connecting to MarkLogic should also be applicable // for connecting to Progress Data Cloud's "/token" endpoint. - OkHttpUtil.configureSocketFactory(clientBuilder, securityContext.getSSLContext(), securityContext.getTrustManager()); - OkHttpUtil.configureHostnameVerifier(clientBuilder, securityContext.getSSLHostnameVerifier()); - - if (logger.isInfoEnabled()) { - logger.info("Calling token endpoint at: " + tokenUrl); - } - - final Call call = clientBuilder.build().newCall( - new Request.Builder() - .url(tokenUrl) - .post(newFormBody()) - .build() - ); + OkHttpUtil.configureSocketFactory(tokenClientBuilder, securityContext.getSSLContext(), securityContext.getTrustManager()); + OkHttpUtil.configureHostnameVerifier(tokenClientBuilder, securityContext.getSSLHostnameVerifier()); + return tokenClientBuilder; + }; - try { - return call.execute(); - } catch (IOException e) { - throw new ProgressDataCloudException(String.format("Unable to call token endpoint at %s; cause: %s", - tokenUrl, e.getMessage(), e)); - } - } - - protected HttpUrl buildTokenUrl() { - // For the near future, it's guaranteed that https and 443 will be required for connecting to Progress Data Cloud, - // so providing the ability to customize this would be misleading. - HttpUrl.Builder builder = new HttpUrl.Builder() - .scheme("https") - .host(host) - .port(443) - .build() - .resolve(securityContext.getTokenEndpoint()).newBuilder(); - - Integer duration = securityContext.getTokenDuration(); - return duration != null ? - builder.addQueryParameter("duration", duration.toString()).build() : - builder.build(); - } - - protected FormBody newFormBody() { - return new FormBody.Builder() - .add("grant_type", securityContext.getGrantType()) - .add("key", securityContext.getApiKey()) - .build(); - } - - private String getAccessTokenFromResponse(Response response) { - String responseBody = null; - JsonNode payload; - try { - responseBody = response.body().string(); - payload = new ObjectMapper().readTree(responseBody); - } catch (IOException ex) { - throw new ProgressDataCloudException("Unable to get access token; response: " + responseBody, ex); - } - if (!payload.has("access_token")) { - throw new ProgressDataCloudException("Unable to get access token; unexpected JSON response: " + payload); - } - return payload.get("access_token").asText(); - } - } - - /** - * OkHttp interceptor that handles adding a token to an HTTP request and renewing it when necessary. - */ - static class TokenAuthenticationInterceptor implements Interceptor { - - private final static Logger logger = LoggerFactory.getLogger(TokenAuthenticationInterceptor.class); - - private final TokenGenerator tokenGenerator; - private String token; - - public TokenAuthenticationInterceptor(TokenGenerator tokenGenerator) { - this.tokenGenerator = tokenGenerator; - this.token = tokenGenerator.generateToken(); - } - - @Override - public Response intercept(Chain chain) throws IOException { - Request.Builder builder = chain.request().newBuilder(); - addTokenToRequest(builder); - Response response = chain.proceed(builder.build()); - if (response.code() == 401) { - logger.info("Received 401; will generate new token if necessary and retry request"); - response.close(); - final String currentToken = this.token; - generateNewTokenIfNecessary(currentToken); - - builder = chain.request().newBuilder(); - addTokenToRequest(builder); - response = chain.proceed(builder.build()); - } - return response; - } - - /** - * In the case of N threads using the same DatabaseClient - e.g. when using DMSDK - all of them - * may make a request at the same time and get a 401 back. Functionally, it should be fine if all - * make their own requests to renew the token, with the last thread being the one whose token - * value is retained on this class. But to simplify matters, this block is synchronized so only one - * thread can be in here. And if that thread finds that this.token is different from currentToken, - * then some other thread already renewed the token - so this thread doesn't need to do anything and - * can just try again. - * - * @param currentToken the value of this instance's token right before calling this method; in the event that - * another thread using this instance got here first, then this value will differ from the - * instance's token field - */ - private synchronized void generateNewTokenIfNecessary(String currentToken) { - if (currentToken.equals(this.token)) { - logger.info("Generating new token based on receiving 401"); - this.token = tokenGenerator.generateToken(); - } else if (logger.isDebugEnabled()) { - logger.debug("This instance's token has already been updated, presumably by another thread"); - } - } - - private synchronized Request.Builder addTokenToRequest(Request.Builder builder) { - return builder.header("Authorization", String.format("Bearer %s", this.token)); - } + Supplier tokenGenerator = new TokenGenerator(this.host, securityContext, okHttpClientBuilderSupplier); + clientBuilder.addInterceptor(new TokenAuthenticationInterceptor(tokenGenerator)); } } diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java index 656e399c5..b2f57e0c3 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/impl/okhttp/RetryIOExceptionInterceptor.java @@ -3,6 +3,7 @@ */ package com.marklogic.client.impl.okhttp; +import com.marklogic.client.MarkLogicIOException; import okhttp3.Interceptor; import okhttp3.Request; import okhttp3.Response; @@ -47,7 +48,7 @@ public Response intercept(Chain chain) throws IOException { for (int attempt = 0; attempt <= maxRetries; attempt++) { try { return chain.proceed(request); - } catch (IOException e) { + } catch (MarkLogicIOException | IOException e) { if (attempt == maxRetries || !isRetryableIOException(e)) { logger.warn("Not retryable: {}; {}", e.getClass(), e.getMessage()); throw e; @@ -65,7 +66,7 @@ public Response intercept(Chain chain) throws IOException { throw new IllegalStateException("Unexpected end of retry loop"); } - private boolean isRetryableIOException(IOException e) { + private boolean isRetryableIOException(Exception e) { return e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof UnknownHostException || diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java index f06846fd1..a19dd138e 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/DOMHandle.java @@ -1,25 +1,11 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.io; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; - -import javax.xml.XMLConstants; -import javax.xml.namespace.QName; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; - +import com.marklogic.client.MarkLogicIOException; +import com.marklogic.client.MarkLogicInternalException; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.marker.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,15 +13,17 @@ import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.w3c.dom.ls.DOMImplementationLS; -import org.w3c.dom.ls.LSException; -import org.w3c.dom.ls.LSInput; -import org.w3c.dom.ls.LSOutput; -import org.w3c.dom.ls.LSParser; -import org.w3c.dom.ls.LSResourceResolver; +import org.w3c.dom.ls.*; -import com.marklogic.client.MarkLogicIOException; -import com.marklogic.client.MarkLogicInternalException; +import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import java.io.*; +import java.nio.charset.StandardCharsets; /** * A DOM Handle represents XML content as a DOM document for reading or writing. @@ -199,7 +187,7 @@ public String toString() { */ public DocumentBuilderFactory getFactory() throws ParserConfigurationException { if (factory == null) - factory = makeDocumentBuilderFactory(); + factory = XmlFactories.getDocumentBuilderFactory(); return factory; } /** @@ -209,32 +197,6 @@ public DocumentBuilderFactory getFactory() throws ParserConfigurationException { public void setFactory(DocumentBuilderFactory factory) { this.factory = factory; } - protected DocumentBuilderFactory makeDocumentBuilderFactory() throws ParserConfigurationException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - // default to best practices for conservative security including recommendations per - // https://github.com/OWASP/CheatSheetSeries/blob/master/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.md - try { - factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://xml.org/sax/features/external-general-entities", false); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); - } catch (ParserConfigurationException e) {} - try { - factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); - } catch (ParserConfigurationException e) {} - factory.setXIncludeAware(false); - factory.setExpandEntityReferences(false); - factory.setNamespaceAware(true); - factory.setValidating(false); - - return factory; - } /** * Get the processor used to evaluate XPath expressions. @@ -246,7 +208,7 @@ protected DocumentBuilderFactory makeDocumentBuilderFactory() throws ParserConfi */ public XPath getXPathProcessor() { if (xpathProcessor == null) - xpathProcessor = makeXPathProcessorFactory().newXPath(); + xpathProcessor = XmlFactories.getXPathFactory().newXPath(); return xpathProcessor; } /** @@ -257,9 +219,6 @@ public XPath getXPathProcessor() { public void setXPathProcessor(XPath xpathProcessor) { this.xpathProcessor = xpathProcessor; } - protected XPathFactory makeXPathProcessorFactory() { - return XPathFactory.newInstance(); - } /** * Evaluate a string XPath expression against the retrieved document. diff --git a/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java b/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java index 643e76ca4..30582b82b 100644 --- a/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java +++ b/marklogic-client-api/src/main/java/com/marklogic/client/io/DocumentMetadataHandle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.io; @@ -577,10 +577,7 @@ protected void receiveContent(InputStream content) { Document document = null; if (content != null) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); document = builder.parse(new InputSource(new InputStreamReader(content, StandardCharsets.UTF_8))); content.close(); } diff --git a/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenAuthenticationInterceptor.java b/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenAuthenticationInterceptor.java new file mode 100644 index 000000000..b7bfbb34f --- /dev/null +++ b/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenAuthenticationInterceptor.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.progress.pdc.auth.okhttp; + +import okhttp3.Interceptor; +import okhttp3.Request; +import okhttp3.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.function.Supplier; + +/** + * An OkHttp interceptor that adds a bearer token to requests, and handles 401 responses by generating + * a new token and retrying the request. + * + * @since 8.1.0 + */ +public class TokenAuthenticationInterceptor implements Interceptor { + + private final static Logger logger = LoggerFactory.getLogger(TokenAuthenticationInterceptor.class); + + private final Supplier tokenGenerator; + private String token; + + public TokenAuthenticationInterceptor(Supplier tokenGenerator) { + this.tokenGenerator = tokenGenerator; + this.token = tokenGenerator.get(); + } + + @Override + public Response intercept(Chain chain) throws IOException { + Request.Builder builder = chain.request().newBuilder(); + builder = addTokenToRequest(builder); + Response response = chain.proceed(builder.build()); + if (response.code() == 401) { + logger.info("Received 401; will generate new token if necessary and retry request"); + response.close(); + final String currentToken = this.token; + generateNewTokenIfNecessary(currentToken); + + builder = chain.request().newBuilder(); + builder = addTokenToRequest(builder); + response = chain.proceed(builder.build()); + } + return response; + } + + /** + * In the case of N threads using the same DatabaseClient - e.g. when using DMSDK - all of them + * may make a request at the same time and get a 401 back. Functionally, it should be fine if all + * make their own requests to renew the token, with the last thread being the one whose token + * value is retained on this class. But to simplify matters, this block is synchronized so only one + * thread can be in here. And if that thread finds that this.token is different from currentToken, + * then some other thread already renewed the token - so this thread doesn't need to do anything and + * can just try again. + * + * @param currentToken the value of this instance's token right before calling this method; in the event that + * another thread using this instance got here first, then this value will differ from the + * instance's token field + */ + private synchronized void generateNewTokenIfNecessary(String currentToken) { + if (currentToken.equals(this.token)) { + logger.info("Generating new token based on receiving 401"); + this.token = tokenGenerator.get(); + } else if (logger.isDebugEnabled()) { + logger.debug("This instance's token has already been updated, presumably by another thread"); + } + } + + private synchronized Request.Builder addTokenToRequest(Request.Builder builder) { + return builder.header("Authorization", String.format("Bearer %s", this.token)); + } + +} diff --git a/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenGenerator.java b/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenGenerator.java new file mode 100644 index 000000000..3410080f4 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenGenerator.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.progress.pdc.auth.okhttp; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.ProgressDataCloudException; +import okhttp3.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.function.Supplier; + +/** + * Handles generating a token from the Progress Data Cloud token endpoint. + * + * @since 8.1.0 + */ +public class TokenGenerator implements Supplier { + + private final static Logger logger = LoggerFactory.getLogger(TokenGenerator.class); + + private final String host; + private final TokenInputs tokenInputs; + private final Supplier okHttpClientBuilderSupplier; + + public TokenGenerator(String host, TokenInputs tokenInputs, Supplier okHttpClientBuilderSupplier) { + this.host = host; + this.tokenInputs = tokenInputs; + this.okHttpClientBuilderSupplier = okHttpClientBuilderSupplier; + } + + public String get() { + final Response tokenResponse = callTokenEndpoint(); + String token = getAccessTokenFromResponse(tokenResponse); + if (logger.isDebugEnabled()) { + logger.debug("Successfully obtained authentication token"); + } + return token; + } + + private Response callTokenEndpoint() { + final HttpUrl tokenUrl = buildTokenUrl(); + OkHttpClient.Builder clientBuilder = okHttpClientBuilderSupplier.get(); + + if (logger.isDebugEnabled()) { + logger.debug("Calling token endpoint at: {}", tokenUrl); + } + + final Call call = clientBuilder.build().newCall( + new Request.Builder() + .url(tokenUrl) + .post(newFormBody()) + .build() + ); + + try { + return call.execute(); + } catch (IOException e) { + throw new ProgressDataCloudException(String.format("Unable to call token endpoint at %s; cause: %s", + tokenUrl, e.getMessage()), e); + } + } + + protected final HttpUrl buildTokenUrl() { + // For the near future, it's guaranteed that https and 443 will be required for connecting to Progress Data Cloud, + // so providing the ability to customize this would be misleading. + HttpUrl.Builder builder = new HttpUrl.Builder() + .scheme("https") + .host(host) + .port(443) + .build() + .resolve(tokenInputs.getTokenEndpoint()) + .newBuilder(); + + Integer duration = tokenInputs.getTokenDuration(); + return duration != null ? + builder.addQueryParameter("duration", duration.toString()).build() : + builder.build(); + } + + protected final FormBody newFormBody() { + return new FormBody.Builder() + .add("grant_type", tokenInputs.getGrantType()) + .add("key", tokenInputs.getApiKey()) + .build(); + } + + private String getAccessTokenFromResponse(Response response) { + String responseBody = null; + JsonNode payload; + try { + responseBody = response.body().string(); + payload = new ObjectMapper().readTree(responseBody); + } catch (IOException ex) { + throw new ProgressDataCloudException("Unable to get access token; response: " + responseBody, ex); + } + if (!payload.has("access_token")) { + throw new ProgressDataCloudException("Unable to get access token; unexpected JSON response: " + payload); + } + return payload.get("access_token").asText(); + } +} diff --git a/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenInputs.java b/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenInputs.java new file mode 100644 index 000000000..bb4c7f309 --- /dev/null +++ b/marklogic-client-api/src/main/java/com/progress/pdc/auth/okhttp/TokenInputs.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.progress.pdc.auth.okhttp; + +/** + * Defines the inputs for obtaining a token from PDC based on an API key. + * + * @since 8.1.0 + */ +public interface TokenInputs { + + default String getTokenEndpoint() { + return "/token"; + } + + default String getGrantType() { + return "api_key"; + } + + String getApiKey(); + + default Integer getTokenDuration() { + return null; + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java new file mode 100644 index 000000000..b7d7b1650 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/FewerServerThreadsThanForestsTest.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import com.marklogic.mgmt.ManageClient; +import com.marklogic.mgmt.resource.appservers.ServerManager; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class FewerServerThreadsThanForestsTest extends AbstractClientTest { + + @Test + void test() { + DatabaseClient client = Common.newClient(); + final int forestCount = client.newDataMovementManager().readForestConfig().listForests().length; + if (forestCount < 2) { + logger.info("This test requires multiple forests so that the server thread count can be set to the " + + "number of forests minus one; skipping test"); + return; + } + + adjustServerThreads(forestCount - 1); + try { + DataMovementManager dmm = client.newDataMovementManager(); + AtomicInteger uriCount = new AtomicInteger(); + QueryBatcher queryBatcher = dmm.newQueryBatcher(client.newQueryManager().newStructuredQueryBuilder().collection("/optic/test")) + .withThreadCount(1) + .onUrisReady(batch -> uriCount.addAndGet(batch.getItems().length)); + dmm.startJob(queryBatcher); + queryBatcher.awaitCompletion(); + dmm.stopJob(queryBatcher); + + assertEquals(4, uriCount.get(), "Verifies that the 4 test documents were found, and more importantly, " + + "that the new default maxDocToUriBatchRatio of 1 was applied correctly when the number of " + + "server threads is less than the number of forests. This is for bug 1872 in GitHub. Prior to this " + + "fix, the maxDocToUriBatchRatio of -1 returned by the server caused an error when the " + + "LinkedBlockingQueue was constructed with a negative capacity."); + } finally { + // We can safely use this number because we know the test-app doesn't change this. + final int defaultServerThreadCount = 32; + adjustServerThreads(defaultServerThreadCount); + } + } + + private void adjustServerThreads(final int threads) { + logger.info("Adjusting server threads to {}", threads); + Common.newAdminManager().invokeActionRequiringRestart(() -> { + ManageClient manageClient = Common.newManageClient(); + ObjectNode payload = Common.newServerPayload().put("threads", threads); + new ServerManager(manageClient).save(payload.toString()); + return true; + }); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteNakedPropertiesTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/WriteNakedPropertiesTest.java similarity index 80% rename from marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteNakedPropertiesTest.java rename to marklogic-client-api/src/test/java/com/marklogic/client/datamovement/WriteNakedPropertiesTest.java index e97f87158..fb7b58d89 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/WriteNakedPropertiesTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/WriteNakedPropertiesTest.java @@ -1,14 +1,12 @@ /* * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -package com.marklogic.client.test.datamovement; +package com.marklogic.client.datamovement; import com.marklogic.client.DatabaseClient; -import com.marklogic.client.datamovement.DataMovementManager; -import com.marklogic.client.datamovement.WriteBatcher; import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import javax.xml.namespace.QName; @@ -16,12 +14,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -public class WriteNakedPropertiesTest { - - @BeforeEach - void setup() { - Common.newRestAdminClient().newXMLDocumentManager().delete("/naked.xml"); - } +class WriteNakedPropertiesTest extends AbstractClientTest { @Test void test() { diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java new file mode 100644 index 000000000..6c136f0e5 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/AbstractIncrementalWriteTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.BeforeEach; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +abstract class AbstractIncrementalWriteTest extends AbstractClientTest { + + static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + AtomicInteger writtenCount = new AtomicInteger(); + AtomicInteger skippedCount = new AtomicInteger(); + AtomicReference batchFailure = new AtomicReference<>(); + ObjectMapper objectMapper = new ObjectMapper(); + + List docs = new ArrayList<>(); + IncrementalWriteFilter filter; + + @BeforeEach + void setup() { + // Need a user with eval privileges so that the eval filter can be tested. + Common.client = Common.newEvalClient(); + + // Default filter implementation, should be suitable for most tests. + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + } + + final void writeDocs(List docs) { + new WriteBatcherTemplate(Common.client).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(filter) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)) + .onBatchFailure((batch, failure) -> batchFailure.set(failure)), + + writeBatcher -> docs.forEach(writeBatcher::add) + ); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java new file mode 100644 index 000000000..1597ef6df --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.JacksonHandle; +import com.marklogic.client.io.StringHandle; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class ApplyExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest { + + @Test + void jsonExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .jsonExclusions("/timestamp", "/metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents with three keys + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2025-01-01T10:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2025-01-01T10:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Document " + i); + doc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed + doc.putObject("metadata") + .put("lastModified", "2026-01-02T15:30:00Z") // Changed + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("id", i); + doc.put("name", "Modified Document " + i); // Changed + doc.put("timestamp", "2026-01-02T16:00:00Z"); + doc.putObject("metadata") + .put("lastModified", "2026-01-02T16:00:00Z") + .put("author", "Test User"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/json-doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } + + @Test + void xmlExclusions() { + filter = IncrementalWriteFilter.newBuilder() + .xmlExclusions("//timestamp", "//metadata/lastModified") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = """ + + %d + Document %d + 2025-01-01T10:00:00Z + + Test User + 2025-01-01T10:00:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = """ + + %d + Document %d + 2026-01-02T15:30:00Z + + Test User + 2026-01-02T15:30:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(5, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(5, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + String xml = """ + + %d + Modified Document %d + 2026-01-02T16:00:00Z + + Test User + 2026-01-02T16:00:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(5, skippedCount.get(), "Skip count should remain at 5"); + } + + /** + * Verifies that JSON Pointer exclusions are only applied to JSON documents and are ignored for XML documents. + * The XML document should use its full content for hashing since no XML exclusions are configured. + */ + @Test + void jsonExclusionsIgnoredForXmlDocuments() { + filter = IncrementalWriteFilter.newBuilder() + .jsonExclusions("/timestamp") + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write one JSON doc and one XML doc + docs = new ArrayList<>(); + ObjectNode jsonDoc = objectMapper.createObjectNode(); + jsonDoc.put("id", 1); + jsonDoc.put("timestamp", "2025-01-01T10:00:00Z"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc))); + + String xmlDoc = """ + + 1 + 2025-01-01T10:00:00Z + + """; + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML))); + + writeDocs(docs); + assertEquals(2, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different timestamp values + docs = new ArrayList<>(); + jsonDoc = objectMapper.createObjectNode(); + jsonDoc.put("id", 1); + jsonDoc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc))); + + xmlDoc = """ + + 1 + 2026-01-02T15:30:00Z + + """; + docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML))); + + writeDocs(docs); + assertEquals(3, writtenCount.get(), "XML doc should be written since its timestamp changed and no XML exclusions are configured"); + assertEquals(1, skippedCount.get(), "JSON doc should be skipped since only the excluded timestamp field changed"); + } + + /** + * Verifies that when canonicalizeJson is false, documents with logically identical content + * but different key ordering will produce different hashes, causing a write to occur. + */ + @Test + void jsonNotCanonicalizedCausesDifferentHashForReorderedKeys() { + filter = IncrementalWriteFilter.newBuilder() + .canonicalizeJson(false) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial document with keys in a specific order + docs = new ArrayList<>(); + String json1 = "{\"name\":\"Test\",\"id\":1,\"value\":100}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/non-canonical.json", METADATA, + new StringHandle(json1).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with same logical content but different key order + docs = new ArrayList<>(); + String json2 = "{\"id\":1,\"value\":100,\"name\":\"Test\"}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/non-canonical.json", METADATA, + new StringHandle(json2).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(2, writtenCount.get(), "Document should be written because key order differs and JSON is not canonicalized"); + assertEquals(0, skippedCount.get(), "No documents should be skipped"); + } + + /** + * Verifies that with the default canonicalizeJson(true), documents with logically identical content + * but different key ordering will produce the same hash, causing the document to be skipped. + */ + @Test + void jsonCanonicalizedProducesSameHashForReorderedKeys() { + filter = IncrementalWriteFilter.newBuilder() + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial document with keys in a specific order + docs = new ArrayList<>(); + String json1 = "{\"name\":\"Test\",\"id\":1,\"value\":100}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/canonical.json", METADATA, + new StringHandle(json1).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with same logical content but different key order + docs = new ArrayList<>(); + String json2 = "{\"id\":1,\"value\":100,\"name\":\"Test\"}"; + docs.add(new DocumentWriteOperationImpl("/incremental/test/canonical.json", METADATA, + new StringHandle(json2).withFormat(Format.JSON))); + + writeDocs(docs); + assertEquals(1, writtenCount.get(), "Document should be skipped because canonicalized JSON produces the same hash"); + assertEquals(1, skippedCount.get(), "One document should be skipped"); + } + + @Test + void xmlExclusionsWithNamespaces() { + filter = IncrementalWriteFilter.newBuilder() + .xmlExclusions("//ns:timestamp", "//ns:metadata/ns:lastModified") + .xmlNamespaces(Map.of("ns", "http://example.com/ns")) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + // Write initial documents with namespaced elements + docs = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + String xml = """ + + %d + Document %d + 2025-01-01T10:00:00Z + + Test User + 2025-01-01T10:00:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/ns-xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(3, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write again with different values for excluded fields - should be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + String xml = """ + + %d + Document %d + 2026-01-02T15:30:00Z + + Test User + 2026-01-02T15:30:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/ns-xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(3, writtenCount.get(), "Documents should be skipped since only excluded fields changed"); + assertEquals(3, skippedCount.get()); + + // Write again with actual content change - should NOT be skipped + docs = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + String xml = """ + + %d + Modified Document %d + 2026-01-02T16:00:00Z + + Test User + 2026-01-02T16:00:00Z + + + """.formatted(i, i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/ns-xml-doc-" + i + ".xml", METADATA, new StringHandle(xml).withFormat(Format.XML))); + } + + writeDocs(docs); + assertEquals(6, writtenCount.get(), "Documents should be written since non-excluded content changed"); + assertEquals(3, skippedCount.get(), "Skip count should remain at 3"); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java new file mode 100644 index 000000000..c71eaa90b --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ApplyInvalidExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest { + + /** + * Verifies that an invalid JSON Pointer expression (missing leading slash) causes the build to fail + * immediately, allowing the user to fix the configuration before any documents are processed. + */ + @Test + void invalidJsonPointerExpression() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder() + .jsonExclusions("timestamp"); // Invalid - missing leading slash + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + + assertTrue(ex.getMessage().contains("Invalid JSON Pointer expression 'timestamp'"), + "Error message should include the invalid expression. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("incremental write"), + "Error message should mention incremental write context. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("must start with '/'"), + "Error message should hint at the fix. Actual: " + ex.getMessage()); + } + + /** + * Verifies that an empty JSON Pointer expression is rejected since it would exclude the entire document, + * leaving nothing to hash. + */ + @Test + void emptyJsonPointerExpression() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder() + .jsonExclusions(""); // Invalid - would exclude entire document + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + + assertTrue(ex.getMessage().contains("Empty JSON Pointer expression"), + "Error message should indicate empty expression. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("would exclude the entire document"), + "Error message should explain why it's invalid. Actual: " + ex.getMessage()); + } + + /** + * Verifies that an invalid XPath expression causes the build to fail immediately, + * allowing the user to fix the configuration before any documents are processed. + */ + @Test + void invalidXPathExpression() { + IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder() + .xmlExclusions("[[[invalid xpath"); // Invalid XPath syntax + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + + assertTrue(ex.getMessage().contains("Invalid XPath expression '[[[invalid xpath'"), + "Error message should include the invalid expression. Actual: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("incremental write"), + "Error message should mention incremental write context. Actual: " + ex.getMessage()); + } + +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java new file mode 100644 index 000000000..8c0cded96 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilterTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.document.DocumentWriteOperation; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import org.junit.jupiter.api.Test; + +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Unit tests that make no connection to MarkLogic. + */ +class IncrementalWriteFilterTest { + + /** + * Verifies that when a hash is added, a new metadata object is created so that a doc-specific hash field can be + * added without affecting any other document that might be sharing the same metadata object. + */ + @Test + void addHashToMetadata() { + DocumentMetadataHandle metadata = new DocumentMetadataHandle() + .withCollections("c1") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ) + .withQuality(2) + .withProperty("prop1", "value1") + .withMetadataValue("meta1", "value1"); + + DocumentWriteOperation doc1 = new DocumentWriteOperationImpl("/1.xml", metadata, new StringHandle("")); + DocumentWriteOperation doc2 = new DocumentWriteOperationImpl("/2.xml", metadata, new StringHandle("")); + + final String timestamp = Instant.now().toString(); + doc2 = IncrementalWriteFilter.addHashToMetadata(doc2, "theField", "abc123", "theTimestamp", timestamp); + + assertEquals(metadata, doc1.getMetadata(), "doc1 should still have the original metadata object"); + + DocumentMetadataHandle metadata2 = (DocumentMetadataHandle) doc2.getMetadata(); + assertEquals("c1", metadata2.getCollections().iterator().next(), "collection should be preserved"); + assertEquals(DocumentMetadataHandle.Capability.READ, metadata2.getPermissions().get("rest-reader").iterator().next(), "permission should be preserved"); + assertEquals(2, metadata2.getQuality(), "quality should be preserved"); + assertEquals("value1", metadata2.getProperties().get("prop1"), "property should be preserved"); + + assertEquals("value1", metadata2.getMetadataValues().get("meta1"), "metadata value should be preserved"); + assertEquals("abc123", metadata2.getMetadataValues().get("theField"), "hash field should be added"); + assertEquals(timestamp, metadata2.getMetadataValues().get("theTimestamp"), "timestamp should be added"); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java new file mode 100644 index 000000000..0ac60b97a --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/IncrementalWriteTest.java @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.marklogic.client.document.*; +import com.marklogic.client.impl.DocumentWriteOperationImpl; +import com.marklogic.client.io.*; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class IncrementalWriteTest extends AbstractIncrementalWriteTest { + + @Test + void opticFilter() { + verifyIncrementalWriteWorks(); + } + + @Test + void evalFilter() { + filter = IncrementalWriteFilter.newBuilder() + .useEvalQuery(true) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + verifyIncrementalWriteWorks(); + } + + @Test + void jsonKeysOutOfOrder() { + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("number", i); + doc.put("text", "hello"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("text", "hello"); + doc.put("number", i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get()); + assertEquals(10, skippedCount.get(), "Since JSON canonicalization is enabled by default, the documents " + + "should be recognized as unchanged even though their keys are in a different order."); + } + + @Test + void jsonKeysOutOfOrderWithNoCanonicalization() { + filter = IncrementalWriteFilter.newBuilder() + .canonicalizeJson(false) + .onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length)) + .build(); + + List docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("number", i); + doc.put("text", "hello"); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + ObjectNode doc = objectMapper.createObjectNode(); + doc.put("text", "hello"); + doc.put("number", i); + docs.add(new DocumentWriteOperationImpl("/incremental/test/doc-" + i + ".json", METADATA, new JacksonHandle(doc))); + } + + writeDocs(docs); + assertEquals(20, writtenCount.get(), "Since JSON canonicalization is disabled, all documents should be " + + "written again since their keys are in a different order."); + assertEquals(0, skippedCount.get()); + } + + @Test + void invalidJsonWithNoFormat() { + docs.add(new DocumentWriteOperationImpl("/not-json.txt", METADATA, new StringHandle("{\"not actually json"))); + writeDocs(docs); + + assertEquals(1, writtenCount.get(), "When the format is not specified and the content looks like JSON " + + "because it starts with a '{', the JSON canonicalization should fail and log a warning. The " + + "document should still be written with a hash generated based on the text in the document."); + + assertNull(batchFailure.get(), "No failure should have been thrown since the format on the content is " + + "not JSON, and thus the content should be hashed as text."); + } + + @Test + void invalidJsonWithFormat() { + docs.add(new DocumentWriteOperationImpl("/not.json", METADATA, new StringHandle("not actually json").withFormat(Format.JSON))); + writeDocs(docs); + + assertNotNull(batchFailure.get(), "A failure should have been thrown by the server since the content is not " + + "JSON. But the failure to canonicalize should still be logged, as the user will be far more interested " + + "in the error from the server."); + + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("failed to apply resource at documents"), + "Expecting the server to throw an error. Actual message: " + message); + } + + @Test + void noRangeIndexForField() { + filter = IncrementalWriteFilter.newBuilder() + .hashKeyName("non-existent-field") + .build(); + + writeTenDocuments(); + + assertNotNull(batchFailure.get()); + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("XDMP-FIELDRIDXNOTFOUND"), + "When the user tries to use the incremental write feature without the required range index, we should " + + "fail with a helpful error message. Actual message: " + message); + } + + @Test + void noRangeIndexForFieldWithEval() { + filter = IncrementalWriteFilter.newBuilder() + .hashKeyName("non-existent-field") + .useEvalQuery(true) + .build(); + + writeTenDocuments(); + + assertNotNull(batchFailure.get()); + String message = batchFailure.get().getMessage(); + assertTrue(message.contains("Unable to query for existing incremental write hashes") && message.contains("XDMP-FIELDRIDXNOTFOUND"), + "When the user tries to use the incremental write feature without the required range index, we should " + + "fail with a helpful error message. Actual message: " + message); + } + + @Test + void customTimestampKeyName() { + filter = IncrementalWriteFilter.newBuilder() + .hashKeyName("myWriteHash") + .timestampKeyName("myTimestamp") + .build(); + + writeTenDocuments(); + + DocumentMetadataHandle metadata = Common.client.newDocumentManager().readMetadata("/incremental/test/doc-1.xml", + new DocumentMetadataHandle()); + + assertNotNull(metadata.getMetadataValues().get("myWriteHash")); + assertNotNull(metadata.getMetadataValues().get("myTimestamp")); + assertFalse(metadata.getMetadataValues().containsKey("incrementalWriteHash")); + assertFalse(metadata.getMetadataValues().containsKey("incrementalWriteTimestamp")); + } + + /** + * The thought for this test is that if the user passes null in (which could happen via our Spark connector), + * they're breaking the feature. So don't let them do that - ignore null and use the default values. + */ + @Test + void nullIsIgnoredForKeyNames() { + filter = IncrementalWriteFilter.newBuilder() + .hashKeyName(null) + .timestampKeyName(null) + .build(); + + writeTenDocuments(); + + DocumentMetadataHandle metadata = Common.client.newDocumentManager().readMetadata("/incremental/test/doc-1.xml", + new DocumentMetadataHandle()); + + assertNotNull(metadata.getMetadataValues().get("incrementalWriteHash")); + assertNotNull(metadata.getMetadataValues().get("incrementalWriteTimestamp")); + } + + @Test + void textDocument() { + final DocumentWriteOperation writeOp = new DocumentWriteOperationImpl("/incremental/test/doc.txt", METADATA, + new StringHandle("Hello world")); + + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write the same text document again + docs = new ArrayList<>(); + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(1, skippedCount.get(), "This is a sanity check to verify that text files work as expected. " + + "Exclusions can't yet be specified for them since we only support JSON Pointer and XPath so far. It may " + + "be worth supporting regex-based exclusions for text files in the future."); + } + + @Test + void binaryDocument() { + byte[] binaryContent = "Binary content example".getBytes(); + final DocumentWriteOperation writeOp = new DocumentWriteOperationImpl("/incremental/test/doc.bin", METADATA, + new BytesHandle(binaryContent).withFormat(Format.BINARY)); + + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(0, skippedCount.get()); + + // Write the same binary document again + docs = new ArrayList<>(); + docs.add(writeOp); + writeDocs(docs); + assertEquals(1, writtenCount.get()); + assertEquals(1, skippedCount.get(), "Another sanity check to make sure that binary documents work as " + + "expected. Exclusions cannot be specified for them."); + } + + private void verifyIncrementalWriteWorks() { + writeTenDocuments(); + verifyDocumentsHasHashInMetadataKey(); + assertEquals(10, writtenCount.get()); + assertEquals(0, skippedCount.get(), "No docs should have been skipped on the first write."); + + writeTenDocuments(); + assertEquals(10, skippedCount.get(), "All docs should have been skipped since their content hasn't changed."); + assertEquals(10, writtenCount.get(), "The count of written should still be 10 since all docs should have been skipped on the second write."); + + modifyFiveDocuments(); + assertEquals(10, skippedCount.get()); + assertEquals(15, writtenCount.get(), "5 documents should have been modified, with their hashes being updated."); + + writeTenDocuments(); + assertEquals(15, skippedCount.get(), "The 5 unmodified documents should have been skipped."); + assertEquals(20, writtenCount.get(), "The 5 modified documents should have been overwritten since their content changed."); + } + + private void writeTenDocuments() { + docs = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + // Consistent URIs are required for incremental writes to work. + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "This is document number " + i + ""; + docs.add(new DocumentWriteOperationImpl(uri, METADATA, new StringHandle(content))); + } + writeDocs(docs); + } + + private void verifyDocumentsHasHashInMetadataKey() { + GenericDocumentManager mgr = Common.client.newDocumentManager(); + mgr.setMetadataCategories(DocumentManager.Metadata.METADATAVALUES); + DocumentPage page = mgr.search(Common.client.newQueryManager().newStructuredQueryBuilder().collection("incremental-test"), 1); + while (page.hasNext()) { + DocumentRecord doc = page.next(); + DocumentMetadataHandle metadata = doc.getMetadata(new DocumentMetadataHandle()); + + String hash = metadata.getMetadataValues().get("incrementalWriteHash"); + try { + // Can use Java's support for parsing unsigned longs in base 16 to verify the hash is valid. + Long.parseUnsignedLong(hash, 16); + } catch (NumberFormatException e) { + fail("Document " + doc.getUri() + " has an invalid incrementalWriteHash value: " + hash); + } + + String timestamp = metadata.getMetadataValues().get("incrementalWriteTimestamp"); + assertNotNull(timestamp, "Document " + doc.getUri() + " should have an incrementalWriteTimestamp value."); + } + } + + private void modifyFiveDocuments() { + docs = new ArrayList<>(); + for (int i = 6; i <= 10; i++) { + String uri = "/incremental/test/doc-" + i + ".xml"; + String content = "This is modified content"; + docs.add(new DocumentWriteOperationImpl(uri, METADATA, new StringHandle(content))); + } + writeDocs(docs); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java new file mode 100644 index 000000000..40a78b817 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/RemoveAllDocumentsFilterTest.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.io.DocumentMetadataHandle; +import com.marklogic.client.io.StringHandle; +import com.marklogic.client.test.AbstractClientTest; +import com.marklogic.client.test.Common; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RemoveAllDocumentsFilterTest extends AbstractClientTest { + + private static final DocumentMetadataHandle METADATA = new DocumentMetadataHandle() + .withCollections("incremental-test") + .withPermission("rest-reader", DocumentMetadataHandle.Capability.READ, DocumentMetadataHandle.Capability.UPDATE); + + AtomicInteger writtenCount = new AtomicInteger(); + + @Test + void filterRemovesAllDocuments() { + new WriteBatcherTemplate(Common.newClient()).runWriteJob( + writeBatcher -> writeBatcher + .withDocumentWriteSetFilter(context -> context.getDatabaseClient().newDocumentManager().newWriteSet()) + .onBatchSuccess(batch -> writtenCount.addAndGet(batch.getItems().length)), + + writeBatcher -> { + for (int i = 1; i <= 10; i++) { + writeBatcher.add("/incremental/test/doc-" + i + ".xml", METADATA, new StringHandle("")); + } + } + ); + + assertEquals(0, writtenCount.get(), "No documents should have been written since the filter removed them all. " + + "This test is verifying that no error will occur either when the filter doesn't return any documents."); + assertCollectionSize("incremental-test", 0); + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java new file mode 100644 index 000000000..62e066949 --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/WriteBatcherTemplate.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.datamovement.filter; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.datamovement.DataMovementManager; +import com.marklogic.client.datamovement.WriteBatcher; + +import java.util.function.Consumer; + +// Experimenting with a template that gets rid of some annoying DMSDK boilerplate. +record WriteBatcherTemplate(DatabaseClient databaseClient) { + + public void runWriteJob(Consumer writeBatcherConfigurer, Consumer writeBatcherUser) { + try (DataMovementManager dmm = databaseClient.newDataMovementManager()) { + WriteBatcher writeBatcher = dmm.newWriteBatcher(); + writeBatcherConfigurer.accept(writeBatcher); + + dmm.startJob(writeBatcher); + writeBatcherUser.accept(writeBatcher); + writeBatcher.flushAndWait(); + writeBatcher.awaitCompletion(); + dmm.stopJob(writeBatcher); + } + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java new file mode 100644 index 000000000..2b454103c --- /dev/null +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/AbstractClientTest.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + */ +package com.marklogic.client.test; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.junit5.AbstractMarkLogicTest; +import org.junit.jupiter.api.AfterEach; + +/** + * Intended to be the base class for all future client API tests, as it properly prepares the database by deleting + * documents from previous test runs that were not created as part of deploying the test app. + */ +public abstract class AbstractClientTest extends AbstractMarkLogicTest { + + @Override + protected final DatabaseClient getDatabaseClient() { + return Common.newServerAdminClient(); + } + + @Override + protected final String getJavascriptForDeletingDocumentsBeforeTestRuns() { + // The "/acme/" directory was previously deleted by AbstractOpticUpdateTest. It still needs to be deleted + // since some tests end up copying URIs to that directory but retain the 'test-data' collection. + return """ + declareUpdate(); + cts.uris('', [], cts.orQuery([ + cts.notQuery(cts.collectionQuery(['test-data', 'temporal-collection'])), + cts.directoryQuery('/acme/', 'infinity') + ])) + .toArray().forEach(item => xdmp.documentDelete(item)) + """; + } + + @AfterEach + void releaseClient() { + if (Common.client != null) { + Common.client.release(); + Common.client = null; + } + } +} diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java index 9d447d098..97bb22d9d 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/BitemporalTest.java @@ -15,8 +15,6 @@ import com.marklogic.client.io.StringHandle; import com.marklogic.client.query.*; import com.marklogic.client.query.StructuredQueryBuilder.TemporalOperator; -import com.marklogic.mgmt.ManageClient; -import com.marklogic.mgmt.resource.temporal.TemporalCollectionLSQTManager; import jakarta.xml.bind.DatatypeConverter; import org.custommonkey.xmlunit.exceptions.XpathException; import org.junit.jupiter.api.AfterEach; @@ -163,21 +161,6 @@ void writeTwoVersionsOfFourDocuments() throws XpathException { @Test void lsqtTest() { - // Due to bug MLE-24511 where LSQT properties aren't updated correctly in ml-gradle 6.0.0, we need to manually - // deploy them for this test. - ManageClient manageClient = Common.newManageClient(); - TemporalCollectionLSQTManager mgr = new TemporalCollectionLSQTManager(manageClient, "java-unittest", "temporal-collection"); - String payload = """ - { - "lsqt-enabled": true, - "automation": { - "enabled": true, - "period": 5000 - } - } - """; - mgr.save(payload); - String version1 = """ %s version1 diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java index 73f9d6c19..5c2e7e756 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/BufferableHandleTest.java @@ -1,11 +1,14 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; import com.marklogic.client.test.util.Referred; import com.marklogic.client.test.util.Refers; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; import org.custommonkey.xmlunit.SimpleNamespaceContext; import org.custommonkey.xmlunit.XMLUnit; import org.custommonkey.xmlunit.XpathEngine; @@ -17,16 +20,14 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.util.HashMap; import java.util.Map; import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class BufferableHandleTest { static private XpathEngine xpather; @@ -61,11 +62,7 @@ public void testReadWrite() throws JAXBException, ParserConfigurationException, SAXException, IOException, XpathException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - - Document domDocument = factory.newDocumentBuilder().newDocument(); + Document domDocument = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); root.setAttribute("foo", "bar"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java index 2437bf255..26c01ec61 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/Common.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -9,10 +9,13 @@ import com.marklogic.client.DatabaseClientBuilder; import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.extra.okhttpclient.OkHttpClientConfigurator; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.impl.okhttp.RetryIOExceptionInterceptor; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.ManageConfig; +import com.marklogic.mgmt.admin.AdminConfig; +import com.marklogic.mgmt.admin.AdminManager; import org.springframework.util.FileCopyUtils; import org.w3c.dom.DOMException; import org.w3c.dom.Document; @@ -226,10 +229,7 @@ public static String testDocumentToString(Document document) { public static Document testStringToDocument(String document) { try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - return factory.newDocumentBuilder().parse( + return XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().parse( new InputSource(new StringReader(document))); } catch (SAXException e) { throw new RuntimeException(e); @@ -258,6 +258,10 @@ public static ManageClient newManageClient() { return new ManageClient(new ManageConfig(HOST, 8002, SERVER_ADMIN_USER, SERVER_ADMIN_PASS)); } + public static AdminManager newAdminManager() { + return new AdminManager(new AdminConfig(HOST, 8001, SERVER_ADMIN_USER, SERVER_ADMIN_PASS)); + } + public static ObjectNode newServerPayload() { ObjectNode payload = new ObjectMapper().createObjectNode(); payload.put("server-name", SERVER_NAME); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java index 901655e81..3012f3800 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/DeleteSearchTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -7,6 +7,7 @@ import com.marklogic.client.document.DocumentDescriptor; import com.marklogic.client.document.GenericDocumentManager; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.query.DeleteQueryDefinition; import com.marklogic.client.query.QueryManager; @@ -15,8 +16,6 @@ import org.w3c.dom.Element; import org.w3c.dom.ls.DOMImplementationLS; -import javax.xml.parsers.DocumentBuilderFactory; - import static org.junit.jupiter.api.Assertions.*; @TestMethodOrder(MethodOrderer.MethodName.class) @@ -32,11 +31,7 @@ public static void beforeClass() throws Exception { } public static void writeDoc() throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - - Document domDocument = factory.newDocumentBuilder().newDocument(); + Document domDocument = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); root.setAttribute("foo", "bar"); @@ -45,7 +40,7 @@ public static void writeDoc() throws Exception { domDocument.appendChild(root); @SuppressWarnings("unused") - String domString = ((DOMImplementationLS) factory.newDocumentBuilder() + String domString = ((DOMImplementationLS) XmlFactories.getDocumentBuilderFactory().newDocumentBuilder() .getDOMImplementation()).createLSSerializer().writeToString(domDocument); XMLDocumentManager docMgr = client.newXMLDocumentManager(); @@ -61,7 +56,7 @@ public void test_A_Delete() { GenericDocumentManager docMgr = client.newDocumentManager(); DocumentDescriptor desc = docMgr.exists(docId); assertNotNull(desc); - assertEquals(desc.getUri(), docId); + assertEquals(docId, desc.getUri()); QueryManager queryMgr = client.newQueryManager(); DeleteQueryDefinition qdef = queryMgr.newDeleteDefinition(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java index fc9599425..17f76ee5c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/EvalTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -18,21 +18,20 @@ import com.marklogic.client.eval.EvalResultIterator; import com.marklogic.client.eval.ServerEvaluationCall; import com.marklogic.client.impl.HandleAccessor; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; import com.marklogic.client.query.DeleteQueryDefinition; import com.marklogic.client.query.QueryManager; -import com.marklogic.mgmt.ManageClient; import com.marklogic.mgmt.resource.appservers.ServerManager; +import jakarta.xml.bind.DatatypeConverter; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.w3c.dom.Document; import org.xml.sax.SAXException; -import jakarta.xml.bind.DatatypeConverter; import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.BufferedReader; import java.io.IOException; @@ -257,12 +256,9 @@ public void getNullTests() throws DatatypeConfigurationException, JsonProcessing } private void runAndTestXQuery(ServerEvaluationCall call) - throws JsonProcessingException, IOException, SAXException, ParserConfigurationException, DatatypeConfigurationException + throws IOException, SAXException, ParserConfigurationException, DatatypeConfigurationException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document document = factory.newDocumentBuilder() + Document document = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder() .parse(this.getClass().getClassLoader().getResourceAsStream("1-empty-1.0.xml")); call = call.addNamespace("myPrefix", "http://marklogic.com/test") .addVariable("myPrefix:myString", "Mars") diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java index a9d60fdeb..2adb961d1 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/GeospatialRegionQueriesTest.java @@ -1,11 +1,12 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.document.DocumentWriteSet; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.io.StringHandle; @@ -19,7 +20,6 @@ import org.w3c.dom.Element; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -54,10 +54,7 @@ private static void buildEnvironment() throws ParserConfigurationException { XMLDocumentManager docMgr = Common.client.newXMLDocumentManager(); DocumentWriteSet writeset =docMgr.newWriteSet(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElement("country"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java index 4ca6f1604..178ebc9f8 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/HandleAsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -31,7 +31,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; @@ -124,10 +123,7 @@ public void testBuiltinReadWrite() String xmlDocId = "/test/testAs1.xml"; XMLDocumentManager xmlMgr = Common.client.newXMLDocumentManager(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder xmlDocBldr = factory.newDocumentBuilder(); + DocumentBuilder xmlDocBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document beforeDocument = xmlDocBldr.newDocument(); Element root = beforeDocument.createElement("doc"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java index 151d760ca..1c45af995 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/PlanGeneratedBase.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.marklogic.client.MarkLogicIOException; import com.marklogic.client.expression.PlanBuilder; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.Format; import com.marklogic.client.row.RowManager; import com.marklogic.client.row.RowRecord; @@ -19,7 +20,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -131,11 +131,7 @@ private void checkXML(String testName, String kind, String expectedRaw, RowRecor // TODO: assertions on kind if set assertEquals(Format.XML, row.getContentFormat("t")); assertEquals("application/xml", row.getContentMimetype("t")); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setCoalescing(true); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document expected = builder.parse(new ByteArrayInputStream(expectedRaw.getBytes())); Document actual = row.getContentAs("t", Document.class); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java index 80c6a1852..603e8919a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/QueryOptionsManagerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -9,7 +9,9 @@ import com.marklogic.client.ResourceNotFoundException; import com.marklogic.client.ResourceNotResendableException; import com.marklogic.client.admin.QueryOptionsManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; +import jakarta.xml.bind.JAXBException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -17,14 +19,10 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.ls.DOMImplementationLS; -import org.xml.sax.SAXException; -import jakarta.xml.bind.JAXBException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.File; -import java.io.IOException; import static org.junit.jupiter.api.Assertions.*; @@ -60,14 +58,11 @@ public void testQueryOptionsManager() @Test public void testXMLDocsAsSearchOptions() - throws ParserConfigurationException, SAXException, IOException, ResourceNotFoundException, ForbiddenUserException, FailedRequestException, ResourceNotResendableException + throws ParserConfigurationException, ResourceNotFoundException, ForbiddenUserException, FailedRequestException, ResourceNotResendableException { String optionsName = "invalid"; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElementNS("http://marklogic.com/appservices/search","options"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java index 90cc36741..f8c8500c5 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/RawQueryDefinitionTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -11,6 +11,7 @@ import com.marklogic.client.admin.QueryOptionsManager; import com.marklogic.client.document.JSONDocumentManager; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.*; import com.marklogic.client.io.marker.StructureWriteHandle; import com.marklogic.client.query.*; @@ -25,7 +26,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -541,10 +541,7 @@ public void test_issue581_RawStructuredQueryFromFileHandle() throws Exception { } private static Document parseXml(String xml) throws Exception { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document document = builder.parse(new InputSource(new StringReader(xml))); return document; } diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java index 8277a25ce..fe9810e8e 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/RequestLoggerTest.java @@ -1,10 +1,11 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; import com.marklogic.client.document.XMLDocumentManager; import com.marklogic.client.impl.OutputStreamTee; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.io.StringHandle; @@ -20,11 +21,11 @@ import org.w3c.dom.ls.DOMImplementationLS; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.*; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class RequestLoggerTest { @BeforeAll @@ -81,13 +82,10 @@ public void testCopyTee() throws IOException { } @Test - public void testWriteReadLog() throws IOException, ParserConfigurationException { + public void testWriteReadLog() throws ParserConfigurationException { String docId = "/test/testWrite1.xml"; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java index 9e37efa19..7287c639c 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/SPARQLManagerTest.java @@ -30,7 +30,8 @@ import static org.junit.jupiter.api.Assertions.*; -public class SPARQLManagerTest { +class SPARQLManagerTest { + private static String graphUri = "http://marklogic.com/java/SPARQLManagerTest"; private static String triple1 = " ."; private static String triple2 = " ."; @@ -362,10 +363,6 @@ public void testSPARQLWithBindings() throws Exception { @Test public void testPagination() { - if (Common.getMarkLogicVersion().getMajor() >= 12) { - // Disabled until MLE-12708 is fixed. - return; - } SPARQLQueryDefinition qdef1 = smgr.newQueryDefinition( "SELECT ?s ?p ?o FROM <" + graphUri + "> { ?s ?p ?o }"); qdef1.setIncludeDefaultRulesets(false); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java index fb55b5274..89909c199 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/SearchFacetTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -8,6 +8,7 @@ import com.marklogic.client.ResourceNotFoundException; import com.marklogic.client.ResourceNotResendableException; import com.marklogic.client.admin.QueryOptionsManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.io.SearchHandle; import com.marklogic.client.query.FacetResult; @@ -21,7 +22,6 @@ import org.xml.sax.SAXException; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.io.StringReader; @@ -124,10 +124,7 @@ public void testFacetSearch() throws IOException, ParserConfigurationException, SAXException, FailedRequestException, ForbiddenUserException, ResourceNotFoundException, ResourceNotResendableException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); Document document = builder.parse(new InputSource(new StringReader(options))); mgr = Common.restAdminClient.newServerConfigManager().newQueryOptionsManager(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java index 6bfde88fe..44e1b16d0 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/XMLDocumentTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test; @@ -27,7 +27,6 @@ import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLStreamException; @@ -81,10 +80,7 @@ public void testReadWrite() { String docId = "/test/testWrite1.xml"; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document domDocument = factory.newDocumentBuilder().newDocument(); + Document domDocument = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element root = domDocument.createElement("root"); root.setAttribute("xml:lang", "en"); root.setAttribute("foo", "bar"); @@ -355,10 +351,7 @@ public void testPatch() throws Exception { DocumentPatchHandle patchHandle = patchBldr.build(); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - DocumentBuilder documentBldr = factory.newDocumentBuilder(); + DocumentBuilder documentBldr = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder(); for (int i=0; i < 2; i++) { Document domDocument = documentBldr.newDocument(); Element root = domDocument.createElement("root"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java index 48c6bb7c7..b332f3395 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/LegalHoldsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.datamovement; @@ -88,7 +88,7 @@ public void scenario10() throws Exception { // TODO This test failed when 2022 became 2023; increasing -7 to a higher number fixed it. The test could obviously // use some rework to ensure that it doesn't fail every time the year changes, but this comment is being left here // so that if/when this does fail in the future, it'll be easy to fix. - date.roll(Calendar.YEAR, -10); + date.roll(Calendar.YEAR, -11); StructuredQueryBuilder sqb = new StructuredQueryBuilder(); StructuredQueryDefinition query = sqb.and( diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java index c20a2181c..ab2001c9a 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/datamovement/ScenariosTest.java @@ -1,10 +1,11 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.datamovement; import com.marklogic.client.DatabaseClient; import com.marklogic.client.datamovement.*; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DOMHandle; import com.marklogic.client.test.Common; import org.junit.jupiter.api.AfterAll; @@ -14,7 +15,6 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; -import javax.xml.parsers.DocumentBuilderFactory; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -64,10 +64,8 @@ private class Message { public Map getBody() throws Exception { Map map = new HashMap<>(); map.put("uri", "http://marklogic.com/my/test/uri"); - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document document = factory.newDocumentBuilder().newDocument(); + + Document document = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element element = document.createElement("test"); document.appendChild(element); map.put("content", document); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java index ac427b445..a2d5087eb 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/io/DocumentMetadataHandleTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. + * Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ package com.marklogic.client.test.io; @@ -7,6 +7,7 @@ import com.marklogic.client.document.BinaryDocumentManager; import com.marklogic.client.document.DocumentManager.Metadata; import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.impl.XmlFactories; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.DocumentMetadataHandle.*; import com.marklogic.client.io.FileHandle; @@ -27,7 +28,6 @@ import org.xml.sax.SAXException; import javax.xml.namespace.QName; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import java.io.File; import java.io.FileInputStream; @@ -88,10 +88,7 @@ public void testReadWriteMetadata() throws SAXException, IOException, XpathExcep ""+ ""; - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setValidating(false); - Document document = factory.newDocumentBuilder().newDocument(); + Document document = XmlFactories.getDocumentBuilderFactory().newDocumentBuilder().newDocument(); Element third = document.createElement("third"); Element child = document.createElement("third.first"); child.setTextContent("value third one"); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java index 0afa19e8c..df9682e37 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/AbstractOpticUpdateTest.java @@ -18,6 +18,7 @@ import com.marklogic.client.row.RawPlanDefinition; import com.marklogic.client.row.RowManager; import com.marklogic.client.row.RowRecord; +import com.marklogic.client.test.AbstractClientTest; import com.marklogic.client.test.Common; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -32,7 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; -public abstract class AbstractOpticUpdateTest { +public abstract class AbstractOpticUpdateTest extends AbstractClientTest { private final static String XML_PREAMBLE = "\n"; @@ -42,10 +43,6 @@ public abstract class AbstractOpticUpdateTest { @BeforeEach public void setup() { - // Subclasses of this test are expected to only write URIs starting with /acme/ (which is used so that test - // URIs show up near the top when exploring the database in qconsole), so delete all of them before running the - // test to ensure a document doesn't already exist. - Common.deleteUrisWithPattern("/acme/*"); Common.client = Common.newClientBuilder().withUsername("writer-no-default-permissions").build(); rowManager = Common.client.newRowManager().withUpdate(true); op = rowManager.newPlanBuilder(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java index f7ed8233b..9e18896f5 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/rows/TransformDocTest.java @@ -13,9 +13,7 @@ import com.marklogic.client.io.marker.AbstractWriteHandle; import com.marklogic.client.row.RowRecord; import com.marklogic.client.test.Common; -import com.marklogic.client.test.MarkLogicVersion; import com.marklogic.client.test.junit5.RequiresML11; -import org.junit.Before; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java index 366bc6074..933ec2e77 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java +++ b/marklogic-client-api/src/test/java/com/marklogic/client/test/ssl/SSLTest.java @@ -81,7 +81,7 @@ public void testSSLAuth() throws NoSuchAlgorithmException, KeyManagementExceptio @Test // Not able to mock the X509Certificate class on Java 21. - @EnabledOnJre({JRE.JAVA_8, JRE.JAVA_11, JRE.JAVA_17}) + @EnabledOnJre({JRE.JAVA_17}) public void testHostnameVerifier() throws SSLException, CertificateParsingException { // three things our SSLHostnameVerifier will capture AtomicReference capturedHost = new AtomicReference<>(); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurerTest.java b/marklogic-client-api/src/test/java/com/progress/pdc/auth/okhttp/ProgressDataCloudAuthenticationConfigurerTest.java similarity index 67% rename from marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurerTest.java rename to marklogic-client-api/src/test/java/com/progress/pdc/auth/okhttp/ProgressDataCloudAuthenticationConfigurerTest.java index 91f67105d..8c1356e52 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/ProgressDataCloudAuthenticationConfigurerTest.java +++ b/marklogic-client-api/src/test/java/com/progress/pdc/auth/okhttp/ProgressDataCloudAuthenticationConfigurerTest.java @@ -1,7 +1,7 @@ /* * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -package com.marklogic.client.impl.okhttp; +package com.progress.pdc.auth.okhttp; import com.marklogic.client.DatabaseClientFactory; import okhttp3.FormBody; @@ -15,14 +15,18 @@ /** * Since we don't yet have a reliable way to test against a Progress Data Cloud instance, including some small unit tests * to ensure that certain things are built as expected. + *

+ * This is in the "pdc" package even though it depends on a MarkLogic Java Client-specific class - this is so that it + * can access protected methods of DefaultTokenGenerator. */ -public class ProgressDataCloudAuthenticationConfigurerTest { +class ProgressDataCloudAuthenticationConfigurerTest { @Test void buildTokenUrl() throws Exception { - ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator client = new ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator("somehost", + TokenGenerator client = new TokenGenerator("somehost", new DatabaseClientFactory.ProgressDataCloudAuthContext("doesnt-matter") - .withSSLContext(SSLContext.getDefault(), null) + .withSSLContext(SSLContext.getDefault(), null), + () -> null ); HttpUrl tokenUrl = client.buildTokenUrl(); @@ -35,9 +39,10 @@ void buildTokenUrl() throws Exception { */ @Test void buildTokenUrlWithCustomTokenPath() throws Exception { - ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator client = new ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator("otherhost", + TokenGenerator client = new TokenGenerator("otherhost", new DatabaseClientFactory.ProgressDataCloudAuthContext("doesnt-matter", "/customToken", "doesnt-matter") - .withSSLContext(SSLContext.getDefault(), null) + .withSSLContext(SSLContext.getDefault(), null), + () -> null ); HttpUrl tokenUrl = client.buildTokenUrl(); @@ -47,9 +52,10 @@ void buildTokenUrlWithCustomTokenPath() throws Exception { @Test void buildTokenUrlWithDuration() throws Exception { Integer duration = 10; - ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator client = new ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator("somehost", + TokenGenerator client = new TokenGenerator("somehost", new DatabaseClientFactory.ProgressDataCloudAuthContext("doesnt-matter", duration) - .withSSLContext(SSLContext.getDefault(), null) + .withSSLContext(SSLContext.getDefault(), null), + () -> null ); HttpUrl tokenUrl = client.buildTokenUrl(); @@ -59,9 +65,10 @@ void buildTokenUrlWithDuration() throws Exception { @Test void buildTokenUrlWithDurationAndCustomPath() throws Exception { Integer duration = 10; - ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator client = new ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator("somehost", + TokenGenerator client = new TokenGenerator("somehost", new DatabaseClientFactory.ProgressDataCloudAuthContext("doesnt-matter", "/customToken", "doesnt-matter", duration) - .withSSLContext(SSLContext.getDefault(), null) + .withSSLContext(SSLContext.getDefault(), null), + () -> null ); HttpUrl tokenUrl = client.buildTokenUrl(); @@ -70,9 +77,9 @@ void buildTokenUrlWithDurationAndCustomPath() throws Exception { @Test void newFormBody() { - FormBody body = new ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator("host-doesnt-matter", - new DatabaseClientFactory.ProgressDataCloudAuthContext("myKey")) - .newFormBody(); + FormBody body = new TokenGenerator("host-doesnt-matter", + new DatabaseClientFactory.ProgressDataCloudAuthContext("myKey"), () -> null) + .newFormBody(); assertEquals("grant_type", body.name(0)); assertEquals("apikey", body.value(0)); assertEquals("key", body.name(1)); @@ -85,9 +92,9 @@ void newFormBody() { */ @Test void newFormBodyWithOverrides() { - FormBody body = new ProgressDataCloudAuthenticationConfigurer.DefaultTokenGenerator("host-doesnt-matter", - new DatabaseClientFactory.ProgressDataCloudAuthContext("myKey", "doesnt-matter", "custom-grant-type")) - .newFormBody(); + FormBody body = new TokenGenerator("host-doesnt-matter", + new DatabaseClientFactory.ProgressDataCloudAuthContext("myKey", "doesnt-matter", "custom-grant-type"), () -> null) + .newFormBody(); assertEquals("grant_type", body.name(0)); assertEquals("custom-grant-type", body.value(0)); assertEquals("key", body.name(1)); diff --git a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java b/marklogic-client-api/src/test/java/com/progress/pdc/auth/okhttp/TokenAuthenticationInterceptorTest.java similarity index 93% rename from marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java rename to marklogic-client-api/src/test/java/com/progress/pdc/auth/okhttp/TokenAuthenticationInterceptorTest.java index b1529fb2c..625317074 100644 --- a/marklogic-client-api/src/test/java/com/marklogic/client/impl/okhttp/TokenAuthenticationInterceptorTest.java +++ b/marklogic-client-api/src/test/java/com/progress/pdc/auth/okhttp/TokenAuthenticationInterceptorTest.java @@ -1,7 +1,7 @@ /* * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ -package com.marklogic.client.impl.okhttp; +package com.progress.pdc.auth.okhttp; import com.marklogic.client.ext.helper.LoggingObject; import mockwebserver3.MockResponse; @@ -16,6 +16,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -37,8 +38,7 @@ void beforeEach() throws IOException { fakeTokenGenerator = new FakeTokenGenerator(); - ProgressDataCloudAuthenticationConfigurer.TokenAuthenticationInterceptor interceptor = - new ProgressDataCloudAuthenticationConfigurer.TokenAuthenticationInterceptor(fakeTokenGenerator); + TokenAuthenticationInterceptor interceptor = new TokenAuthenticationInterceptor(fakeTokenGenerator); assertEquals(1, fakeTokenGenerator.timesInvoked, "When the interceptor is created, it should immediately generate a token so that when multiple threads " + "are using the DatabaseClient, they will all use the same token."); @@ -148,11 +148,11 @@ private int callMockWebServer() { * Fake token generator that allows us to assert on how many times it's invoked, which ensures that new tokens are * or are not being generated when required. */ - private static class FakeTokenGenerator implements ProgressDataCloudAuthenticationConfigurer.TokenGenerator { + private static class FakeTokenGenerator implements Supplier { int timesInvoked; @Override - public String generateToken() { + public String get() { // A slight delay is added here for the multipleThread test case to simulate the token generation taking // some amount of time. This allows us to verify that the synchronization is working properly in the // interceptor. diff --git a/ml-development-tools/build.gradle b/ml-development-tools/build.gradle index f7bfb277f..37836848e 100644 --- a/ml-development-tools/build.gradle +++ b/ml-development-tools/build.gradle @@ -29,13 +29,9 @@ dependencies { // Sticking with this older version for now as the latest 1.x version introduces breaking changes. implementation 'com.networknt:json-schema-validator:1.0.88' - // Not yet migrating this project to JUnit 5. Will reconsider it once we have a reason to enhance - // this project. + // Sticking with JUnit 4 as there are no vulnerabilities with it, and shifting to JUnit 5 in this module will be + // a significant and tedious effort. testImplementation 'junit:junit:4.13.2' - testImplementation 'xmlunit:xmlunit:1.6' - testCompileOnly gradleTestKit() - - testImplementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } // Added to avoid problem where processResources fails because - somehow - the plugin properties file is getting diff --git a/test-app/build.gradle b/test-app/build.gradle index a06a500b5..f8d21f472 100644 --- a/test-app/build.gradle +++ b/test-app/build.gradle @@ -2,17 +2,34 @@ * Copyright (c) 2010-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved. */ +buildscript { + repositories { + mavenCentral() + + mavenLocal() + + // Needed for ml-gradle 6.2-SNAPSHOT + maven { + url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" + } + } + dependencies { + classpath "com.marklogic:ml-gradle:6.2-SNAPSHOT" + } +} + plugins { id "net.saliman.properties" version "1.5.2" - id 'com.marklogic.ml-gradle' version '6.0.1' id "com.github.psxpaul.execfork" version "0.2.2" } +apply plugin: "com.marklogic.ml-gradle" + dependencies { - implementation "io.undertow:undertow-core:2.3.19.Final" - implementation "io.undertow:undertow-servlet:2.3.19.Final" + implementation "io.undertow:undertow-core:2.3.20.Final" + implementation "io.undertow:undertow-servlet:2.3.20.Final" implementation 'org.slf4j:slf4j-api:2.0.17' - implementation 'ch.qos.logback:logback-classic:1.5.18' + implementation 'ch.qos.logback:logback-classic:1.5.23' implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" implementation "com.squareup.okhttp3:okhttp:${okhttpVersion}" } diff --git a/test-app/src/main/ml-config/databases/content-database.json b/test-app/src/main/ml-config/databases/content-database.json index ec7a36f01..8ca808124 100644 --- a/test-app/src/main/ml-config/databases/content-database.json +++ b/test-app/src/main/ml-config/databases/content-database.json @@ -1,303 +1,328 @@ { - "database-name": "java-unittest", - "schema-database": "java-unittest-schemas", - "triple-index": true, - "uri-lexicon": true, - "collection-lexicon": true, - "maintain-last-modified": true, - "range-element-index": [ - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "lastModified", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "http://marklogic.com/xdmp/property", - "localname": "last-modified", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "http://nwalsh.com/ns/photolib", - "localname": "tag", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "http://nwalsh.com/ns/photolib", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "double", - "namespace-uri": "", - "localname": "double", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "int", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "grandchild", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "string", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "system-start", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "system-end", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "valid-start", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "valid-end", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "range-element-attribute-index": [ - { - "scalar-type": "date", - "parent-namespace-uri": "http://nwalsh.com/ns/photolib", - "parent-localname": "view", - "namespace-uri": "", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "element-word-lexicon": [ - { - "namespace-uri": "", - "localname": "suggest", - "collation": "http://marklogic.com/collation/" - } - ], - "path-namespace": [ - { - "prefix": "rootOrg", - "namespace-uri": "root.org" - }, - { - "prefix": "targetOrg", - "namespace-uri": "target.org" - } - ], - "range-path-index": [ - { - "scalar-type": "long", - "path-expression": "com.marklogic.client.test.City/population", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.test.City/alternateNames", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.test.Country/continent", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.test.TimeTest/calendarTest", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.test.TimeTest/calendarTestCet", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.test.TimeTest/dateTest", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - } - ], - "field": [ - { - "field-name": "", - "include-root": true - }, - { - "field-name": "int1", - "include-root": false - }, - { - "field-name": "int2", - "include-root": false - } - ], - "range-field-index": [ - { - "scalar-type": "int", - "field-name": "int1", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "field-name": "int2", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-index": [ - { - "namespace-uri": "", - "localname": "latLong", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-child-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "com.marklogic.client.test.City", - "namespace-uri": "", - "localname": "latLong", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-pair-index": [ - { - "parent-namespace-uri": "http://marklogic.com/ns/test/places", - "parent-localname": "place", - "latitude-namespace-uri": "http://marklogic.com/ns/test/places", - "latitude-localname": "lat", - "longitude-namespace-uri": "http://marklogic.com/ns/test/places", - "longitude-localname": "long", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "parent-namespace-uri": "", - "parent-localname": "com.marklogic.client.test.City", - "latitude-namespace-uri": "", - "latitude-localname": "latitude", - "longitude-namespace-uri": "", - "longitude-localname": "longitude", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "parent-namespace-uri": "", - "parent-localname": "point", - "latitude-namespace-uri": "", - "latitude-localname": "lat", - "longitude-namespace-uri": "", - "longitude-localname": "lon", - "coordinate-system": "wgs84/double", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-path-index": [ - { - "path-expression": "com.marklogic.client.test.City/latLong", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "path-expression": "/rootOrg:geo/targetOrg:path", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-region-path-index": [ - { - "path-expression": "/country/region", - "coordinate-system": "wgs84", - "units": "miles", - "geohash-precision": 3, - "invalid-values": "ignore" - }, - { - "path-expression": "/country/region", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 3, - "invalid-values": "ignore" - } - ], - "default-ruleset": [ - { - "location": "rdfs.rules" - } - ] -} \ No newline at end of file + "database-name": "java-unittest", + "schema-database": "java-unittest-schemas", + "triple-index": true, + "uri-lexicon": true, + "collection-lexicon": true, + "maintain-last-modified": true, + "range-element-index": [ + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "lastModified", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "http://marklogic.com/xdmp/property", + "localname": "last-modified", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "http://nwalsh.com/ns/photolib", + "localname": "tag", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "http://nwalsh.com/ns/photolib", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "double", + "namespace-uri": "", + "localname": "double", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "int", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "grandchild", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "string", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "system-start", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "system-end", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "valid-start", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "valid-end", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "range-element-attribute-index": [ + { + "scalar-type": "date", + "parent-namespace-uri": "http://nwalsh.com/ns/photolib", + "parent-localname": "view", + "namespace-uri": "", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "element-word-lexicon": [ + { + "namespace-uri": "", + "localname": "suggest", + "collation": "http://marklogic.com/collation/" + } + ], + "path-namespace": [ + { + "prefix": "rootOrg", + "namespace-uri": "root.org" + }, + { + "prefix": "targetOrg", + "namespace-uri": "target.org" + } + ], + "range-path-index": [ + { + "scalar-type": "long", + "path-expression": "com.marklogic.client.test.City/population", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.test.City/alternateNames", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.test.Country/continent", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.test.TimeTest/calendarTest", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.test.TimeTest/calendarTestCet", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.test.TimeTest/dateTest", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + } + ], + "field": [ + { + "field-name": "", + "include-root": true + }, + { + "field-name": "int1", + "include-root": false + }, + { + "field-name": "int2", + "include-root": false + }, + { + "field-name": "incrementalWriteHash", + "metadata": "", + "stemmed-searches": "off", + "word-searches": false, + "fast-phrase-searches": false, + "fast-case-sensitive-searches": false, + "fast-diacritic-sensitive-searches": false + }, + { + "field-name": "myWriteHash", + "metadata": "", + "stemmed-searches": "off", + "word-searches": false, + "fast-phrase-searches": false, + "fast-case-sensitive-searches": false, + "fast-diacritic-sensitive-searches": false + } + ], + "range-field-index": [ + { + "scalar-type": "int", + "field-name": "int1", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "field-name": "int2", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "collation": "http://marklogic.com/collation/", + "field-name": "incrementalWriteHash", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-index": [ + { + "namespace-uri": "", + "localname": "latLong", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-child-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "com.marklogic.client.test.City", + "namespace-uri": "", + "localname": "latLong", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-pair-index": [ + { + "parent-namespace-uri": "http://marklogic.com/ns/test/places", + "parent-localname": "place", + "latitude-namespace-uri": "http://marklogic.com/ns/test/places", + "latitude-localname": "lat", + "longitude-namespace-uri": "http://marklogic.com/ns/test/places", + "longitude-localname": "long", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "parent-namespace-uri": "", + "parent-localname": "com.marklogic.client.test.City", + "latitude-namespace-uri": "", + "latitude-localname": "latitude", + "longitude-namespace-uri": "", + "longitude-localname": "longitude", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "parent-namespace-uri": "", + "parent-localname": "point", + "latitude-namespace-uri": "", + "latitude-localname": "lat", + "longitude-namespace-uri": "", + "longitude-localname": "lon", + "coordinate-system": "wgs84/double", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-path-index": [ + { + "path-expression": "com.marklogic.client.test.City/latLong", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "path-expression": "/rootOrg:geo/targetOrg:path", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-region-path-index": [ + { + "path-expression": "/country/region", + "coordinate-system": "wgs84", + "units": "miles", + "geohash-precision": 3, + "invalid-values": "ignore" + }, + { + "path-expression": "/country/region", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 3, + "invalid-values": "ignore" + } + ], + "default-ruleset": [ + { + "location": "rdfs.rules" + } + ] +} diff --git a/test-app/src/main/ml-config/databases/func-content-database.json b/test-app/src/main/ml-config/databases/func-content-database.json index 28b4ef38a..20417a0aa 100644 --- a/test-app/src/main/ml-config/databases/func-content-database.json +++ b/test-app/src/main/ml-config/databases/func-content-database.json @@ -1,213 +1,213 @@ { - "database-name": "java-functest", - "schema-database": "java-functest-schemas", - "triple-index": true, - "uri-lexicon": true, - "collection-lexicon": true, - "maintain-last-modified": false, - "trailing-wildcard-searches": true, - "stemmed-searches": "basic", - "range-element-index": [ - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "srchNumber", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "", - "localname": "srchDate", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "srchLevel", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "srchCity", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "http://action/", - "localname": "title", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "http://noun/", - "localname": "title", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "decimal", - "namespace-uri": "http://example.com", - "localname": "rating", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "http://example.com", - "localname": "scoville", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "http://purl.org/dc/elements/1.1/", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "city", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "decimal", - "namespace-uri": "http://test.aggr.com", - "localname": "score", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "http://test.tups.com", - "localname": "rate", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "title", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "popularity", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "double", - "namespace-uri": "", - "localname": "distance", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "namespace-uri": "", - "localname": "bday", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "height1", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "date", - "namespace-uri": "", - "localname": "height2", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "cityName", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "cityTeam", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "long", - "namespace-uri": "", - "localname": "cityPopulation", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "long", - "namespace-uri": "", - "localname": "inventory", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "int", - "namespace-uri": "", - "localname": "id", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "namespace-uri": "", - "localname": "animal", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "reject" - }, + "database-name": "java-functest", + "schema-database": "java-functest-schemas", + "triple-index": true, + "uri-lexicon": true, + "collection-lexicon": true, + "maintain-last-modified": false, + "trailing-wildcard-searches": true, + "stemmed-searches": "basic", + "range-element-index": [ + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "srchNumber", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "", + "localname": "srchDate", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "srchLevel", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "srchCity", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "http://action/", + "localname": "title", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "http://noun/", + "localname": "title", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "decimal", + "namespace-uri": "http://example.com", + "localname": "rating", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "http://example.com", + "localname": "scoville", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "http://purl.org/dc/elements/1.1/", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "city", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "decimal", + "namespace-uri": "http://test.aggr.com", + "localname": "score", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "http://test.tups.com", + "localname": "rate", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "title", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "popularity", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "double", + "namespace-uri": "", + "localname": "distance", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "namespace-uri": "", + "localname": "bday", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "height1", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "date", + "namespace-uri": "", + "localname": "height2", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "cityName", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "cityTeam", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "long", + "namespace-uri": "", + "localname": "cityPopulation", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "long", + "namespace-uri": "", + "localname": "inventory", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "int", + "namespace-uri": "", + "localname": "id", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "namespace-uri": "", + "localname": "animal", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "reject" + }, { "scalar-type": "int", "namespace-uri": "", @@ -241,300 +241,300 @@ "invalid-values": "reject" } ], - "element-word-lexicon": [ - { - "namespace-uri": "", - "localname": "city", - "collation": "http://marklogic.com/collation/" - } - ], - "geospatial-element-index": [ - { - "namespace-uri": "", - "localname": "latLonPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "range-path-index": [ - { - "scalar-type": "string", - "path-expression": "/Employee/fn", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "int", - "path-expression": "/root/popularity", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "decimal", - "path-expression": "//@amt", - "collation": "", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "/doc/name", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.functionaltest.Artifact/name", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "string", - "path-expression": "com.marklogic.client.functionaltest.Artifact/manufacturer/com.marklogic.client.functionaltest.Company/name", - "collation": "http://marklogic.com/collation/", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "scalar-type": "long", - "path-expression": "com.marklogic.client.functionaltest.Artifact/inventory", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "path-expression": "com.marklogic.client.functionaltest.ArtifactIndexedOnCalendar/expiryDate", - "collation": "", - "range-value-positions": true, - "invalid-values": "reject" - } - ], - "range-element-attribute-index": [ - { - "scalar-type": "decimal", - "parent-namespace-uri": "http://cloudbank.com", - "parent-localname": "price", - "namespace-uri": "", - "localname": "amt", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "scalar-type": "dateTime", - "parent-namespace-uri": "http://example.com", - "parent-localname": "entry", - "namespace-uri": "", - "localname": "date", - "collation": "", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-path-index": [ - { - "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": "false", - "invalid-values": "ignore" - } - ], - "field": [ - { - "field-name": "", - "include-root": true - }, - { - "field-name": "pop", - "include-root": false, - "included-element": [ - { - "namespace-uri": "", - "localname": "popularity", - "weight": 2, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - }, - { - "field-name": "para", - "include-root": false, - "included-element": [ - { - "namespace-uri": "", - "localname": "p", - "weight": 5, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - }, - { - "field-name": "description", - "include-root": true, - "included-element": [ - { - "namespace-uri": "", - "localname": "description", - "weight": 1, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - }, - { - "field-name": "bbqtext", - "include-root": true, - "included-element": [ - { - "namespace-uri": "http://example.com", - "localname": "title", - "weight": 1, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - }, - { - "namespace-uri": "http://example.com", - "localname": "abstract", - "weight": 1, - "attribute-namespace-uri": "", - "attribute-localname": "", - "attribute-value": "" - } - ] - } - ], - "geospatial-element-index": [ - { - "namespace-uri": "", - "localname": "latLonPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - }, - { - "namespace-uri": "", - "localname": "g-elem-point", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-child-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "g-elem-child-parent", - "namespace-uri": "", - "localname": "g-elem-child-point", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-pair-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "com.marklogic.client.functionaltest.GeoCompany", - "latitude-namespace-uri": "", - "latitude-localname": "latitude", - "longitude-namespace-uri": "", - "longitude-localname": "longitude", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "parent-namespace-uri": "", - "parent-localname": "g-elem-pair", - "latitude-namespace-uri": "", - "latitude-localname": "lat", - "longitude-namespace-uri": "", - "longitude-localname": "long", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-element-attribute-pair-index": [ - { - "parent-namespace-uri": "", - "parent-localname": "g-attr-pair", - "latitude-namespace-uri": "", - "latitude-localname": "lat", - "longitude-namespace-uri": "", - "longitude-localname": "long", - "coordinate-system": "wgs84", - "range-value-positions": false, - "invalid-values": "reject" - } - ], - "geospatial-path-index": [ - { - "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "ignore" - }, - { - "path-expression": "/doc/g-elem-point", - "coordinate-system": "wgs84", - "point-format": "point", - "range-value-positions": false, - "invalid-values": "ignore" - } - ], - "geospatial-region-path-index": [ - { - "path-expression": "/root/item/linestring", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/point", - "coordinate-system": "wgs84", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/circle", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/box", - "coordinate-system": "wgs84/double", - "units": "miles", - "geohash-precision": 2, - "invalid-values": "reject" - }, - { - "path-expression": "/root/item/polygon", - "coordinate-system": "wgs84", - "units": "miles", - "geohash-precision": 3, - "invalid-values": "reject" - } - ] + "element-word-lexicon": [ + { + "namespace-uri": "", + "localname": "city", + "collation": "http://marklogic.com/collation/" + } + ], + "geospatial-element-index": [ + { + "namespace-uri": "", + "localname": "latLonPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "range-path-index": [ + { + "scalar-type": "string", + "path-expression": "/Employee/fn", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "int", + "path-expression": "/root/popularity", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "decimal", + "path-expression": "//@amt", + "collation": "", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "/doc/name", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.functionaltest.Artifact/name", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "string", + "path-expression": "com.marklogic.client.functionaltest.Artifact/manufacturer/com.marklogic.client.functionaltest.Company/name", + "collation": "http://marklogic.com/collation/", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "scalar-type": "long", + "path-expression": "com.marklogic.client.functionaltest.Artifact/inventory", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "path-expression": "com.marklogic.client.functionaltest.ArtifactIndexedOnCalendar/expiryDate", + "collation": "", + "range-value-positions": true, + "invalid-values": "reject" + } + ], + "range-element-attribute-index": [ + { + "scalar-type": "decimal", + "parent-namespace-uri": "http://cloudbank.com", + "parent-localname": "price", + "namespace-uri": "", + "localname": "amt", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "scalar-type": "dateTime", + "parent-namespace-uri": "http://example.com", + "parent-localname": "entry", + "namespace-uri": "", + "localname": "date", + "collation": "", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-path-index": [ + { + "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": "false", + "invalid-values": "ignore" + } + ], + "field": [ + { + "field-name": "", + "include-root": true + }, + { + "field-name": "pop", + "include-root": false, + "included-element": [ + { + "namespace-uri": "", + "localname": "popularity", + "weight": 2, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + }, + { + "field-name": "para", + "include-root": false, + "included-element": [ + { + "namespace-uri": "", + "localname": "p", + "weight": 5, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + }, + { + "field-name": "description", + "include-root": true, + "included-element": [ + { + "namespace-uri": "", + "localname": "description", + "weight": 1, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + }, + { + "field-name": "bbqtext", + "include-root": true, + "included-element": [ + { + "namespace-uri": "http://example.com", + "localname": "title", + "weight": 1, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + }, + { + "namespace-uri": "http://example.com", + "localname": "abstract", + "weight": 1, + "attribute-namespace-uri": "", + "attribute-localname": "", + "attribute-value": "" + } + ] + } + ], + "geospatial-element-index": [ + { + "namespace-uri": "", + "localname": "latLonPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + }, + { + "namespace-uri": "", + "localname": "g-elem-point", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-child-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "g-elem-child-parent", + "namespace-uri": "", + "localname": "g-elem-child-point", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-pair-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "com.marklogic.client.functionaltest.GeoCompany", + "latitude-namespace-uri": "", + "latitude-localname": "latitude", + "longitude-namespace-uri": "", + "longitude-localname": "longitude", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "parent-namespace-uri": "", + "parent-localname": "g-elem-pair", + "latitude-namespace-uri": "", + "latitude-localname": "lat", + "longitude-namespace-uri": "", + "longitude-localname": "long", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-element-attribute-pair-index": [ + { + "parent-namespace-uri": "", + "parent-localname": "g-attr-pair", + "latitude-namespace-uri": "", + "latitude-localname": "lat", + "longitude-namespace-uri": "", + "longitude-localname": "long", + "coordinate-system": "wgs84", + "range-value-positions": false, + "invalid-values": "reject" + } + ], + "geospatial-path-index": [ + { + "path-expression": "com.marklogic.client.functionaltest.GeoCompany/latlongPoint", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "ignore" + }, + { + "path-expression": "/doc/g-elem-point", + "coordinate-system": "wgs84", + "point-format": "point", + "range-value-positions": false, + "invalid-values": "ignore" + } + ], + "geospatial-region-path-index": [ + { + "path-expression": "/root/item/linestring", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/point", + "coordinate-system": "wgs84", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/circle", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/box", + "coordinate-system": "wgs84/double", + "units": "miles", + "geohash-precision": 2, + "invalid-values": "reject" + }, + { + "path-expression": "/root/item/polygon", + "coordinate-system": "wgs84", + "units": "miles", + "geohash-precision": 3, + "invalid-values": "reject" + } + ] } diff --git a/test-app/src/main/ml-config/databases/func-schemas-database.json b/test-app/src/main/ml-config/databases/func-schemas-database.json index 50c471e93..5a694f552 100644 --- a/test-app/src/main/ml-config/databases/func-schemas-database.json +++ b/test-app/src/main/ml-config/databases/func-schemas-database.json @@ -1,3 +1,3 @@ { - "database-name": "java-functest-schemas" -} \ No newline at end of file + "database-name": "java-functest-schemas" +} diff --git a/test-app/src/main/ml-config/databases/modules-database.json b/test-app/src/main/ml-config/databases/modules-database.json index 35657f4da..3d747b864 100644 --- a/test-app/src/main/ml-config/databases/modules-database.json +++ b/test-app/src/main/ml-config/databases/modules-database.json @@ -1,3 +1,3 @@ { - "database-name": "%%MODULES_DATABASE%%" -} \ No newline at end of file + "database-name": "%%MODULES_DATABASE%%" +} diff --git a/test-app/src/main/ml-config/databases/schemas-database.json b/test-app/src/main/ml-config/databases/schemas-database.json index f0f14c748..582808531 100644 --- a/test-app/src/main/ml-config/databases/schemas-database.json +++ b/test-app/src/main/ml-config/databases/schemas-database.json @@ -1,3 +1,3 @@ { - "database-name": "%%SCHEMAS_DATABASE%%" -} \ No newline at end of file + "database-name": "%%SCHEMAS_DATABASE%%" +} diff --git a/test-app/src/main/ml-data/optic/test/collections.properties b/test-app/src/main/ml-data/optic/test/collections.properties index ee57b7b93..6ba0c7efd 100644 --- a/test-app/src/main/ml-data/optic/test/collections.properties +++ b/test-app/src/main/ml-data/optic/test/collections.properties @@ -1 +1 @@ -*=/optic/test,/optic/music +*=/optic/test,/optic/music,test-data diff --git a/test-app/src/main/ml-data/optic/vectors/collections.properties b/test-app/src/main/ml-data/optic/vectors/collections.properties new file mode 100644 index 000000000..a777f445c --- /dev/null +++ b/test-app/src/main/ml-data/optic/vectors/collections.properties @@ -0,0 +1 @@ +*=test-data diff --git a/test-app/src/main/ml-data/optic/zipcodes/collections.properties b/test-app/src/main/ml-data/optic/zipcodes/collections.properties index 0b92404e2..175de43d4 100644 --- a/test-app/src/main/ml-data/optic/zipcodes/collections.properties +++ b/test-app/src/main/ml-data/optic/zipcodes/collections.properties @@ -1 +1 @@ -*=zipcode +*=zipcode,test-data diff --git a/test-app/src/main/ml-data/sample/collections.properties b/test-app/src/main/ml-data/sample/collections.properties index e41971f1e..ffb43069a 100644 --- a/test-app/src/main/ml-data/sample/collections.properties +++ b/test-app/src/main/ml-data/sample/collections.properties @@ -1,5 +1,6 @@ -suggestion.xml=http://some.org/suggestions -first.xml=http://some.org/collection1,http://some.org/collection2 -lexicon-test1.xml=http://some.org/collection1,http://some.org/collection2 -lexicon-test2.xml=http://some.org/collection1,http://some.org/collection2 -second.txt=document-format-query-test +*=test-data +suggestion.xml=http://some.org/suggestions,test-data +first.xml=http://some.org/collection1,http://some.org/collection2,test-data +lexicon-test1.xml=http://some.org/collection1,http://some.org/collection2,test-data +lexicon-test2.xml=http://some.org/collection1,http://some.org/collection2,test-data +second.txt=document-format-query-test,test-data diff --git a/test-app/src/main/ml-data/sample2/collections.properties b/test-app/src/main/ml-data/sample2/collections.properties index e7d83da23..ebdc32646 100644 --- a/test-app/src/main/ml-data/sample2/collections.properties +++ b/test-app/src/main/ml-data/sample2/collections.properties @@ -1 +1 @@ -*=http://some.org/suggestions +*=http://some.org/suggestions,test-data